diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 5c5a46261..b66b7b0b4 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -113,8 +113,8 @@ func configHelper(name string) { type FsDropbox struct { db *dropbox.Dropbox // the connection to the dropbox server root string // the path we are working on - slashRoot string // root with "/" prefix - slashRootSlash string // root with "/" prefix and postix + slashRoot string // root with "/" prefix, lowercase + slashRootSlash string // root with "/" prefix and postfix, lowercase datastoreManager *dropbox.DatastoreManager datastore *dropbox.Datastore table *dropbox.Table @@ -196,9 +196,11 @@ func NewFs(name, root string) (fs.Fs, error) { // Sets root in f func (f *FsDropbox) setRoot(root string) { f.root = strings.Trim(root, "/") - f.slashRoot = "/" + f.root + lowerCaseRoot := strings.ToLower(f.root) + + f.slashRoot = "/" + lowerCaseRoot f.slashRootSlash = f.slashRoot - if f.root != "" { + if lowerCaseRoot != "" { f.slashRootSlash += "/" } } @@ -254,17 +256,26 @@ func (f *FsDropbox) NewFsObject(remote string) fs.Object { return f.newFsObjectWithInfo(remote, nil) } -// Strips the root off entry and returns it -func (f *FsDropbox) stripRoot(entry *dropbox.Entry) string { - path := entry.Path - if strings.HasPrefix(path, f.slashRootSlash) { - path = path[len(f.slashRootSlash):] +// Strips the root off path and returns it +func (f *FsDropbox) stripRoot(path string) *string { + lowercase := strings.ToLower(path) + + if !strings.HasPrefix(lowercase, f.slashRootSlash) { + fs.Stats.Error() + fs.Log(f, "Path '%s' is not under root '%s'", path, f.slashRootSlash) + return nil } - return path + + stripped := path[len(f.slashRootSlash):] + return &stripped } // Walk the root returning a channel of FsObjects func (f *FsDropbox) list(out fs.ObjectsChan) { + // Track path component case, it could be different for entries coming from DropBox API + // See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us + // and https://github.com/ncw/rclone/issues/53 + nameTree := NewNameTree() cursor := "" for { deltaPage, err := f.db.Delta(cursor, f.slashRoot) @@ -291,13 +302,38 @@ func (f *FsDropbox) list(out fs.ObjectsChan) { fs.Debug(f, "Failed to delete metadata for %q", deltaEntry.Path) // Don't accumulate Error here } - } else { - if entry.IsDir { - // ignore directories + if len(entry.Path) <= 1 || entry.Path[0] != '/' { + fs.Stats.Error() + fs.Log(f, "dropbox API inconsistency: a path should always start with a slash and be at least 2 characters: %s", entry.Path) + continue + } + + lastSlashIndex := strings.LastIndex(entry.Path, "/") + + var parentPath string + if lastSlashIndex == 0 { + parentPath = "" } else { - path := f.stripRoot(entry) - out <- f.newFsObjectWithInfo(path, entry) + parentPath = entry.Path[1:lastSlashIndex] + } + lastComponent := entry.Path[lastSlashIndex+1:] + + if entry.IsDir { + nameTree.PutCaseCorrectDirectoryName(parentPath, lastComponent) + } else { + parentPathCorrectCase := nameTree.GetPathWithCorrectCase(parentPath) + if parentPathCorrectCase != nil { + path := f.stripRoot(*parentPathCorrectCase + "/" + lastComponent) + if path == nil { + // an error occurred and logged by stripRoot + continue + } + + out <- f.newFsObjectWithInfo(*path, entry) + } else { + nameTree.PutFile(parentPath, lastComponent, entry) + } } } } @@ -307,6 +343,17 @@ func (f *FsDropbox) list(out fs.ObjectsChan) { cursor = deltaPage.Cursor.Cursor } } + + walkFunc := func(caseCorrectFilePath string, entry *dropbox.Entry) { + path := f.stripRoot("/" + caseCorrectFilePath) + if path == nil { + // an error occurred and logged by stripRoot + return + } + + out <- f.newFsObjectWithInfo(*path, entry) + } + nameTree.WalkFiles(f.root, walkFunc) } // Walk the path returning a channel of FsObjects @@ -332,8 +379,14 @@ func (f *FsDropbox) ListDir() fs.DirChan { for i := range entry.Contents { entry := &entry.Contents[i] if entry.IsDir { + name := f.stripRoot(entry.Path) + if name == nil { + // an error occurred and logged by stripRoot + continue + } + out <- &fs.Dir{ - Name: f.stripRoot(entry), + Name: *name, When: time.Time(entry.ClientMtime), Bytes: int64(entry.Bytes), Count: -1, diff --git a/dropbox/nametree.go b/dropbox/nametree.go new file mode 100644 index 000000000..b51ff6b45 --- /dev/null +++ b/dropbox/nametree.go @@ -0,0 +1,179 @@ +package dropbox + +import ( + "bytes" + "fmt" + "github.com/ncw/rclone/fs" + "github.com/stacktic/dropbox" + "strings" +) + +type NameTreeNode struct { + // Map from lowercase directory name to tree node + Directories map[string]*NameTreeNode + + // Map from file name (case sensitive) to dropbox entry + Files map[string]*dropbox.Entry + + // Empty string if exact case is unknown or root node + CaseCorrectName string +} + +// ------------------------------------------------------------ + +func newNameTreeNode(caseCorrectName string) *NameTreeNode { + return &NameTreeNode{ + CaseCorrectName: caseCorrectName, + Directories: make(map[string]*NameTreeNode), + Files: make(map[string]*dropbox.Entry), + } +} + +func NewNameTree() *NameTreeNode { + return newNameTreeNode("") +} + +func (tree *NameTreeNode) String() string { + if len(tree.CaseCorrectName) == 0 { + return "NameTreeNode/" + } else { + return fmt.Sprintf("NameTreeNode/%q", tree.CaseCorrectName) + } +} + +func (tree *NameTreeNode) getTreeNode(path string) *NameTreeNode { + if len(path) == 0 { + // no lookup required, just return root + return tree + } + + current := tree + for _, component := range strings.Split(path, "/") { + if len(component) == 0 { + fs.Stats.Error() + fs.Log(tree, "getTreeNode: path component is empty (full path %q)", path) + return nil + } + + lowercase := strings.ToLower(component) + + lookup := current.Directories[lowercase] + if lookup == nil { + lookup = newNameTreeNode("") + current.Directories[lowercase] = lookup + } + + current = lookup + } + + return current +} + +func (tree *NameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) { + if len(caseCorrectDirectoryName) == 0 { + fs.Stats.Error() + fs.Log(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath) + return + } + + node := tree.getTreeNode(parentPath) + if node == nil { + return + } + + lowerCaseDirectoryName := strings.ToLower(caseCorrectDirectoryName) + directory := node.Directories[lowerCaseDirectoryName] + if directory == nil { + directory = newNameTreeNode(caseCorrectDirectoryName) + node.Directories[lowerCaseDirectoryName] = directory + } else { + if len(directory.CaseCorrectName) > 0 { + fs.Stats.Error() + fs.Log(tree, "PutCaseCorrectDirectoryName: directory %q is already exists under parent path %q", caseCorrectDirectoryName, parentPath) + return + } + + directory.CaseCorrectName = caseCorrectDirectoryName + } +} + +func (tree *NameTreeNode) PutFile(parentPath string, caseCorrectFileName string, dropboxEntry *dropbox.Entry) { + node := tree.getTreeNode(parentPath) + if node == nil { + return + } + + if node.Files[caseCorrectFileName] != nil { + fs.Stats.Error() + fs.Log(tree, "PutFile: file %q is already exists at %q", caseCorrectFileName, parentPath) + return + } + + node.Files[caseCorrectFileName] = dropboxEntry +} + +func (tree *NameTreeNode) GetPathWithCorrectCase(path string) *string { + if path == "" { + empty := "" + return &empty + } + + var result bytes.Buffer + + current := tree + for _, component := range strings.Split(path, "/") { + if component == "" { + fs.Stats.Error() + fs.Log(tree, "GetPathWithCorrectCase: path component is empty (full path %q)", path) + return nil + } + + lowercase := strings.ToLower(component) + + current = current.Directories[lowercase] + if current == nil || current.CaseCorrectName == "" { + return nil + } + + result.WriteString("/") + result.WriteString(current.CaseCorrectName) + } + + resultString := result.String() + return &resultString +} + +type NameTreeFileWalkFunc func(caseCorrectFilePath string, entry *dropbox.Entry) + +func (tree *NameTreeNode) walkFilesRec(currentPath string, walkFunc NameTreeFileWalkFunc) { + var prefix string + if currentPath == "" { + prefix = "" + } else { + prefix = currentPath + "/" + } + + for name, entry := range tree.Files { + walkFunc(prefix+name, entry) + } + + for lowerCaseName, directory := range tree.Directories { + caseCorrectName := directory.CaseCorrectName + if caseCorrectName == "" { + fs.Stats.Error() + fs.Log(tree, "WalkFiles: exact name of the directory %q is unknown (parent path: %q)", lowerCaseName, currentPath) + continue + } + + directory.walkFilesRec(prefix+caseCorrectName, walkFunc) + } +} + +func (tree *NameTreeNode) WalkFiles(rootPath string, walkFunc NameTreeFileWalkFunc) { + node := tree.getTreeNode(rootPath) + if node == nil { + return + } + + node.walkFilesRec(rootPath, walkFunc) +} diff --git a/dropbox/nametree_test.go b/dropbox/nametree_test.go new file mode 100644 index 000000000..d97b461d9 --- /dev/null +++ b/dropbox/nametree_test.go @@ -0,0 +1,124 @@ +package dropbox_test + +import ( + "github.com/ncw/rclone/dropbox" + "github.com/ncw/rclone/fs" + dropboxapi "github.com/stacktic/dropbox" + "testing" +) + +func assert(t *testing.T, shouldBeTrue bool, failMessage string) { + if !shouldBeTrue { + t.Fatal(failMessage) + } +} + +func TestPutCaseCorrectDirectoryName(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := dropbox.NewNameTree() + tree.PutCaseCorrectDirectoryName("a/b", "C") + + assert(t, tree.CaseCorrectName == "", "Root CaseCorrectName should be empty") + + a := tree.Directories["a"] + assert(t, a.CaseCorrectName == "", "CaseCorrectName at 'a' should be empty") + + b := a.Directories["b"] + assert(t, b.CaseCorrectName == "", "CaseCorrectName at 'a/b' should be empty") + + c := b.Directories["c"] + assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'a/b/c' should be 'C'") + + assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") +} + +func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := dropbox.NewNameTree() + tree.PutCaseCorrectDirectoryName("/a", "C") + tree.PutCaseCorrectDirectoryName("b/", "C") + tree.PutCaseCorrectDirectoryName("a//b", "C") + + assert(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported") +} + +func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := dropbox.NewNameTree() + tree.PutCaseCorrectDirectoryName("", "C") + + c := tree.Directories["c"] + assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'") + + assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") +} + +func TestGetPathWithCorrectCase(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := dropbox.NewNameTree() + tree.PutCaseCorrectDirectoryName("a", "C") + assert(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available") + + tree.PutCaseCorrectDirectoryName("", "A") + assert(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'") + + assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") +} + +func TestPutAndWalk(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := dropbox.NewNameTree() + tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"}) + tree.PutCaseCorrectDirectoryName("", "A") + + numCalled := 0 + walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) { + assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath) + assert(t, entry.Path == "xxx", "entry.Path should be xxx") + numCalled++ + } + tree.WalkFiles("", walkFunc) + + assert(t, numCalled == 1, "walk func should be called only once") + + assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") +} + +func TestPutAndWalkWithPrefix(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := dropbox.NewNameTree() + tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"}) + tree.PutCaseCorrectDirectoryName("", "A") + + numCalled := 0 + walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) { + assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath) + assert(t, entry.Path == "xxx", "entry.Path should be xxx") + numCalled++ + } + tree.WalkFiles("A", walkFunc) + + assert(t, numCalled == 1, "walk func should be called only once") + + assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") +} + +func TestPutAndWalkIncompleteTree(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := dropbox.NewNameTree() + tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"}) + + walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) { + t.Fatal("Should not be called") + } + tree.WalkFiles("", walkFunc) + + assert(t, fs.Stats.GetErrors() == errors+1, "One error should be reported") +}