mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 00:13:49 +01:00
dropbox: remove metadata when we remove files
This commit is contained in:
parent
b185e104ed
commit
f8bb0d9cc8
@ -6,7 +6,10 @@ Limitations of dropbox
|
||||
|
||||
File system is case insensitive
|
||||
|
||||
FIXME need to delete metadata when we delete files!
|
||||
The datastore is limited to 100,000 records which therefore is the
|
||||
limit of the number of files that rclone can use on dropbox.
|
||||
|
||||
FIXME only open datastore if we need it?
|
||||
|
||||
FIXME Getting this sometimes
|
||||
Failed to copy: Upload failed: invalid character '<' looking for beginning of value
|
||||
@ -261,14 +264,22 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
|
||||
deltaEntry := &deltaPage.Entries[i]
|
||||
entry := deltaEntry.Entry
|
||||
if entry == nil {
|
||||
// This notifies of a deleted object which we ignore
|
||||
continue
|
||||
}
|
||||
if entry.IsDir {
|
||||
// ignore directories
|
||||
// This notifies of a deleted object
|
||||
fs.Debug(f, "Deleting metadata for %q", deltaEntry.Path)
|
||||
key := metadataKey(deltaEntry.Path) // Path is lowercased
|
||||
err := f.deleteMetadata(key)
|
||||
if err != nil {
|
||||
fs.Debug(f, "Failed to delete metadata for %q", deltaEntry.Path)
|
||||
// Don't accumulate Error here
|
||||
}
|
||||
|
||||
} else {
|
||||
path := f.stripRoot(entry)
|
||||
out <- f.NewFsObjectWithInfo(path, entry)
|
||||
if entry.IsDir {
|
||||
// ignore directories
|
||||
} else {
|
||||
path := f.stripRoot(entry)
|
||||
out <- f.NewFsObjectWithInfo(path, entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !deltaPage.HasMore {
|
||||
@ -375,8 +386,26 @@ func (fs *FsDropbox) Precision() time.Duration {
|
||||
|
||||
// Purge deletes all the files and the container
|
||||
//
|
||||
// Returns an error if it isn't empty
|
||||
// Optional interface: Only implement this if you have a way of
|
||||
// deleting all the files quicker than just running Remove() on the
|
||||
// result of List()
|
||||
func (f *FsDropbox) Purge() error {
|
||||
// Delete metadata first
|
||||
var wg sync.WaitGroup
|
||||
to_be_deleted := f.List()
|
||||
wg.Add(fs.Config.Transfers)
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for dst := range to_be_deleted {
|
||||
o := dst.(*FsObjectDropbox)
|
||||
o.deleteMetadata()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Let dropbox delete the filesystem tree
|
||||
_, err := f.db.Delete(f.slashRoot)
|
||||
return err
|
||||
}
|
||||
@ -409,6 +438,21 @@ func (f *FsDropbox) transaction(fn func() error) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deletes the medadata associated with this key
|
||||
func (f *FsDropbox) deleteMetadata(key string) error {
|
||||
return f.transaction(func() error {
|
||||
record, err := f.table.Get(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Couldn't get record: %s", err)
|
||||
}
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
record.DeleteRecord()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// Reads the record attached to key
|
||||
//
|
||||
// Holds datastore mutex while in progress
|
||||
@ -513,11 +557,16 @@ func (o *FsObjectDropbox) remotePath() string {
|
||||
return o.dropbox.slashRootSlash + o.remote
|
||||
}
|
||||
|
||||
// Returns the key for the metadata database for a given path
|
||||
func metadataKey(path string) string {
|
||||
// NB File system is case insensitive
|
||||
path = strings.ToLower(path)
|
||||
return fmt.Sprintf("%x", md5.Sum([]byte(path)))
|
||||
}
|
||||
|
||||
// Returns the key for the metadata database
|
||||
func (o *FsObjectDropbox) metadataKey() string {
|
||||
// NB File system is case insensitive
|
||||
key := strings.ToLower(o.remotePath())
|
||||
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
|
||||
return metadataKey(o.remotePath())
|
||||
}
|
||||
|
||||
// readMetaData gets the info if it hasn't already been fetched
|
||||
@ -619,6 +668,18 @@ func (o *FsObjectDropbox) setModTimeAndMd5sum(modTime time.Time, md5sum string)
|
||||
})
|
||||
}
|
||||
|
||||
// Deletes the medadata associated with this file
|
||||
//
|
||||
// It logs any errors
|
||||
func (o *FsObjectDropbox) deleteMetadata() {
|
||||
fs.Debug(o, "Deleting metadata from datastore")
|
||||
err := o.dropbox.deleteMetadata(o.metadataKey())
|
||||
if err != nil {
|
||||
fs.Log(o, "Error deleting metadata: %v", err)
|
||||
fs.Stats.Error()
|
||||
}
|
||||
}
|
||||
|
||||
// Sets the modification time of the local fs object
|
||||
//
|
||||
// Commits the datastore
|
||||
@ -662,6 +723,7 @@ func (o *FsObjectDropbox) Update(in io.Reader, modTime time.Time, size int64) er
|
||||
|
||||
// Remove an object
|
||||
func (o *FsObjectDropbox) Remove() error {
|
||||
o.deleteMetadata()
|
||||
_, err := o.dropbox.db.Delete(o.remotePath())
|
||||
return err
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user