mirror of
https://github.com/rclone/rclone.git
synced 2024-12-23 07:29:35 +01:00
cache: cache lists using batch writes
This commit is contained in:
parent
d758e1908e
commit
8a84975993
14
backend/cache/cache.go
vendored
14
backend/cache/cache.go
vendored
@ -725,6 +725,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
fs.Debugf(dir, "list: source entries: %v", entries)
|
||||
|
||||
// and then iterate over the ones from source (temp Objects will override source ones)
|
||||
var batchDirectories []*Directory
|
||||
for _, entry := range entries {
|
||||
switch o := entry.(type) {
|
||||
case fs.Object:
|
||||
@ -746,18 +747,19 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
cdd := DirectoryFromOriginal(f, o)
|
||||
// check if the dir isn't expired and add it in cache if it isn't
|
||||
if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(f.fileAge)) {
|
||||
err := f.cache.AddDir(cdd)
|
||||
if err != nil {
|
||||
fs.Errorf(dir, "list: error caching dir from listing %v", o)
|
||||
} else {
|
||||
fs.Debugf(dir, "list: cached dir: %v", cdd)
|
||||
}
|
||||
batchDirectories = append(batchDirectories, cdd)
|
||||
}
|
||||
cachedEntries = append(cachedEntries, cdd)
|
||||
default:
|
||||
fs.Debugf(entry, "list: Unknown object type %T", entry)
|
||||
}
|
||||
}
|
||||
err = f.cache.AddBatchDir(batchDirectories)
|
||||
if err != nil {
|
||||
fs.Errorf(dir, "list: error caching directories from listing %v", dir)
|
||||
} else {
|
||||
fs.Debugf(dir, "list: cached directories: %v", len(batchDirectories))
|
||||
}
|
||||
|
||||
// cache dir meta
|
||||
t := time.Now()
|
||||
|
34
backend/cache/storage_persistent.go
vendored
34
backend/cache/storage_persistent.go
vendored
@ -192,19 +192,35 @@ func (b *Persistent) GetDir(remote string) (*Directory, error) {
|
||||
|
||||
// AddDir will update a CachedDirectory metadata and all its entries
|
||||
func (b *Persistent) AddDir(cachedDir *Directory) error {
|
||||
return b.AddBatchDir([]*Directory{cachedDir})
|
||||
}
|
||||
|
||||
// AddBatchDir will update a list of CachedDirectory metadata and all their entries
|
||||
func (b *Persistent) AddBatchDir(cachedDirs []*Directory) error {
|
||||
if len(cachedDirs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return b.db.Update(func(tx *bolt.Tx) error {
|
||||
bucket := b.getBucket(cachedDir.abs(), true, tx)
|
||||
bucket := b.getBucket(cachedDirs[0].Dir, true, tx)
|
||||
if bucket == nil {
|
||||
return errors.Errorf("couldn't open bucket (%v)", cachedDir)
|
||||
return errors.Errorf("couldn't open bucket (%v)", cachedDirs[0].Dir)
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(cachedDir)
|
||||
if err != nil {
|
||||
return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
|
||||
}
|
||||
err = bucket.Put([]byte("."), encoded)
|
||||
if err != nil {
|
||||
return err
|
||||
for _, cachedDir := range cachedDirs {
|
||||
b, err := bucket.CreateBucketIfNotExists([]byte(cachedDir.Name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(cachedDir)
|
||||
if err != nil {
|
||||
return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
|
||||
}
|
||||
err = b.Put([]byte("."), encoded)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
Loading…
Reference in New Issue
Block a user