cache: fix dir cache issue - #2117

backend/cache/cache.go

@@ -644,13 +644,15 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			fs.Debugf(dir, "list: cached object: %v", co)
 		case fs.Directory:
 			cdd := DirectoryFromOriginal(f, o)
-			// FIXME this overrides a possible expired dir
-			//err := f.cache.AddDir(cdd)
-			//if err != nil {
-			//	fs.Errorf(dir, "list: error caching dir from listing %v", o)
-			//} else {
-			//	fs.Debugf(dir, "list: cached dir: %v", cdd)
-			//}
+			// check if the dir isn't expired and add it in cache if it isn't
+			if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(f.fileAge)) {
+				err := f.cache.AddDir(cdd)
+				if err != nil {
+					fs.Errorf(dir, "list: error caching dir from listing %v", o)
+				} else {
+					fs.Debugf(dir, "list: cached dir: %v", cdd)
+				}
+			}
 			cachedEntries = append(cachedEntries, cdd)
 		default:
 			fs.Debugf(entry, "list: Unknown object type %T", entry)

backend/cache/cache_internal_test.go

@@ -97,6 +97,7 @@ func TestMain(m *testing.M) {
 	goflag.Parse()
 	var rc int
 
+	log.Printf("Running with the following params: \n remote: %v, \n mount: %v", remoteName, useMount)
 	runInstance = newRun()
 	rc = m.Run()
 	os.Exit(rc)
@@ -697,6 +698,58 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestInternalBug2117(t *testing.T) {
+	vfsflags.Opt.DirCacheTime = time.Second * 10
+
+	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil,
+		map[string]string{"cache-info-age": "72h", "cache-chunk-clean-interval": "15m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	if runInstance.rootIsCrypt {
+		t.Skipf("skipping crypt")
+	}
+
+	cfs, err := runInstance.getCacheFs(rootFs)
+	require.NoError(t, err)
+
+	err = cfs.UnWrap().Mkdir("test")
+	require.NoError(t, err)
+	for i := 1; i <= 4; i++ {
+		err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d", i))
+		require.NoError(t, err)
+
+		for j := 1; j <= 4; j++ {
+			err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d/dir%d", i, j))
+			require.NoError(t, err)
+
+			runInstance.writeObjectString(t, cfs.UnWrap(), fmt.Sprintf("test/dir%d/dir%d/test.txt", i, j), "test")
+		}
+	}
+
+	di, err := runInstance.list(t, rootFs, "test/dir1/dir2")
+	require.NoError(t, err)
+	log.Printf("len: %v", len(di))
+	require.Len(t, di, 1)
+
+	time.Sleep(time.Second * 30)
+
+	di, err = runInstance.list(t, rootFs, "test/dir1/dir2")
+	require.NoError(t, err)
+	log.Printf("len: %v", len(di))
+	require.Len(t, di, 1)
+
+	di, err = runInstance.list(t, rootFs, "test/dir1")
+	require.NoError(t, err)
+	log.Printf("len: %v", len(di))
+	require.Len(t, di, 4)
+
+	di, err = runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	log.Printf("len: %v", len(di))
+	require.Len(t, di, 4)
+}
+
 func TestInternalUploadTempDirCreated(t *testing.T) {
 	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,

backend/cache/storage_persistent.go

@@ -169,6 +169,27 @@ func (b *Persistent) getBucket(dir string, createIfMissing bool, tx *bolt.Tx) *b
 	return bucket
 }
 
+// GetDir will retrieve data of a cached directory
+func (b *Persistent) GetDir(remote string) (*Directory, error) {
+	cd := &Directory{}
+
+	err := b.db.View(func(tx *bolt.Tx) error {
+		bucket := b.getBucket(remote, false, tx)
+		if bucket == nil {
+			return errors.Errorf("couldn't open bucket (%v)", remote)
+		}
+
+		data := bucket.Get([]byte("."))
+		if data != nil {
+			return json.Unmarshal(data, cd)
+		}
+
+		return errors.Errorf("%v not found", remote)
+	})
+
+	return cd, err
+}
+
 // AddDir will update a CachedDirectory metadata and all its entries
 func (b *Persistent) AddDir(cachedDir *Directory) error {
 	return b.db.Update(func(tx *bolt.Tx) error {