vfs: keep track of directories in the cache also #1860

This makes managing empty directories more reliable.

vfs/cache_test.go

@@ -1,10 +1,13 @@
 package vfs
 
 import (
+	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -44,6 +47,18 @@ func TestCacheModeType(t *testing.T) {
 	assert.Equal(t, "string", m.Type())
 }
 
+// convert c.item to a string
+func itemAsString(c *cache) string {
+	c.itemMu.Lock()
+	defer c.itemMu.Unlock()
+	var out []string
+	for name, item := range c.item {
+		out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d", name, item.isFile, item.opens))
+	}
+	sort.Strings(out)
+	return strings.Join(out, "\n")
+}
+
 func TestCacheNew(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
@@ -56,11 +71,13 @@ func TestCacheNew(t *testing.T) {
 
 	assert.Contains(t, c.root, "vfs")
 	assert.Contains(t, c.f.Root(), filepath.Base(r.Fremote.Root()))
+	assert.Equal(t, "", itemAsString(c))
 
 	// mkdir
 	p, err := c.mkdir("potato")
 	require.NoError(t, err)
 	assert.Equal(t, "potato", filepath.Base(p))
+	assert.Equal(t, `name="" isFile=false opens=0`, itemAsString(c))
 
 	fi, err := os.Stat(filepath.Dir(p))
 	require.NoError(t, err)
@@ -87,7 +104,11 @@ func TestCacheNew(t *testing.T) {
 	assert.Equal(t, 0, item.opens)
 
 	// open
+	assert.Equal(t, `name="" isFile=false opens=0
+name="potato" isFile=true opens=0`, itemAsString(c))
 	c.open("potato")
+	assert.Equal(t, `name="" isFile=false opens=1
+name="potato" isFile=true opens=1`, itemAsString(c))
 	item = c.get("potato")
 	assert.WithinDuration(t, time.Now(), item.atime, time.Second)
 	assert.Equal(t, 1, item.opens)
@@ -107,6 +128,8 @@ func TestCacheNew(t *testing.T) {
 	item.atime = time.Now().Add(-24 * time.Hour)
 	err = c.updateAtimes()
 	require.NoError(t, err)
+	assert.Equal(t, `name="" isFile=false opens=1
+name="potato" isFile=true opens=1`, itemAsString(c))
 	item = c.get("potato")
 	assert.Equal(t, atime, item.atime)
 
@@ -116,8 +139,14 @@ func TestCacheNew(t *testing.T) {
 	assert.NoError(t, err)
 
 	// close
+	assert.Equal(t, `name="" isFile=false opens=1
+name="potato" isFile=true opens=1`, itemAsString(c))
 	c.updateTime("potato", t2)
+	assert.Equal(t, `name="" isFile=false opens=1
+name="potato" isFile=true opens=1`, itemAsString(c))
 	c.close("potato")
+	assert.Equal(t, `name="" isFile=false opens=0
+name="potato" isFile=true opens=0`, itemAsString(c))
 	item = c.get("potato")
 	assert.WithinDuration(t, time.Now(), item.atime, time.Second)
 	assert.Equal(t, 0, item.opens)
@@ -142,3 +171,148 @@ func TestCacheNew(t *testing.T) {
 	_, err = os.Stat(c.root)
 	assert.True(t, os.IsNotExist(err))
 }
+
+func TestCacheOpens(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	require.NoError(t, err)
+
+	assert.Equal(t, "", itemAsString(c))
+	c.open("potato")
+	assert.Equal(t, `name="" isFile=false opens=1
+name="potato" isFile=true opens=1`, itemAsString(c))
+	c.open("potato")
+	assert.Equal(t, `name="" isFile=false opens=2
+name="potato" isFile=true opens=2`, itemAsString(c))
+	c.close("potato")
+	assert.Equal(t, `name="" isFile=false opens=1
+name="potato" isFile=true opens=1`, itemAsString(c))
+	c.close("potato")
+	assert.Equal(t, `name="" isFile=false opens=0
+name="potato" isFile=true opens=0`, itemAsString(c))
+
+	c.open("potato")
+	c.open("a/b/c/d/one")
+	c.open("a/b/c/d/e/two")
+	c.open("a/b/c/d/e/f/three")
+	assert.Equal(t, `name="" isFile=false opens=4
+name="a" isFile=false opens=3
+name="a/b" isFile=false opens=3
+name="a/b/c" isFile=false opens=3
+name="a/b/c/d" isFile=false opens=3
+name="a/b/c/d/e" isFile=false opens=2
+name="a/b/c/d/e/f" isFile=false opens=1
+name="a/b/c/d/e/f/three" isFile=true opens=1
+name="a/b/c/d/e/two" isFile=true opens=1
+name="a/b/c/d/one" isFile=true opens=1
+name="potato" isFile=true opens=1`, itemAsString(c))
+
+	c.close("potato")
+	c.close("a/b/c/d/one")
+	c.close("a/b/c/d/e/two")
+	c.close("a/b/c/d/e/f/three")
+	assert.Equal(t, `name="" isFile=false opens=0
+name="a" isFile=false opens=0
+name="a/b" isFile=false opens=0
+name="a/b/c" isFile=false opens=0
+name="a/b/c/d" isFile=false opens=0
+name="a/b/c/d/e" isFile=false opens=0
+name="a/b/c/d/e/f" isFile=false opens=0
+name="a/b/c/d/e/f/three" isFile=true opens=0
+name="a/b/c/d/e/two" isFile=true opens=0
+name="a/b/c/d/one" isFile=true opens=0
+name="potato" isFile=true opens=0`, itemAsString(c))
+
+}
+
+func TestCacheCacheDir(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	require.NoError(t, err)
+
+	assert.Equal(t, "", itemAsString(c))
+
+	c.cacheDir("dir")
+	assert.Equal(t, `name="" isFile=false opens=0
+name="dir" isFile=false opens=0`, itemAsString(c))
+
+	c.cacheDir("dir/sub")
+	assert.Equal(t, `name="" isFile=false opens=0
+name="dir" isFile=false opens=0
+name="dir/sub" isFile=false opens=0`, itemAsString(c))
+
+	c.cacheDir("dir/sub2/subsub2")
+	assert.Equal(t, `name="" isFile=false opens=0
+name="dir" isFile=false opens=0
+name="dir/sub" isFile=false opens=0
+name="dir/sub2" isFile=false opens=0
+name="dir/sub2/subsub2" isFile=false opens=0`, itemAsString(c))
+}
+
+func TestCachePurgeOld(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	require.NoError(t, err)
+
+	// Test funcs
+	var removed []string
+	removedDir := true
+	removeFile := func(name string) {
+		removed = append(removed, name)
+	}
+	removeDir := func(name string) bool {
+		if removedDir {
+			removed = append(removed, name+"/")
+		}
+		return removedDir
+	}
+
+	removed = nil
+	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string(nil), removed)
+
+	c.open("sub/dir/potato")
+	c.close("sub/dir/potato")
+
+	assert.Equal(t, `name="" isFile=false opens=0
+name="sub" isFile=false opens=0
+name="sub/dir" isFile=false opens=0
+name="sub/dir/potato" isFile=true opens=0`, itemAsString(c))
+
+	removed = nil
+	removedDir = false
+	c._purgeOld(10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string(nil), removed)
+
+	assert.Equal(t, `name="" isFile=false opens=0
+name="sub" isFile=false opens=0
+name="sub/dir" isFile=false opens=0
+name="sub/dir/potato" isFile=true opens=0`, itemAsString(c))
+
+	removed = nil
+	removedDir = true
+	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	assert.Equal(t, []string{
+		"sub/dir/potato",
+		"sub/dir/",
+		"sub/",
+		"/",
+	}, removed)
+
+	assert.Equal(t, ``, itemAsString(c))
+}