From 43ad7b10a2c243cf06f14eb815ece214a9c533b5 Mon Sep 17 00:00:00 2001
From: Leo Luan <leoluan@us.ibm.com>
Date: Mon, 5 Oct 2020 01:47:54 -0700
Subject: [PATCH] vfs: Add exponential backoff during ENOSPC retries

Add an exponentially increasing delay during retries up ENOSPC error
to avoid exhausting the 10 retries too soon when the cache space
recovery from item resets is not available from the file system yet
or consumed by other large cache writes.
---
 vfs/vfscache/cache.go | 1 -
 vfs/vfscache/item.go  | 7 +++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go
index 0ecb42a85..e329bc4b0 100644
--- a/vfs/vfscache/cache.go
+++ b/vfs/vfscache/cache.go
@@ -609,7 +609,6 @@ func (c *Cache) clean(removeCleanFiles bool) {
 	if os.IsNotExist(err) {
 		return
 	}
-
 	c.mu.Lock()
 	oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used)
 	c.mu.Unlock()
diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index 2ca929b95..22ca1801d 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -1182,6 +1182,7 @@ func (item *Item) setModTime(modTime time.Time) {
 // ReadAt bytes from the file at off
 func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
 	n = 0
+	var expBackOff int
 	for retries := 0; retries < fs.Config.LowLevelRetries; retries++ {
 		item.preAccess()
 		n, err = item.readAt(b, off)
@@ -1195,6 +1196,12 @@ func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
 			break
 		}
 		item.c.KickCleaner()
+		expBackOff = 2 << uint(retries)
+		time.Sleep(time.Duration(expBackOff) * time.Millisecond) // Exponential back-off the retries
+	}
+
+	if fserrors.IsErrNoSpace(err) {
+		fs.Errorf(item.name, "vfs cache: failed to _ensure cache after retries %v", err)
 	}
 
 	return n, err