vfs: add flag --vfs-case-insensitive for windows/macOS mounts

rclone mount when run on Windows & macOS will now default to `--vfs-case-insensitive`.
This means that

.appveyor.yml

@@ -40,6 +40,7 @@ build_script:
 
 test_script:
 - make GOTAGS=cmount quicktest
+- make GOTAGS=cmount racequicktest
 
 artifacts:
 - path: rclone.exe

azure-pipelines.yml

@@ -49,6 +49,7 @@ strategy:
       GO_VERSION: latest
       BUILD_FLAGS: '-include "^windows/amd64" -cgo'
       MAKE_QUICKTEST: true
+      MAKE_RACEQUICKTEST: true
       DEPLOY: true
     windows_386:
       imageName: windows-2019

docs/content/commands/rclone_mount.md

@@ -287,11 +287,46 @@ This mode should support all normal file system operations.
 If an upload or download fails it will be retried up to
 --low-level-retries times.
 
-
 ```
 rclone mount remote:path /path/to/mountpoint [flags]
 ```
 
+### Case Sensitivity
+
+Linux file systems are case-sensitive: two files can differ only
+by case, and the exact case must be used when opening a file.
+
+Windows is not like most other operating systems supported by rclone.
+File systems in modern Windows are case-insensitive but case-preserving:
+although existing files can be opened using any case, the exact case used
+to create the file is preserved and available for programs to query.
+It is not allowed for two files in the same directory to differ only by case.
+
+Usually file systems on MacOS are case-insensitive. It is possible to make MacOS
+file systems case-sensitive but that is not the default
+
+The `--vfs-case-insensitive` mount flag controls how rclone handles these
+two cases. If its value is `false`, rclone passes file names to the mounted
+file system as is. If the flag is `true` (or appears without a value on
+command line), rclone may perform a "fixup" as explained below.
+
+The user may specify a file name to open/delete/rename/etc with a case
+different than what is stored on mounted file system. If an argument refers
+to an existing file with exactly the same name, then the case of the existing
+file on the disk will be used. However, if a file name with exactly the same
+name is not found but a name differing only by case exists, rclone will
+transparently fixup the name. This fixup happens only when an existing file
+is requested. Case sensitivity of file names created anew by rclone is
+controlled by an underlying mounted file system.
+
+Note that case sensitivity of the operating system running rclone (the target)
+may differ from case sensitivity of a file system mounted by rclone (the source).
+The flag controls whether "fixup" is performed to satisfy the target.
+
+If the flag is not provided on command line, then its default value depends
+on the operating system where rclone runs: `true` on Windows and MacOS, `false`
+otherwise. If the flag is provided without a value, then it is `true`.
+
 ### Options
 
 ```
@@ -322,6 +357,7 @@ rclone mount remote:path /path/to/mountpoint [flags]
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache. (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-case-insensitive [bool]            Case insensitive mount true|false (default depends on operating system)
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks. (default 128M)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
       --volname string                         Set the volume name (not supported by all OSes).

vfs/dir.go

@@ -323,6 +323,8 @@ func (d *Dir) readDir() error {
 // stat a single item in the directory
 //
 // returns ENOENT if not found.
+// returns a custom error if directory on a case-insensitive file system
+// contains files with names that differ only by case.
 func (d *Dir) stat(leaf string) (Node, error) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
@@ -331,6 +333,22 @@ func (d *Dir) stat(leaf string) (Node, error) {
 		return nil, err
 	}
 	item, ok := d.items[leaf]
+
+	if !ok && d.vfs.Opt.CaseInsensitive {
+		leafLower := strings.ToLower(leaf)
+		for name, node := range d.items {
+			if strings.ToLower(name) == leafLower {
+				if ok {
+					// duplicate case insensitive match is an error
+					return nil, errors.Errorf("duplicate filename %q detected with --vfs-case-insensitive set", leaf)
+				}
+				// found a case insenstive match
+				ok = true
+				item = node
+			}
+		}
+	}
+
 	if !ok {
 		return nil, ENOENT
 	}

vfs/vfs.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"runtime"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -51,6 +52,7 @@ var DefaultOpt = Options{
 	ChunkSize:         128 * fs.MebiByte,
 	ChunkSizeLimit:    -1,
 	CacheMaxSize:      -1,
+	CaseInsensitive:   runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
 }
 
 // Node represents either a directory (*Dir) or a file (*File)
@@ -199,6 +201,7 @@ type Options struct {
 	CacheMaxAge       time.Duration
 	CacheMaxSize      fs.SizeSuffix
 	CachePollInterval time.Duration
+	CaseInsensitive   bool
 }
 
 // New creates a new VFS and root directory.  If opt is nil, then

vfs/vfs_case_test.go

@@ -0,0 +1,156 @@
+package vfs
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCaseSensitivity(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	// Create test files
+	ctx := context.Background()
+	file1 := r.WriteObject(ctx, "FiLeA", "data1", t1)
+	file2 := r.WriteObject(ctx, "FiLeB", "data2", t2)
+	fstest.CheckItems(t, r.Fremote, file1, file2)
+
+	// Create file3 with name differing from file2 name only by case.
+	// On a case-Sensitive remote this will be a separate file.
+	// On a case-INsensitive remote this file will either not exist
+	// or overwrite file2 depending on how file system diverges.
+	file3 := r.WriteObject(ctx, "FilEb", "data3", t3)
+
+	// Create a case-Sensitive and case-INsensitive VFS
+	optCS := DefaultOpt
+	optCS.CaseInsensitive = false
+	vfsCS := New(r.Fremote, &optCS)
+
+	optCI := DefaultOpt
+	optCI.CaseInsensitive = true
+	vfsCI := New(r.Fremote, &optCI)
+
+	// Run basic checks that must pass on VFS of any type.
+	assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
+	assertFileDataVFS(t, vfsCS, "FiLeA", "data1")
+
+	// Detect case sensitivity of the underlying remote.
+	remoteIsOK := true
+	if !checkFileDataVFS(t, vfsCS, "FiLeA", "data1") {
+		remoteIsOK = false
+	}
+	if !checkFileDataVFS(t, vfsCS, "FiLeB", "data2") {
+		remoteIsOK = false
+	}
+	if !checkFileDataVFS(t, vfsCS, "FilEb", "data3") {
+		remoteIsOK = false
+	}
+
+	// The remaining test is only meaningful on a case-Sensitive file system.
+	if !remoteIsOK {
+		t.Logf("SKIP: TestCaseSensitivity - remote is not fully case-sensitive")
+		return
+	}
+
+	// Continue with test as the underlying remote is fully case-Sensitive.
+	fstest.CheckItems(t, r.Fremote, file1, file2, file3)
+
+	// See how VFS handles case-INsensitive flag
+	assertFileDataVFS(t, vfsCI, "FiLeA", "data1")
+	assertFileDataVFS(t, vfsCI, "fileA", "data1")
+	assertFileDataVFS(t, vfsCI, "filea", "data1")
+	assertFileDataVFS(t, vfsCI, "FILEA", "data1")
+
+	assertFileDataVFS(t, vfsCI, "FiLeB", "data2")
+	assertFileDataVFS(t, vfsCI, "FilEb", "data3")
+
+	fd, err := vfsCI.OpenFile("fileb", os.O_RDONLY, 0777)
+	assert.Nil(t, fd)
+	assert.Error(t, err)
+	assert.NotEqual(t, err, ENOENT)
+
+	fd, err = vfsCI.OpenFile("FILEB", os.O_RDONLY, 0777)
+	assert.Nil(t, fd)
+	assert.Error(t, err)
+	assert.NotEqual(t, err, ENOENT)
+
+	// Run the same set of checks with case-Sensitive VFS, for comparison.
+	assertFileDataVFS(t, vfsCS, "FiLeA", "data1")
+
+	assertFileAbsentVFS(t, vfsCS, "fileA")
+	assertFileAbsentVFS(t, vfsCS, "filea")
+	assertFileAbsentVFS(t, vfsCS, "FILEA")
+
+	assertFileDataVFS(t, vfsCS, "FiLeB", "data2")
+	assertFileDataVFS(t, vfsCS, "FilEb", "data3")
+
+	assertFileAbsentVFS(t, vfsCS, "fileb")
+	assertFileAbsentVFS(t, vfsCS, "FILEB")
+}
+
+func checkFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) bool {
+	fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
+	if fd == nil || err != nil {
+		return false
+	}
+	defer func() {
+		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
+		_ = fd.Close()
+	}()
+
+	fh, ok := fd.(*ReadFileHandle)
+	if !ok {
+		return false
+	}
+
+	size := len(expect)
+	buf := make([]byte, size)
+	num, err := fh.Read(buf)
+	if err != nil || num != size {
+		return false
+	}
+
+	return string(buf) == expect
+}
+
+func assertFileDataVFS(t *testing.T, vfs *VFS, name string, expect string) {
+	fd, errOpen := vfs.OpenFile(name, os.O_RDONLY, 0777)
+	assert.NotNil(t, fd)
+	assert.NoError(t, errOpen)
+
+	defer func() {
+		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
+		if errOpen == nil && fd != nil {
+			_ = fd.Close()
+		}
+	}()
+
+	fh, ok := fd.(*ReadFileHandle)
+	require.True(t, ok)
+
+	size := len(expect)
+	buf := make([]byte, size)
+	numRead, errRead := fh.Read(buf)
+	assert.NoError(t, errRead)
+	assert.Equal(t, numRead, size)
+
+	assert.Equal(t, string(buf), expect)
+}
+
+func assertFileAbsentVFS(t *testing.T, vfs *VFS, name string) {
+	fd, err := vfs.OpenFile(name, os.O_RDONLY, 0777)
+	defer func() {
+		// File must be closed - otherwise Run.cleanUp() will fail on Windows.
+		if err == nil && fd != nil {
+			_ = fd.Close()
+		}
+	}()
+	assert.Nil(t, fd)
+	assert.Error(t, err)
+	assert.Equal(t, err, ENOENT)
+}

vfs/vfsflags/vfsflags.go

@@ -32,5 +32,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.")
 	flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions")
 	flags.FVarP(flagSet, FilePerms, "file-perms", "", "File permissions")
+	flags.BoolVarP(flagSet, &Opt.CaseInsensitive, "vfs-case-insensitive", "", Opt.CaseInsensitive, "If a file name not found, find a case insensitive match.")
 	platformFlags(flagSet)
 }