union: Implement union backend which reads from multiple backends

backend/all/all.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/ncw/rclone/backend/s3"
 	_ "github.com/ncw/rclone/backend/sftp"
 	_ "github.com/ncw/rclone/backend/swift"
+	_ "github.com/ncw/rclone/backend/union"
 	_ "github.com/ncw/rclone/backend/webdav"
 	_ "github.com/ncw/rclone/backend/yandex"
 )

backend/union/union.go

@@ -0,0 +1,211 @@
+package union
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/ncw/rclone/fs/hash"
+)
+
+// Register with Fs
+func init() {
+	fsi := &fs.RegInfo{
+		Name:        "union",
+		Description: "Builds a stackable unification remote, which can appear to merge the contents of several remotes",
+		NewFs:       NewFs,
+		Options: []fs.Option{{
+			Name:     "remotes",
+			Help:     "List of space separated remotes.\nCan be 'remotea:test/dir remoteb:', '\"remotea:test/space dir\" remoteb:', etc.\nThe last remote is used to write to.",
+			Required: true,
+		}},
+	}
+	fs.Register(fsi)
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	Remotes fs.SpaceSepList `config:"remotes"`
+}
+
+// Fs represents a remote acd server
+type Fs struct {
+	name     string       // name of this remote
+	features *fs.Features // optional features
+	opt      Options      // options for this Fs
+	root     string       // the path we are working on
+	remotes  []fs.Fs      // slice of remotes
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("union root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Rmdir removes the root directory of the Fs object
+func (f *Fs) Rmdir(dir string) error {
+	return f.remotes[len(f.remotes)-1].Rmdir(dir)
+}
+
+// Hashes returns hash.HashNone to indicate remote hashing is unavailable
+func (f *Fs) Hashes() hash.Set {
+	// This could probably be set if all remotes share the same hashing algorithm
+	return hash.Set(hash.None)
+}
+
+// Mkdir makes the root directory of the Fs object
+func (f *Fs) Mkdir(dir string) error {
+	return f.remotes[len(f.remotes)-1].Mkdir(dir)
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.remotes[len(f.remotes)-1].Put(in, src, options...)
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+	set := make(map[string]fs.DirEntry)
+	for _, remote := range f.remotes {
+		var remoteEntries, err = remote.List(dir)
+		if err != nil {
+			continue
+		}
+		for _, remoteEntry := range remoteEntries {
+			set[remoteEntry.Remote()] = remoteEntry
+		}
+	}
+
+	for key := range set {
+		entries = append(entries, set[key])
+	}
+	return entries, nil
+}
+
+// NewObject creates a new remote union file object based on the first Object it finds (reverse remote order)
+func (f *Fs) NewObject(path string) (fs.Object, error) {
+	for i := range f.remotes {
+		var remote = f.remotes[len(f.remotes)-i-1]
+		var obj, err = remote.NewObject(path)
+		if err != nil {
+			continue
+		}
+		return obj, nil
+	}
+	return nil, fs.ErrorObjectNotFound
+}
+
+// Precision is the greatest Precision of all remotes
+func (f *Fs) Precision() time.Duration {
+	var greatestPrecision = time.Second
+	for _, remote := range f.remotes {
+		if remote.Precision() <= greatestPrecision {
+			continue
+		}
+		greatestPrecision = remote.Precision()
+	}
+	return greatestPrecision
+}
+
+// NewFs constructs an Fs from the path.
+//
+// The returned Fs is the actual Fs, referenced by remote in the config
+func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	if len(opt.Remotes) == 0 {
+		return nil, errors.New("union can't point to an empty remote - check the value of the remotes setting")
+	}
+	if len(opt.Remotes) == 1 {
+		return nil, errors.New("union can't point to a single remote - check the value of the remotes setting")
+	}
+	for _, remote := range opt.Remotes {
+		if strings.HasPrefix(remote, name+":") {
+			return nil, errors.New("can't point union remote at itself - check the value of the remote setting")
+		}
+	}
+
+	var remotes []fs.Fs
+	for i := range opt.Remotes {
+		// Last remote first so we return the correct (last) matching fs in case of fs.ErrorIsFile
+		var remote = opt.Remotes[len(opt.Remotes)-i-1]
+		_, configName, fsPath, err := fs.ParseRemote(remote)
+		if err != nil {
+			return nil, err
+		}
+		var rootString = path.Join(fsPath, filepath.ToSlash(root))
+		if configName != "local" {
+			rootString = configName + ":" + rootString
+		}
+		myFs, err := fs.NewFs(rootString)
+		if err != nil {
+			if err == fs.ErrorIsFile {
+				return myFs, err
+			}
+			return nil, err
+		}
+		remotes = append(remotes, myFs)
+	}
+
+	// Reverse the remotes again so they are in the order as before
+	for i, j := 0, len(remotes)-1; i < j; i, j = i+1, j-1 {
+		remotes[i], remotes[j] = remotes[j], remotes[i]
+	}
+
+	f := &Fs{
+		name:    name,
+		root:    root,
+		opt:     *opt,
+		remotes: remotes,
+	}
+	var features = (&fs.Features{}).Fill(f)
+	for _, remote := range f.remotes {
+		features = features.Mask(remote)
+	}
+	f.features = features
+
+	return f, nil
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs = &Fs{}
+)

backend/union/union_test.go

@@ -0,0 +1,17 @@
+// Test Union filesystem interface
+package union_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestUnion:",
+		NilObject:  (*local.Object)(nil),
+	})
+}

docs/content/docs.md

@@ -42,6 +42,7 @@ See the following for detailed instructions for
   * [Pcloud](/pcloud/)
   * [QingStor](/qingstor/)
   * [SFTP](/sftp/)
+  * [Union](/union/)
   * [WebDAV](/webdav/)
   * [Yandex Disk](/yandex/)
   * [The local filesystem](/local/)

docs/content/union.md

@@ -0,0 +1,144 @@
+---
+title: "Union"
+description: "Remote Unification"
+date: "2018-08-29"
+---
+
+<i class="fa fa-link"></i> Union
+-----------------------------------------
+
+The `union` remote provides a unification similar to UnionFS using other remotes.
+
+Paths may be as deep as required or a local path, 
+eg `remote:directory/subdirectory` or `/directory/subdirectory`.
+
+During the initial setup with `rclone config` you will specify the target
+remotes as a space separated list. The target remotes can either be a local paths or other remotes.
+
+The order of the remotes is important as it defines which remotes take precedence over others if there are files with the same name in the same logical path.
+The last remote is the topmost remote and replaces files with the same name from previous remotes.
+
+Only the last remote is used to write to and delete from, all other remotes are read-only.
+
+Subfolders can be used in target remote. Asume a union remote named `backup`
+with the remotes `mydrive:private/backup mydrive2:/backup`. Invoking `rclone mkdir backup:desktop`
+is exactly the same as invoking `rclone mkdir mydrive2:/backup/desktop`.
+
+There will be no special handling of paths containing `..` segments.
+Invoking `rclone mkdir backup:../desktop` is exactly the same as invoking
+`rclone mkdir mydrive2:/backup/../desktop`.
+
+Here is an example of how to make a union called `remote` for local folders.
+First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Alias for a existing remote
+   \ "alias"
+ 2 / Amazon Drive
+   \ "amazon cloud drive"
+ 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)
+   \ "s3"
+ 4 / Backblaze B2
+   \ "b2"
+ 5 / Box
+   \ "box"
+ 6 / Builds a stackable unification remote, which can appear to merge the contents of several remotes
+   \ "union"
+ 7 / Cache a remote
+   \ "cache"
+ 8 / Dropbox
+   \ "dropbox"
+ 9 / Encrypt/Decrypt a remote
+   \ "crypt"
+10 / FTP Connection
+   \ "ftp"
+11 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+12 / Google Drive
+   \ "drive"
+13 / Hubic
+   \ "hubic"
+14 / JottaCloud
+   \ "jottacloud"
+15 / Local Disk
+   \ "local"
+16 / Mega
+   \ "mega"
+17 / Microsoft Azure Blob Storage
+   \ "azureblob"
+18 / Microsoft OneDrive
+   \ "onedrive"
+19 / OpenDrive
+   \ "opendrive"
+20 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+21 / Pcloud
+   \ "pcloud"
+22 / QingCloud Object Storage
+   \ "qingstor"
+23 / SSH/SFTP Connection
+   \ "sftp"
+24 / Webdav
+   \ "webdav"
+25 / Yandex Disk
+   \ "yandex"
+26 / http Connection
+   \ "http"
+Storage> union
+List of space separated remotes.
+Can be 'remotea:test/dir remoteb:', '"remotea:test/space dir" remoteb:', etc.
+The last remote is used to write to.
+Enter a string value. Press Enter for the default ("").
+remotes>
+Remote config
+--------------------
+[remote]
+type = union
+remotes = C:\dir1 C:\dir2 C:\dir3
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+remote               union
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> q
+```
+
+Once configured you can then use `rclone` like this,
+
+List directories in top level in `C:\dir1`, `C:\dir2` and `C:\dir3`
+
+    rclone lsd remote:
+
+List all the files in `C:\dir1`, `C:\dir2` and `C:\dir3`
+
+    rclone ls remote:
+
+Copy another local directory to the union directory called source, which will be placed into `C:\dir3`
+
+    rclone copy C:\source remote:source
+

fstest/test_all/test_all.go

@@ -147,6 +147,11 @@ var (
 			SubDir:   false,
 			FastList: false,
 		},
+		{
+			Name:     "TestUnion:",
+			SubDir:   false,
+			FastList: false,
+		},
 	}
 	// Flags
 	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")