extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-23 07:29:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

union: Implement union backend which reads from multiple backends

Browse Source
This commit is contained in:
Felix Brucker 2018-08-18 02:39:49 +02:00 committed by Nick Craig-Wood
parent 0fb12112f5
commit 9e3ea3c6ac
6 changed files with 379 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/all/all.go
View File

@ -25,6 +25,7 @@ import (
_ "github.com/ncw/rclone/backend/s3"
_ "github.com/ncw/rclone/backend/sftp"
_ "github.com/ncw/rclone/backend/swift"
_ "github.com/ncw/rclone/backend/union"
_ "github.com/ncw/rclone/backend/webdav"
_ "github.com/ncw/rclone/backend/yandex"
)

211
backend/union/union.go Normal file
View File

@ -0,0 +1,211 @@
package union
import (
"errors"
"fmt"
"io"
"path"
"path/filepath"
"strings"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config/configmap"
"github.com/ncw/rclone/fs/config/configstruct"
"github.com/ncw/rclone/fs/hash"
)
// Register with Fs
func init() {
fsi := &fs.RegInfo{
Name: "union",
Description: "Builds a stackable unification remote, which can appear to merge the contents of several remotes",
NewFs: NewFs,
Options: []fs.Option{{
Name: "remotes",
Help: "List of space separated remotes.\nCan be 'remotea:test/dir remoteb:', '\"remotea:test/space dir\" remoteb:', etc.\nThe last remote is used to write to.",
Required: true,
}},
}
fs.Register(fsi)
}
// Options defines the configuration for this backend
type Options struct {
Remotes fs.SpaceSepList `config:"remotes"`
}
// Fs represents a remote acd server
type Fs struct {
name string // name of this remote
features *fs.Features // optional features
opt Options // options for this Fs
root string // the path we are working on
remotes []fs.Fs // slice of remotes
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
// String converts this Fs to a string
func (f *Fs) String() string {
return fmt.Sprintf("union root '%s'", f.root)
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// Rmdir removes the root directory of the Fs object
func (f *Fs) Rmdir(dir string) error {
return f.remotes[len(f.remotes)-1].Rmdir(dir)
}
// Hashes returns hash.HashNone to indicate remote hashing is unavailable
func (f *Fs) Hashes() hash.Set {
// This could probably be set if all remotes share the same hashing algorithm
return hash.Set(hash.None)
}
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(dir string) error {
return f.remotes[len(f.remotes)-1].Mkdir(dir)
}
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return f.remotes[len(f.remotes)-1].Put(in, src, options...)
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
set := make(map[string]fs.DirEntry)
for _, remote := range f.remotes {
var remoteEntries, err = remote.List(dir)
if err != nil {
continue
}
for _, remoteEntry := range remoteEntries {
set[remoteEntry.Remote()] = remoteEntry
}
}
for key := range set {
entries = append(entries, set[key])
}
return entries, nil
}
// NewObject creates a new remote union file object based on the first Object it finds (reverse remote order)
func (f *Fs) NewObject(path string) (fs.Object, error) {
for i := range f.remotes {
var remote = f.remotes[len(f.remotes)-i-1]
var obj, err = remote.NewObject(path)
if err != nil {
continue
}
return obj, nil
}
return nil, fs.ErrorObjectNotFound
}
// Precision is the greatest Precision of all remotes
func (f *Fs) Precision() time.Duration {
var greatestPrecision = time.Second
for _, remote := range f.remotes {
if remote.Precision() <= greatestPrecision {
continue
}
greatestPrecision = remote.Precision()
}
return greatestPrecision
}
// NewFs constructs an Fs from the path.
//
// The returned Fs is the actual Fs, referenced by remote in the config
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
}
if len(opt.Remotes) == 0 {
return nil, errors.New("union can't point to an empty remote - check the value of the remotes setting")
}
if len(opt.Remotes) == 1 {
return nil, errors.New("union can't point to a single remote - check the value of the remotes setting")
}
for _, remote := range opt.Remotes {
if strings.HasPrefix(remote, name+":") {
return nil, errors.New("can't point union remote at itself - check the value of the remote setting")
}
}
var remotes []fs.Fs
for i := range opt.Remotes {
// Last remote first so we return the correct (last) matching fs in case of fs.ErrorIsFile
var remote = opt.Remotes[len(opt.Remotes)-i-1]
_, configName, fsPath, err := fs.ParseRemote(remote)
if err != nil {
return nil, err
}
var rootString = path.Join(fsPath, filepath.ToSlash(root))
if configName != "local" {
rootString = configName + ":" + rootString
}
myFs, err := fs.NewFs(rootString)
if err != nil {
if err == fs.ErrorIsFile {
return myFs, err
}
return nil, err
}
remotes = append(remotes, myFs)
}
// Reverse the remotes again so they are in the order as before
for i, j := 0, len(remotes)-1; i < j; i, j = i+1, j-1 {
remotes[i], remotes[j] = remotes[j], remotes[i]
}
f := &Fs{
name: name,
root: root,
opt: *opt,
remotes: remotes,
}
var features = (&fs.Features{}).Fill(f)
for _, remote := range f.remotes {
features = features.Mask(remote)
}
f.features = features
return f, nil
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
)

17
backend/union/union_test.go Normal file
View File

@ -0,0 +1,17 @@
// Test Union filesystem interface
package union_test
import (
"testing"
"github.com/ncw/rclone/backend/local"
"github.com/ncw/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestUnion:",
NilObject: (*local.Object)(nil),
})
}

1
docs/content/docs.md
View File

@ -42,6 +42,7 @@ See the following for detailed instructions for
* [Pcloud](/pcloud/)
* [QingStor](/qingstor/)
* [SFTP](/sftp/)
* [Union](/union/)
* [WebDAV](/webdav/)
* [Yandex Disk](/yandex/)
* [The local filesystem](/local/)

144
docs/content/union.md Normal file
View File

@ -0,0 +1,144 @@
---
title: "Union"
description: "Remote Unification"
date: "2018-08-29"
---
<i class="fa fa-link"></i> Union
-----------------------------------------
The `union` remote provides a unification similar to UnionFS using other remotes.
Paths may be as deep as required or a local path,
eg `remote:directory/subdirectory` or `/directory/subdirectory`.
During the initial setup with `rclone config` you will specify the target
remotes as a space separated list. The target remotes can either be a local paths or other remotes.
The order of the remotes is important as it defines which remotes take precedence over others if there are files with the same name in the same logical path.
The last remote is the topmost remote and replaces files with the same name from previous remotes.
Only the last remote is used to write to and delete from, all other remotes are read-only.
Subfolders can be used in target remote. Asume a union remote named `backup`
with the remotes `mydrive:private/backup mydrive2:/backup`. Invoking `rclone mkdir backup:desktop`
is exactly the same as invoking `rclone mkdir mydrive2:/backup/desktop`.
There will be no special handling of paths containing `..` segments.
Invoking `rclone mkdir backup:../desktop` is exactly the same as invoking
`rclone mkdir mydrive2:/backup/../desktop`.
Here is an example of how to make a union called `remote` for local folders.
First run:
rclone config
This will guide you through an interactive setup process:
```
No remotes found - make a new one
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> remote
Type of storage to configure.
Choose a number from below, or type in your own value
1 / Alias for a existing remote
\ "alias"
2 / Amazon Drive
\ "amazon cloud drive"
3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)
\ "s3"
4 / Backblaze B2
\ "b2"
5 / Box
\ "box"
6 / Builds a stackable unification remote, which can appear to merge the contents of several remotes
\ "union"
7 / Cache a remote
\ "cache"
8 / Dropbox
\ "dropbox"
9 / Encrypt/Decrypt a remote
\ "crypt"
10 / FTP Connection
\ "ftp"
11 / Google Cloud Storage (this is not Google Drive)
\ "google cloud storage"
12 / Google Drive
\ "drive"
13 / Hubic
\ "hubic"
14 / JottaCloud
\ "jottacloud"
15 / Local Disk
\ "local"
16 / Mega
\ "mega"
17 / Microsoft Azure Blob Storage
\ "azureblob"
18 / Microsoft OneDrive
\ "onedrive"
19 / OpenDrive
\ "opendrive"
20 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
\ "swift"
21 / Pcloud
\ "pcloud"
22 / QingCloud Object Storage
\ "qingstor"
23 / SSH/SFTP Connection
\ "sftp"
24 / Webdav
\ "webdav"
25 / Yandex Disk
\ "yandex"
26 / http Connection
\ "http"
Storage> union
List of space separated remotes.
Can be 'remotea:test/dir remoteb:', '"remotea:test/space dir" remoteb:', etc.
The last remote is used to write to.
Enter a string value. Press Enter for the default ("").
remotes>
Remote config
--------------------
[remote]
type = union
remotes = C:\dir1 C:\dir2 C:\dir3
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
Current remotes:
Name Type
==== ====
remote union
e) Edit existing remote
n) New remote
d) Delete remote
r) Rename remote
c) Copy remote
s) Set configuration password
q) Quit config
e/n/d/r/c/s/q> q
```
Once configured you can then use `rclone` like this,
List directories in top level in `C:\dir1`, `C:\dir2` and `C:\dir3`
rclone lsd remote:
List all the files in `C:\dir1`, `C:\dir2` and `C:\dir3`
rclone ls remote:
Copy another local directory to the union directory called source, which will be placed into `C:\dir3`
rclone copy C:\source remote:source

5
fstest/test_all/test_all.go
View File

@ -147,6 +147,11 @@ var (
SubDir: false,
FastList: false,
},
{
Name: "TestUnion:",
SubDir: false,
FastList: false,
},
}
// Flags
maxTries = flag.Int("maxtries", 5, "Number of times to try each test")