From b35123ba48efacdb1c663a216a44b8728ad0400a Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Thu, 3 Nov 2016 11:51:36 +0000
Subject: [PATCH] Make -x/--one-file-system compile under Windows and add docs
---
docs/content/local.md | 45 ++++++++++++++++++++++++++++++++++++++
local/local.go | 31 +++++++++++++++-----------
local/read_device_other.go | 13 +++++++++++
local/read_device_unix.go | 31 ++++++++++++++++++++++++++
4 files changed, 107 insertions(+), 13 deletions(-)
create mode 100644 local/read_device_other.go
create mode 100644 local/read_device_unix.go
diff --git a/docs/content/local.md b/docs/content/local.md
index 28f8a0715..3858e62fe 100644
--- a/docs/content/local.md
+++ b/docs/content/local.md
@@ -74,3 +74,48 @@ And use rclone like this:
This will use UNC paths on `c:\src` but not on `z:\dst`.
Of course this will cause problems if the absolute path length of a
file exceeds 258 characters on z, so only use this option if you have to.
+
+### Specific options ###
+
+Here are the command line options specific to local storage
+
+#### --one-file-system, -x ####
+
+This tells rclone to stay in the filesystem specified by the root and
+not to recurse into different file systems.
+
+For example if you have a directory heirachy like this
+
+```
+root
+├── disk1 - disk1 mounted on the root
+│ └── file3 - stored on disk1
+├── disk2 - disk2 mounted on the root
+│ └── file4 - stored on disk12
+├── file1 - stored on the root disk
+└── file2 - stored on the root disk
+```
+
+Using `rclone --one-file-system copy root remote:` will only copy `file1` and `file2`. Eg
+
+```
+$ rclone -q --one-file-system ls root
+ 0 file1
+ 0 file2
+```
+
+```
+$ rclone -q ls root
+ 0 disk1/file3
+ 0 disk2/file4
+ 0 file1
+ 0 file2
+```
+
+**NB** Rclone (like most unix tools such as `du`, `rsync` and `tar`)
+treats a bind mount to the same device as being on the same
+filesystem.
+
+**NB** This flag is only available on Unix based systems. On systems
+where it isn't supported (eg Windows) it will not appear as an valid
+flag.
diff --git a/local/local.go b/local/local.go
index c0ca3f56a..1974f1044 100644
--- a/local/local.go
+++ b/local/local.go
@@ -12,7 +12,6 @@ import (
"runtime"
"strings"
"sync"
- "syscall"
"time"
"unicode/utf8"
@@ -20,10 +19,10 @@ import (
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
- "github.com/spf13/pflag"
)
-var oneFileSystem = pflag.BoolP("one-file-system", "x", false, "Don't cross filesystem boundaries.")
+// Constants
+const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
// Register with Fs
func init() {
@@ -48,6 +47,7 @@ func init() {
type Fs struct {
name string // the name of the remote
root string // The root directory (OS path)
+ dev uint64 // device number of root node
precisionOk sync.Once // Whether we need to read the precision
precision time.Duration // precision of local filesystem
wmu sync.Mutex // used for locking access to 'warned'.
@@ -75,11 +75,15 @@ func NewFs(name, root string) (fs.Fs, error) {
name: name,
warned: make(map[string]struct{}),
nounc: nounc == "true",
+ dev: devUnset,
}
f.root = f.cleanPath(root)
// Check to see if this points to a file
fi, err := os.Lstat(f.root)
+ if err == nil {
+ f.dev = readDevice(fi)
+ }
if err == nil && fi.Mode().IsRegular() {
// It is a file, so use the parent as the root
f.root, _ = getDirFile(f.root)
@@ -156,14 +160,6 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su
return nil
}
- // Obtain dirpath's device
- fdFi, err := os.Stat(dirpath)
- if err != nil {
- out.SetError(errors.Wrapf(err, "failed to stat directory %q", dirpath))
- return nil
- }
- fdDev := fdFi.Sys().(*syscall.Stat_t).Dev
-
defer func() {
err := fd.Close()
if err != nil {
@@ -199,7 +195,7 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su
if out.AddDir(dir) {
return nil
}
- if level > 0 && !(*oneFileSystem && !((fi.Sys().(*syscall.Stat_t)).Dev == fdDev)) {
+ if level > 0 && f.dev == readDevice(fi) {
subdirs = append(subdirs, listArgs{remote: newRemote, dirpath: newPath, level: level - 1})
}
}
@@ -302,7 +298,16 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
// Mkdir creates the directory if it doesn't exist
func (f *Fs) Mkdir() error {
// FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go
- return os.MkdirAll(f.root, 0777)
+ err := os.MkdirAll(f.root, 0777)
+ if err != nil {
+ return err
+ }
+ fi, err := os.Lstat(f.root)
+ if err != nil {
+ return err
+ }
+ f.dev = readDevice(fi)
+ return nil
}
// Rmdir removes the directory
diff --git a/local/read_device_other.go b/local/read_device_other.go
new file mode 100644
index 000000000..1429c7ddc
--- /dev/null
+++ b/local/read_device_other.go
@@ -0,0 +1,13 @@
+// Device reading functions
+
+// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
+
+package local
+
+import "os"
+
+// readDevice turns a valid os.FileInfo into a device number,
+// returning devUnset if it fails.
+func readDevice(fi os.FileInfo) uint64 {
+ return devUnset
+}
diff --git a/local/read_device_unix.go b/local/read_device_unix.go
new file mode 100644
index 000000000..472e72f8d
--- /dev/null
+++ b/local/read_device_unix.go
@@ -0,0 +1,31 @@
+// Device reading functions
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package local
+
+import (
+ "os"
+ "syscall"
+
+ "github.com/ncw/rclone/fs"
+ "github.com/spf13/pflag"
+)
+
+var (
+ oneFileSystem = pflag.BoolP("one-file-system", "x", false, "Don't cross filesystem boundaries.")
+)
+
+// readDevice turns a valid os.FileInfo into a device number,
+// returning devUnset if it fails.
+func readDevice(fi os.FileInfo) uint64 {
+ if !*oneFileSystem {
+ return devUnset
+ }
+ statT, ok := fi.Sys().(*syscall.Stat_t)
+ if !ok {
+ fs.Debug(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys())
+ return devUnset
+ }
+ return uint64(statT.Dev)
+}