2017-06-19 14:44:49 +02:00
package mountlib
import (
2017-11-09 01:37:27 +01:00
"io"
2017-06-19 14:44:49 +02:00
"log"
2017-11-09 01:37:27 +01:00
"os"
2020-07-23 14:08:38 +02:00
"os/signal"
2019-01-10 15:18:00 +01:00
"path/filepath"
2017-11-16 13:20:53 +01:00
"runtime"
2018-05-03 10:34:07 +02:00
"strings"
2020-11-27 11:50:10 +01:00
"sync"
2020-07-23 14:08:38 +02:00
"syscall"
2018-03-02 17:39:42 +01:00
"time"
2017-06-19 14:44:49 +02:00
2020-09-18 17:37:54 +02:00
sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
2017-11-09 01:37:27 +01:00
"github.com/pkg/errors"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/flags"
2020-07-23 18:17:01 +02:00
"github.com/rclone/rclone/fs/rc"
2020-07-23 14:08:38 +02:00
"github.com/rclone/rclone/lib/atexit"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
2017-06-19 14:44:49 +02:00
"github.com/spf13/cobra"
2020-07-23 18:17:01 +02:00
"github.com/spf13/pflag"
2017-06-19 14:44:49 +02:00
)
2020-07-23 18:17:01 +02:00
// Options for creating the mount
type Options struct {
DebugFUSE bool
AllowNonEmpty bool
AllowRoot bool
AllowOther bool
DefaultPermissions bool
WritebackCache bool
Daemon bool
MaxReadAhead fs . SizeSuffix
2017-11-07 18:09:08 +01:00
ExtraOptions [ ] string
ExtraFlags [ ] string
2020-07-23 18:17:01 +02:00
AttrTimeout time . Duration // how long the kernel caches attribute for
2018-05-03 10:34:07 +02:00
VolumeName string
2020-07-23 18:17:01 +02:00
NoAppleDouble bool
NoAppleXattr bool
2018-07-18 17:21:35 +02:00
DaemonTimeout time . Duration // OSXFUSE only
2020-07-23 18:17:01 +02:00
AsyncRead bool
2020-11-06 14:21:38 +01:00
NetworkMode bool // Windows only
2020-07-23 18:17:01 +02:00
}
// DefaultOpt is the default values for creating the mount
var DefaultOpt = Options {
MaxReadAhead : 128 * 1024 ,
AttrTimeout : 1 * time . Second , // how long the kernel caches attribute for
NoAppleDouble : true , // use noappledouble by default
NoAppleXattr : false , // do not use noapplexattr by default
AsyncRead : true , // do async reads by default
}
2017-06-19 14:44:49 +02:00
2020-04-25 07:03:07 +02:00
type (
// UnmountFn is called to unmount the file system
UnmountFn func ( ) error
// MountFn is called to mount the file system
2020-07-23 18:17:01 +02:00
MountFn func ( VFS * vfs . VFS , mountpoint string , opt * Options ) ( <- chan error , func ( ) error , error )
2020-04-25 07:03:07 +02:00
)
2019-10-18 11:53:07 +02:00
// Global constants
const (
2020-01-19 15:54:55 +01:00
MaxLeafSize = 1024 // don't pass file names longer than this
2019-10-18 11:53:07 +02:00
)
2019-06-24 12:54:38 +02:00
func init ( ) {
2019-10-12 13:41:36 +02:00
// DaemonTimeout defaults to non zero for macOS
if runtime . GOOS == "darwin" {
2021-02-21 13:56:19 +01:00
DefaultOpt . DaemonTimeout = 10 * time . Minute
2020-07-23 18:17:01 +02:00
}
}
// Options set by command line flags
var (
Opt = DefaultOpt
)
// AddFlags adds the non filing system specific flags to the command
func AddFlags ( flagSet * pflag . FlagSet ) {
rc . AddOption ( "mount" , & Opt )
flags . BoolVarP ( flagSet , & Opt . DebugFUSE , "debug-fuse" , "" , Opt . DebugFUSE , "Debug the FUSE internals - needs -v." )
flags . DurationVarP ( flagSet , & Opt . AttrTimeout , "attr-timeout" , "" , Opt . AttrTimeout , "Time for which file/directory attributes are cached." )
flags . StringArrayVarP ( flagSet , & Opt . ExtraOptions , "option" , "o" , [ ] string { } , "Option for libfuse/WinFsp. Repeat if required." )
flags . StringArrayVarP ( flagSet , & Opt . ExtraFlags , "fuse-flag" , "" , [ ] string { } , "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required." )
2020-11-11 00:17:25 +01:00
// Non-Windows only
flags . BoolVarP ( flagSet , & Opt . Daemon , "daemon" , "" , Opt . Daemon , "Run mount as a daemon (background mode). Not supported on Windows." )
flags . DurationVarP ( flagSet , & Opt . DaemonTimeout , "daemon-timeout" , "" , Opt . DaemonTimeout , "Time limit for rclone to respond to kernel. Not supported on Windows." )
flags . BoolVarP ( flagSet , & Opt . DefaultPermissions , "default-permissions" , "" , Opt . DefaultPermissions , "Makes kernel enforce access control based on the file mode. Not supported on Windows." )
flags . BoolVarP ( flagSet , & Opt . AllowNonEmpty , "allow-non-empty" , "" , Opt . AllowNonEmpty , "Allow mounting over a non-empty directory. Not supported on Windows." )
flags . BoolVarP ( flagSet , & Opt . AllowRoot , "allow-root" , "" , Opt . AllowRoot , "Allow access to root user. Not supported on Windows." )
flags . BoolVarP ( flagSet , & Opt . AllowOther , "allow-other" , "" , Opt . AllowOther , "Allow access to other users. Not supported on Windows." )
flags . BoolVarP ( flagSet , & Opt . AsyncRead , "async-read" , "" , Opt . AsyncRead , "Use asynchronous reads. Not supported on Windows." )
flags . FVarP ( flagSet , & Opt . MaxReadAhead , "max-read-ahead" , "" , "The number of bytes that can be prefetched for sequential reads. Not supported on Windows." )
flags . BoolVarP ( flagSet , & Opt . WritebackCache , "write-back-cache" , "" , Opt . WritebackCache , "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used. Not supported on Windows." )
// Windows and OSX
flags . StringVarP ( flagSet , & Opt . VolumeName , "volname" , "" , Opt . VolumeName , "Set the volume name. Supported on Windows and OSX only." )
// OSX only
flags . BoolVarP ( flagSet , & Opt . NoAppleDouble , "noappledouble" , "" , Opt . NoAppleDouble , "Ignore Apple Double (._) and .DS_Store files. Supported on OSX only." )
flags . BoolVarP ( flagSet , & Opt . NoAppleXattr , "noapplexattr" , "" , Opt . NoAppleXattr , "Ignore all \"com.apple.*\" extended attributes. Supported on OSX only." )
// Windows only
flags . BoolVarP ( flagSet , & Opt . NetworkMode , "network-mode" , "" , Opt . NetworkMode , "Mount as remote network drive, instead of fixed disk drive. Supported on Windows only" )
2019-06-24 12:54:38 +02:00
}
2020-10-26 11:36:57 +01:00
// Check if folder is empty
2017-11-09 01:37:27 +01:00
func checkMountEmpty ( mountpoint string ) error {
fp , fpErr := os . Open ( mountpoint )
if fpErr != nil {
return errors . Wrap ( fpErr , "Can not open: " + mountpoint )
}
defer fs . CheckClose ( fp , & fpErr )
_ , fpErr = fp . Readdirnames ( 1 )
// directory is not empty
if fpErr != io . EOF {
var e error
var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option"
if fpErr == nil {
e = errors . New ( errorMsg )
} else {
e = errors . Wrap ( fpErr , errorMsg )
}
return e
}
return nil
}
2019-01-10 15:18:00 +01:00
// Check the root doesn't overlap the mountpoint
func checkMountpointOverlap ( root , mountpoint string ) error {
abs := func ( x string ) string {
if absX , err := filepath . EvalSymlinks ( x ) ; err == nil {
x = absX
}
if absX , err := filepath . Abs ( x ) ; err == nil {
x = absX
}
x = filepath . ToSlash ( x )
if ! strings . HasSuffix ( x , "/" ) {
x += "/"
}
return x
}
rootAbs , mountpointAbs := abs ( root ) , abs ( mountpoint )
if strings . HasPrefix ( rootAbs , mountpointAbs ) || strings . HasPrefix ( mountpointAbs , rootAbs ) {
return errors . Errorf ( "mount point %q and directory to be mounted %q mustn't overlap" , mountpoint , root )
}
return nil
}
2017-06-19 14:44:49 +02:00
// NewMountCommand makes a mount command with the given name and Mount function
2020-07-23 14:08:38 +02:00
func NewMountCommand ( commandName string , hidden bool , mount MountFn ) * cobra . Command {
2019-10-11 17:58:11 +02:00
var commandDefinition = & cobra . Command {
2020-02-11 13:05:43 +01:00
Use : commandName + " remote:path /path/to/mountpoint" ,
Hidden : hidden ,
Short : ` Mount the remote as file system on a mountpoint. ` ,
2020-05-25 09:50:20 +02:00
// Warning! "|" will be replaced by backticks below
// "@" will be replaced by the command name
Long : strings . ReplaceAll ( strings . ReplaceAll ( `
rclone @ allows Linux , FreeBSD , macOS and Windows to
2017-06-19 14:44:49 +02:00
mount any of Rclone ' s cloud storage systems as a file system with
FUSE .
2020-05-25 09:50:20 +02:00
First set up your remote using | rclone config | . Check it works with | rclone ls | etc .
2017-06-19 14:44:49 +02:00
2020-11-11 00:17:25 +01:00
On Linux and OSX , you can either run mount in foreground mode or background ( daemon ) mode .
2020-05-25 09:50:20 +02:00
Mount runs in foreground mode by default , use the | -- daemon | flag to specify background mode .
2020-11-11 00:17:25 +01:00
You can only run mount in foreground mode on Windows .
2020-02-10 16:21:22 +01:00
2020-05-25 09:50:20 +02:00
On Linux / macOS / FreeBSD start the mount like this , where | / path / to / local / mount |
2020-11-12 21:36:51 +01:00
is an * * empty * * * * existing * * directory :
2017-06-19 14:44:49 +02:00
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files / path / to / local / mount
2017-06-19 14:44:49 +02:00
2020-11-12 21:36:51 +01:00
On Windows you can start a mount in different ways . See [ below ] ( # mounting - modes - on - windows )
for details . The following examples will mount to an automatically assigned drive ,
2021-02-08 15:50:56 +01:00
to specific drive letter | X : | , to path | C : \ path \ parent \ mount |
( where parent directory or drive must exist , and mount must * * not * * exist ,
2020-11-12 21:36:51 +01:00
and is not supported when [ mounting as a network drive ] ( # mounting - modes - on - windows ) ) , and
2020-05-25 09:50:20 +02:00
the last example will mount as network share | \ \ cloud \ remote | and map it to an
2020-11-12 21:36:51 +01:00
automatically assigned drive :
2017-06-19 14:44:49 +02:00
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files *
rclone @ remote : path / to / files X :
2021-02-08 15:50:56 +01:00
rclone @ remote : path / to / files C : \ path \ parent \ mount
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files \ \ cloud \ remote
2020-02-10 16:21:22 +01:00
When the program ends while in foreground mode , either via Ctrl + C or receiving
2020-11-12 21:36:51 +01:00
a SIGINT or SIGTERM signal , the mount should be automatically stopped .
2020-02-10 16:21:22 +01:00
2020-11-12 21:36:51 +01:00
When running in background mode the user will have to stop the mount manually :
2017-06-19 14:44:49 +02:00
# Linux
fusermount - u / path / to / local / mount
# OS X
umount / path / to / local / mount
2020-11-12 21:36:51 +01:00
The umount operation can fail , for example when the mountpoint is busy .
When that happens , it is the user ' s responsibility to stop the mount manually .
2020-11-26 14:08:01 +01:00
The size of the mounted file system will be set according to information retrieved
from the remote , the same as returned by the [ rclone about ] ( https : //rclone.org/commands/rclone_about/)
command . Remotes with unlimited storage may report the used size only ,
2021-03-02 20:11:57 +01:00
then an additional 1 PiB of free space is assumed . If the remote does not
2020-11-26 14:08:01 +01:00
[ support ] ( https : //rclone.org/overview/#optional-features) the about feature
2021-03-02 20:11:57 +01:00
at all , then 1 PiB is set as both the total and the free size .
2020-11-26 14:08:01 +01:00
2020-05-25 09:50:20 +02:00
* * Note * * : As of | rclone | 1.52 .2 , | rclone mount | now requires Go version 1.13
2020-07-23 11:45:45 +02:00
or newer on some platforms depending on the underlying FUSE library in use .
2018-01-18 20:16:21 +01:00
# # # Installing on Windows
2017-07-23 13:13:29 +02:00
2020-05-25 09:50:20 +02:00
To run rclone @ on Windows , you will need to
2017-07-23 13:13:29 +02:00
download and install [ WinFsp ] ( http : //www.secfs.net/winfsp/).
2020-05-25 10:08:16 +02:00
[ WinFsp ] ( https : //github.com/billziss-gh/winfsp) is an open source
2017-07-23 13:13:29 +02:00
Windows File System Proxy which makes it easy to write user space file
systems for Windows . It provides a FUSE emulation layer which rclone
2020-11-12 21:36:51 +01:00
uses combination with [ cgofuse ] ( https : //github.com/billziss-gh/cgofuse).
Both of these packages are by Bill Zissimopoulos who was very helpful
2020-05-25 09:50:20 +02:00
during the implementation of rclone @ for Windows .
2017-07-23 13:13:29 +02:00
2020-11-12 21:36:51 +01:00
# # # # Mounting modes on windows
2020-10-28 01:27:36 +01:00
Unlike other operating systems , Microsoft Windows provides a different filesystem
type for network and fixed drives . It optimises access on the assumption fixed
disk drives are fast and reliable , while network drives have relatively high latency
and less reliability . Some settings can also be differentiated between the two types ,
for example that Windows Explorer should just display icons and not create preview
thumbnails for image and video files on network drives .
2020-11-12 21:36:51 +01:00
In most cases , rclone will mount the remote as a normal , fixed disk drive by default .
However , you can also choose to mount it as a remote network drive , often described
as a network share . If you mount an rclone remote using the default , fixed drive mode
and experience unexpected program errors , freezes or other issues , consider mounting
2020-10-28 01:27:36 +01:00
as a network drive instead .
2020-11-12 21:36:51 +01:00
When mounting as a fixed disk drive you can either mount to an unused drive letter ,
2021-02-08 15:50:56 +01:00
or to a path representing a * * non - existent * * subdirectory of an * * existing * * parent
2020-05-25 09:50:20 +02:00
directory or drive . Using the special value | * | will tell rclone to
2020-11-12 21:36:51 +01:00
automatically assign the next available drive letter , starting with Z : and moving backward .
Examples :
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files *
rclone @ remote : path / to / files X :
2021-02-08 15:50:56 +01:00
rclone @ remote : path / to / files C : \ path \ parent \ mount
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files X :
2020-11-12 21:36:51 +01:00
2020-05-25 09:50:20 +02:00
Option | -- volname | can be used to set a custom volume name for the mounted
2020-11-12 21:36:51 +01:00
file system . The default is to use the remote name and path .
2020-05-25 09:50:20 +02:00
To mount as network drive , you can add option | -- network - mode |
to your @ command . Mounting to a directory path is not supported in
2020-11-12 21:36:51 +01:00
this mode , it is a limitation Windows imposes on junctions , so the remote must always
be mounted to a drive letter .
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files X : -- network - mode
2020-10-28 01:27:36 +01:00
2020-05-25 09:50:20 +02:00
A volume name specified with | -- volname | will be used to create the network share path .
A complete UNC path , such as | \ \ cloud \ remote | , optionally with path
| \ \ cloud \ remote \ madeup \ path | , will be used as is . Any other
string will be used as the share part , after a default prefix | \ \ server \ | .
If no volume name is specified then | \ \ server \ share | will be used .
2020-11-12 21:36:51 +01:00
You must make sure the volume name is unique when you are mounting more than one drive ,
or else the mount command will fail . The share name will treated as the volume label for
the mapped drive , shown in Windows Explorer etc , while the complete
2020-05-25 09:50:20 +02:00
| \ \ server \ share | will be reported as the remote UNC path by
| net use | etc , just like a normal network drive mapping .
2020-10-28 01:27:36 +01:00
2020-05-25 09:50:20 +02:00
If you specify a full network share UNC path with | -- volname | , this will implicitely
set the | -- network - mode | option , so the following two examples have same result :
2020-11-12 21:36:51 +01:00
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files X : -- network - mode
rclone @ remote : path / to / files X : -- volname \ \ server \ share
2020-11-12 21:36:51 +01:00
You may also specify the network share UNC path as the mountpoint itself . Then rclone
2020-05-25 09:50:20 +02:00
will automatically assign a drive letter , same as with | * | and use that as
2020-11-12 21:36:51 +01:00
mountpoint , and instead use the UNC path specified as the volume name , as if it were
2020-05-25 09:50:20 +02:00
specified with the | -- volname | option . This will also implicitely set
the | -- network - mode | option . This means the following two examples have same result :
2020-11-12 21:36:51 +01:00
2020-05-25 09:50:20 +02:00
rclone @ remote : path / to / files \ \ cloud \ remote
rclone @ remote : path / to / files * -- volname \ \ cloud \ remote
2020-11-12 21:36:51 +01:00
There is yet another way to enable network mode , and to set the share path ,
and that is to pass the "native" libfuse / WinFsp option directly :
2020-05-25 09:50:20 +02:00
| -- fuse - flag -- VolumePrefix = \ server \ share | . Note that the path
2020-11-12 21:36:51 +01:00
must be with just a single backslash prefix in this case .
* Note : * In previous versions of rclone this was the only supported method .
2020-02-10 15:42:09 +01:00
[ Read more about drive mapping ] ( https : //en.wikipedia.org/wiki/Drive_mapping)
2020-11-12 21:36:51 +01:00
See also [ Limitations ] ( # limitations ) section below .
2020-12-08 00:20:31 +01:00
# # # # Windows filesystem permissions
The FUSE emulation layer on Windows must convert between the POSIX - based
permission model used in FUSE , and the permission model used in Windows ,
based on access - control lists ( ACL ) .
The mounted filesystem will normally get three entries in its access - control list ( ACL ) ,
representing permissions for the POSIX permission scopes : Owner , group and others .
By default , the owner and group will be taken from the current user , and the built - in
group "Everyone" will be used to represent others . The user / group can be customized
with FUSE options "UserName" and "GroupName" ,
2020-05-25 09:50:20 +02:00
e . g . | - o UserName = user123 - o GroupName = "Authenticated Users" | .
2020-12-08 00:20:31 +01:00
The permissions on each entry will be set according to
2020-05-25 09:50:20 +02:00
[ options ] ( # options ) | -- dir - perms | and | -- file - perms | ,
2020-12-08 00:20:31 +01:00
which takes a value in traditional [ numeric notation ] ( https : //en.wikipedia.org/wiki/File-system_permissions#Numeric_notation),
2020-05-25 09:50:20 +02:00
where the default corresponds to | -- file - perms 0666 -- dir - perms 0777 | .
2020-12-08 00:20:31 +01:00
Note that the mapping of permissions is not always trivial , and the result
you see in Windows Explorer may not be exactly like you expected .
For example , when setting a value that includes write access , this will be
mapped to individual permissions "write attributes" , "write data" and "append data" ,
2021-02-17 01:44:28 +01:00
but not "write extended attributes" . Windows will then show this as basic
permission "Special" instead of "Write" , because "Write" includes the
"write extended attributes" permission .
If you set POSIX permissions for only allowing access to the owner , using
| -- file - perms 0600 -- dir - perms 0700 | , the user group and the built - in "Everyone"
group will still be given some special permissions , such as "read attributes"
and "read permissions" , in Windows . This is done for compatibility reasons ,
e . g . to allow users without additional permissions to be able to read basic
metadata about files like in UNIX . One case that may arise is that other programs
( incorrectly ) interprets this as the file being accessible by everyone . For example
an SSH client may warn about "unprotected private key file" .
2021-04-07 08:13:40 +02:00
WinFsp 2021 ( version 1.9 ) introduces a new FUSE option "FileSecurity" ,
2021-02-17 01:44:28 +01:00
that allows the complete specification of file security descriptors using
[ SDDL ] ( https : //docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format).
With this you can work around issues such as the mentioned "unprotected private key file"
by specifying | - o FileSecurity = "D:P(A;;FA;;;OW)" | , for file all access ( FA ) to the owner ( OW ) .
2020-12-08 00:20:31 +01:00
2020-11-12 21:36:51 +01:00
# # # # Windows caveats
2021-04-11 21:52:21 +02:00
Drives created as Administrator are not visible to other accounts ,
not even an account that was elevated to Administrator with the
User Account Control ( UAC ) feature . A result of this is that if you mount
to a drive letter from a Command Prompt run as Administrator , and then try
to access the same drive from Windows Explorer ( which does not run as
Administrator ) , you will not be able to see the mounted drive .
If you don ' t need to access the drive from applications running with
administrative privileges , the easiest way around this is to always
create the mount from a non - elevated command prompt .
To make mapped drives available to the user account that created them
regardless if elevated or not , there is a special Windows setting called
[ linked connections ] ( https : //docs.microsoft.com/en-us/troubleshoot/windows-client/networking/mapped-drives-not-available-from-elevated-command#detail-to-configure-the-enablelinkedconnections-registry-entry)
that can be enabled .
It is also possible to make a drive mount available to everyone on the system ,
by running the process creating it as the built - in SYSTEM account .
There are several ways to do this : One is to use the command - line
utility [ PsExec ] ( https : //docs.microsoft.com/en-us/sysinternals/downloads/psexec),
from Microsoft ' s Sysinternals suite , which has option | - s | to start
processes as the SYSTEM account . Another alternative is to run the mount
command from a Windows Scheduled Task , or a Windows Service , configured
to run as the SYSTEM account . A third alternative is to use the
[ WinFsp . Launcher infrastructure ] ( https : //github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)).
Note that when running rclone as another user , it will not use
the configuration file from your profile unless you tell it to
with the [ | -- config | ] ( https : //rclone.org/docs/#config-config-file) option.
Read more in the [ install documentation ] ( https : //rclone.org/install/).
Note that mapping to a directory path , instead of a drive letter ,
does not suffer from the same limitations .
2020-11-12 21:36:51 +01:00
2018-01-18 20:16:21 +01:00
# # # Limitations
2017-06-19 14:44:49 +02:00
2020-05-25 09:50:20 +02:00
Without the use of | -- vfs - cache - mode | this can only write files
2018-02-10 10:28:20 +01:00
sequentially , it can only seek when reading . This means that many
applications won ' t work with their files on an rclone mount without
2020-05-25 09:50:20 +02:00
| -- vfs - cache - mode writes | or | -- vfs - cache - mode full | .
2021-02-13 01:00:43 +01:00
See the [ VFS File Caching ] ( # vfs - file - caching ) section for more info .
2017-06-19 14:44:49 +02:00
2020-10-13 23:49:58 +02:00
The bucket based remotes ( e . g . Swift , S3 , Google Compute Storage , B2 ,
2019-08-16 16:36:32 +02:00
Hubic ) do not support the concept of empty directories , so empty
2017-06-19 14:44:49 +02:00
directories will have a tendency to disappear once they fall out of
the directory cache .
Only supported on Linux , FreeBSD , OS X and Windows at the moment .
2020-05-25 09:50:20 +02:00
# # # rclone @ vs rclone sync / copy
2017-06-19 14:44:49 +02:00
File systems expect things to be 100 % reliable , whereas cloud storage
systems are a long way from 100 % reliable . The rclone sync / copy
2020-05-25 09:50:20 +02:00
commands cope with this with lots of retries . However rclone @
2017-06-19 14:44:49 +02:00
can ' t use retries in the same way without making local copies of the
2021-02-13 01:00:43 +01:00
uploads . Look at the [ VFS File Caching ] ( # vfs - file - caching )
2020-05-25 09:50:20 +02:00
for solutions to make @ more reliable .
2017-06-19 14:44:49 +02:00
2018-03-02 17:39:42 +01:00
# # # Attribute caching
2020-05-25 09:50:20 +02:00
You can use the flag | -- attr - timeout | to set the time the kernel caches
2020-10-14 00:07:12 +02:00
the attributes ( size , modification time , etc . ) for directory entries .
2018-03-02 17:39:42 +01:00
2020-05-25 09:50:20 +02:00
The default is | 1 s | which caches files just long enough to avoid
2018-03-23 23:42:51 +01:00
too many callbacks to rclone from the kernel .
In theory 0 s should be the correct value for filesystems which can
change outside the control of the kernel . However this causes quite a
few problems such as
2019-07-28 19:47:38 +02:00
[ rclone using too much memory ] ( https : //github.com/rclone/rclone/issues/2157),
2018-03-23 23:42:51 +01:00
[ rclone not serving files to samba ] ( https : //forum.rclone.org/t/rclone-1-39-vs-1-40-mount-issue/5112)
2019-07-28 19:47:38 +02:00
and [ excessive time listing directories ] ( https : //github.com/rclone/rclone/issues/2095#issuecomment-371141147).
2018-03-23 23:42:51 +01:00
The kernel can cache the info about a file for the time given by
2020-05-25 09:50:20 +02:00
| -- attr - timeout | . You may see corruption if the remote file changes
2018-03-23 23:42:51 +01:00
length during this window . It will show up as either a truncated file
2020-05-25 09:50:20 +02:00
or a file with garbage on the end . With | -- attr - timeout 1 s | this is
very unlikely but not impossible . The higher you set | -- attr - timeout |
2018-03-23 23:42:51 +01:00
the more likely it is . The default setting of "1s" is the lowest
setting which mitigates the problems above .
2020-05-25 09:50:20 +02:00
If you set it higher ( | 10 s | or | 1 m | say ) then the kernel will call
2018-03-23 23:42:51 +01:00
back to rclone less often making it more efficient , however there is
more chance of the corruption issue above .
If files don ' t change on the remote outside of the control of rclone
then there is no chance of corruption .
2018-03-02 17:39:42 +01:00
This is the same as setting the attr_timeout option in mount . fuse .
2018-01-18 20:16:21 +01:00
# # # Filters
2017-06-19 14:44:49 +02:00
Note that all the rclone filters can be used to select a subset of the
files to be visible in the mount .
2018-01-18 20:16:21 +01:00
# # # systemd
2017-11-19 23:03:49 +01:00
2020-05-25 09:50:20 +02:00
When running rclone @ as a systemd service , it is possible
2018-01-18 20:16:21 +01:00
to use Type = notify . In this case the service will enter the started state
2017-11-19 23:03:49 +01:00
after the mountpoint has been successfully set up .
2020-05-25 09:50:20 +02:00
Units having the rclone @ service specified as a requirement
2017-11-19 23:03:49 +01:00
will see all files and folders immediately in this mode .
2018-02-18 15:18:12 +01:00
2020-12-08 00:20:31 +01:00
# # # chunked reading
2018-02-18 15:18:12 +01:00
2020-05-25 09:50:20 +02:00
| -- vfs - read - chunk - size | will enable reading the source objects in parts .
2018-02-18 15:18:12 +01:00
This can reduce the used download quota for some remotes by requesting only chunks
from the remote that are actually read at the cost of an increased number of requests .
2020-05-25 09:50:20 +02:00
When | -- vfs - read - chunk - size - limit | is also specified and greater than
| -- vfs - read - chunk - size | , the chunk size for each open file will get doubled
for each chunk read , until the specified value is reached . A value of | - 1 | will disable
2020-10-28 01:27:36 +01:00
the limit and the chunk size will grow indefinitely .
2018-02-18 15:18:12 +01:00
2020-05-25 09:50:20 +02:00
With | -- vfs - read - chunk - size 100 M | and | -- vfs - read - chunk - size - limit 0 |
2020-10-28 01:27:36 +01:00
the following parts will be downloaded : 0 - 100 M , 100 M - 200 M , 200 M - 300 M , 300 M - 400 M and so on .
2020-05-25 09:50:20 +02:00
When | -- vfs - read - chunk - size - limit 500 M | is specified , the result would be
2018-02-18 15:18:12 +01:00
0 - 100 M , 100 M - 300 M , 300 M - 700 M , 700 M - 1200 M , 1200 M - 1700 M and so on .
2020-05-25 09:50:20 +02:00
` , "|", " ` "), " @ " , commandName ) + vfs . Help ,
2017-06-19 14:44:49 +02:00
Run : func ( command * cobra . Command , args [ ] string ) {
cmd . CheckArgs ( 2 , 2 , command , args )
2020-07-29 15:23:00 +02:00
opt := Opt // make a copy of the options
2018-08-21 10:41:16 +02:00
2020-07-29 15:23:00 +02:00
if opt . Daemon {
2018-08-21 10:41:16 +02:00
config . PassConfigKeyForDaemonization = true
}
2019-01-10 15:18:00 +01:00
mountpoint := args [ 1 ]
2018-05-07 18:58:16 +02:00
fdst := cmd . NewFsDir ( args )
2019-01-10 15:18:00 +01:00
if fdst . Name ( ) == "" || fdst . Name ( ) == "local" {
err := checkMountpointOverlap ( fdst . Root ( ) , mountpoint )
if err != nil {
log . Fatalf ( "Fatal error: %v" , err )
}
}
2017-06-19 14:44:49 +02:00
// Show stats if the user has specifically requested them
if cmd . ShowStats ( ) {
2018-10-03 22:46:18 +02:00
defer cmd . StartStats ( ) ( )
2017-06-19 14:44:49 +02:00
}
2020-10-26 11:36:57 +01:00
// Inform about ignored flags on Windows,
// and if not on Windows and not --allow-non-empty flag is used
// verify that mountpoint is empty.
if runtime . GOOS == "windows" {
if opt . AllowNonEmpty {
fs . Logf ( nil , "--allow-non-empty flag does nothing on Windows" )
}
if opt . AllowRoot {
fs . Logf ( nil , "--allow-root flag does nothing on Windows" )
}
if opt . AllowOther {
fs . Logf ( nil , "--allow-other flag does nothing on Windows" )
}
} else if ! opt . AllowNonEmpty {
2019-01-10 15:18:00 +01:00
err := checkMountEmpty ( mountpoint )
2017-11-09 01:37:27 +01:00
if err != nil {
log . Fatalf ( "Fatal error: %v" , err )
}
}
2018-05-03 10:34:07 +02:00
// Work out the volume name, removing special
// characters from it if necessary
2020-07-29 15:23:00 +02:00
if opt . VolumeName == "" {
opt . VolumeName = fdst . Name ( ) + ":" + fdst . Root ( )
2018-05-03 10:34:07 +02:00
}
2020-07-29 15:23:00 +02:00
opt . VolumeName = strings . Replace ( opt . VolumeName , ":" , " " , - 1 )
opt . VolumeName = strings . Replace ( opt . VolumeName , "/" , " " , - 1 )
opt . VolumeName = strings . TrimSpace ( opt . VolumeName )
if runtime . GOOS == "windows" && len ( opt . VolumeName ) > 32 {
opt . VolumeName = opt . VolumeName [ : 32 ]
2020-03-03 19:59:47 +01:00
}
2018-05-03 10:34:07 +02:00
2018-03-02 14:30:04 +01:00
// Start background task if --background is specified
2020-07-29 15:23:00 +02:00
if opt . Daemon {
2018-03-02 14:30:04 +01:00
daemonized := startBackgroundMode ( )
if daemonized {
return
}
}
2020-07-22 18:58:49 +02:00
VFS := vfs . New ( fdst , & vfsflags . Opt )
2020-07-29 15:23:00 +02:00
err := Mount ( VFS , mountpoint , mount , & opt )
2017-06-19 14:44:49 +02:00
if err != nil {
log . Fatalf ( "Fatal error: %v" , err )
}
} ,
}
// Register the command
2019-10-11 17:58:11 +02:00
cmd . Root . AddCommand ( commandDefinition )
2017-06-19 14:44:49 +02:00
// Add flags
2019-10-11 17:55:04 +02:00
cmdFlags := commandDefinition . Flags ( )
2020-07-23 18:17:01 +02:00
AddFlags ( cmdFlags )
2019-10-11 17:55:04 +02:00
vfsflags . AddFlags ( cmdFlags )
2017-10-24 22:06:06 +02:00
2019-10-11 17:58:11 +02:00
return commandDefinition
2017-06-19 14:44:49 +02:00
}
2018-06-26 10:26:34 +02:00
2020-05-25 08:05:53 +02:00
// ClipBlocks clips the blocks pointed to the OS max
2018-06-26 10:26:34 +02:00
func ClipBlocks ( b * uint64 ) {
var max uint64
switch runtime . GOOS {
case "windows" :
2019-01-25 18:31:56 +01:00
if runtime . GOARCH == "386" {
max = ( 1 << 32 ) - 1
} else {
max = ( 1 << 43 ) - 1
}
2018-06-26 10:26:34 +02:00
case "darwin" :
// OSX FUSE only supports 32 bit number of blocks
// https://github.com/osxfuse/osxfuse/issues/396
max = ( 1 << 32 ) - 1
default :
// no clipping
return
}
if * b > max {
* b = max
}
}
2020-07-23 14:08:38 +02:00
// Mount mounts the remote at mountpoint.
//
// If noModTime is set then it
2020-07-23 18:17:01 +02:00
func Mount ( VFS * vfs . VFS , mountpoint string , mount MountFn , opt * Options ) error {
2020-07-27 16:24:28 +02:00
if opt == nil {
2020-07-23 18:17:01 +02:00
opt = & DefaultOpt
}
2020-07-23 14:08:38 +02:00
// Mount it
2020-07-23 18:17:01 +02:00
errChan , unmount , err := mount ( VFS , mountpoint , opt )
2020-07-23 14:08:38 +02:00
if err != nil {
return errors . Wrap ( err , "failed to mount FUSE fs" )
}
// Unmount on exit
2020-11-27 11:50:10 +01:00
var finaliseOnce sync . Once
finalise := func ( ) {
finaliseOnce . Do ( func ( ) {
_ = sysdnotify . Stopping ( )
_ = unmount ( )
} )
}
fnHandle := atexit . Register ( finalise )
2020-07-23 14:08:38 +02:00
defer atexit . Unregister ( fnHandle )
// Notify systemd
2020-09-18 17:37:54 +02:00
if err := sysdnotify . Ready ( ) ; err != nil {
2020-07-23 14:08:38 +02:00
return errors . Wrap ( err , "failed to notify systemd" )
}
// Reload VFS cache on SIGHUP
sigHup := make ( chan os . Signal , 1 )
signal . Notify ( sigHup , syscall . SIGHUP )
waitloop :
for {
select {
// umount triggered outside the app
case err = <- errChan :
break waitloop
// user sent SIGHUP to clear the cache
case <- sigHup :
root , err := VFS . Root ( )
if err != nil {
fs . Errorf ( VFS . Fs ( ) , "Error reading root: %v" , err )
} else {
root . ForgetAll ( )
}
}
}
2020-11-27 11:50:10 +01:00
finalise ( )
2020-07-23 14:08:38 +02:00
if err != nil {
return errors . Wrap ( err , "failed to umount FUSE fs" )
}
return nil
}