diff --git a/backend/local/directio_other.go b/backend/local/directio_other.go new file mode 100644 index 000000000..86d20392a --- /dev/null +++ b/backend/local/directio_other.go @@ -0,0 +1,13 @@ +//go:build !linux + +package local + +import ( + "os" +) + +const directIOSupported = false + +func directIOOpenFile(name string, flag int, perm os.FileMode) (file *os.File, err error) { + panic("no implementation") +} diff --git a/backend/local/directio_unix.go b/backend/local/directio_unix.go new file mode 100644 index 000000000..5bf47d30a --- /dev/null +++ b/backend/local/directio_unix.go @@ -0,0 +1,14 @@ +//go:build linux + +package local + +import ( + "os" + "syscall" +) + +const directIOSupported = true + +func directIOOpenFile(name string, flag int, perm os.FileMode) (file *os.File, err error) { + return os.OpenFile(name, flag|syscall.O_DIRECT, perm) +} diff --git a/backend/local/local.go b/backend/local/local.go index 1a96d8036..fb6e37776 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -17,6 +17,7 @@ import ( "time" "unicode/utf8" + "github.com/brk0v/directio" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/config" @@ -271,6 +272,18 @@ enabled, rclone will no longer update the modtime after copying a file.`, Default: false, Advanced: true, }, + { + Name: "direct_io", + Help: `Using direct I/O to write files (unix only).`, + Default: false, + Advanced: true, + }, + { + Name: "direct_io_block_size", + Help: `If using direct I/O, this sets the block size to use.`, + Default: 4 * 1024 * 1024, // 1MiB + Advanced: true, + }, { Name: "time_type", Help: `Set what kind of time is returned. @@ -333,6 +346,8 @@ type Options struct { NoPreAllocate bool `config:"no_preallocate"` NoSparse bool `config:"no_sparse"` NoSetModTime bool `config:"no_set_modtime"` + DirectIO bool `config:"direct_io"` + DirectIOBlockSize fs.SizeSuffix `config:"direct_io_block_size"` TimeType timeType `config:"time_type"` Enc encoder.MultiEncoder `config:"encoding"` NoClone bool `config:"no_clone"` @@ -393,6 +408,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if opt.TranslateSymlinks && opt.FollowSymlinks { return nil, errLinksAndCopyLinks } + if !directIOSupported && opt.DirectIO { + return nil, errors.New("direct IO is not supported on this platform") + } f := &Fs{ name: name, @@ -428,6 +446,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Disable server-side copy when --local-no-clone is set f.features.Copy = nil } + if opt.DirectIO { + // Disable multi-thread copy when --local-direct-io is set + f.features.OpenWriterAt = nil + } // Check to see if this points to a file fi, err := f.lstat(f.root) @@ -1382,7 +1404,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // If it is a translated link, just read in the contents, and // then create a symlink if !o.translatedLink { - f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC + var f *os.File + if o.fs.opt.DirectIO { + f, err = directIOOpenFile(o.path, flags, 0666) + } else { + f, err = file.OpenFile(o.path, flags, 0666) + } if err != nil { if runtime.GOOS == "windows" && os.IsPermission(err) { // If permission denied on Windows might be trying to update a @@ -1408,6 +1436,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } } out = f + if o.fs.opt.DirectIO { + out, err = directio.NewSize(f, int(o.fs.opt.DirectIOBlockSize)) + if err != nil { + _ = f.Close() + return err + } + } } else { out = nopWriterCloser{&symlinkData} } diff --git a/docs/content/local.md b/docs/content/local.md index 9d5c8363c..c40d92160 100644 --- a/docs/content/local.md +++ b/docs/content/local.md @@ -588,6 +588,28 @@ Properties: - Type: bool - Default: false +#### --local-direct-io + +Using direct I/O to write files (unix only). + +Properties: + +- Config: direct_io +- Env Var: RCLONE_LOCAL_DIRECT_IO +- Type: bool +- Default: false + +#### --local-direct-io-block-size + +If using direct I/O, this sets the block size to use. + +Properties: + +- Config: direct_io_block_size +- Env Var: RCLONE_LOCAL_DIRECT_IO_BLOCK_SIZE +- Type: SizeSuffix +- Default: 4Mi + #### --local-time-type Set what kind of time is returned. diff --git a/go.mod b/go.mod index fd6c96bc8..a3b286f3b 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 github.com/aws/smithy-go v1.20.3 + github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee github.com/buengese/sgzip v0.1.1 github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 github.com/colinmarc/hdfs/v2 v2.4.0 diff --git a/go.sum b/go.sum index e1dff78eb..506dd305e 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,8 @@ github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW github.com/bradenaw/juniper v0.15.2/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= +github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee h1:gwVgfLo8dvQMauElELV1b1kwMuUh2ETzX6weEbdAKgA= +github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee/go.mod h1:M/KA3XJG5PJaApPiv4gWNsgcSJquOQTqumZNLyYE0KM= github.com/buengese/sgzip v0.1.1 h1:ry+T8l1mlmiWEsDrH/YHZnCVWD2S3im1KLsyO+8ZmTU= github.com/buengese/sgzip v0.1.1/go.mod h1:i5ZiXGF3fhV7gL1xaRRL1nDnmpNj0X061FQzOS8VMas= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=