local: implement --local-direct-io to use direct i/o to write files

backend/local/directio_other.go

@@ -0,0 +1,13 @@
+//go:build !linux
+
+package local
+
+import (
+	"os"
+)
+
+const directIOSupported = false
+
+func directIOOpenFile(name string, flag int, perm os.FileMode) (file *os.File, err error) {
+	panic("no implementation")
+}

backend/local/directio_unix.go

@@ -0,0 +1,14 @@
+//go:build linux
+
+package local
+
+import (
+	"os"
+	"syscall"
+)
+
+const directIOSupported = true
+
+func directIOOpenFile(name string, flag int, perm os.FileMode) (file *os.File, err error) {
+	return os.OpenFile(name, flag|syscall.O_DIRECT, perm)
+}

backend/local/local.go

@@ -17,6 +17,7 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/brk0v/directio"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -271,6 +272,18 @@ enabled, rclone will no longer update the modtime after copying a file.`,
 				Default:  false,
 				Advanced: true,
 			},
+			{
+				Name:     "direct_io",
+				Help:     `Using direct I/O to write files (unix only).`,
+				Default:  false,
+				Advanced: true,
+			},
+			{
+				Name:     "direct_io_block_size",
+				Help:     `If using direct I/O, this sets the block size to use.`,
+				Default:  4 * 1024 * 1024, // 1MiB
+				Advanced: true,
+			},
 			{
 				Name: "time_type",
 				Help: `Set what kind of time is returned.
@@ -333,6 +346,8 @@ type Options struct {
 	NoPreAllocate     bool                 `config:"no_preallocate"`
 	NoSparse          bool                 `config:"no_sparse"`
 	NoSetModTime      bool                 `config:"no_set_modtime"`
+	DirectIO          bool                 `config:"direct_io"`
+	DirectIOBlockSize fs.SizeSuffix        `config:"direct_io_block_size"`
 	TimeType          timeType             `config:"time_type"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 	NoClone           bool                 `config:"no_clone"`
@@ -393,6 +408,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.TranslateSymlinks && opt.FollowSymlinks {
 		return nil, errLinksAndCopyLinks
 	}
+	if !directIOSupported && opt.DirectIO {
+		return nil, errors.New("direct IO is not supported on this platform")
+	}
 
 	f := &Fs{
 		name:   name,
@@ -428,6 +446,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// Disable server-side copy when --local-no-clone is set
 		f.features.Copy = nil
 	}
+	if opt.DirectIO {
+		// Disable multi-thread copy when --local-direct-io is set
+		f.features.OpenWriterAt = nil
+	}
 
 	// Check to see if this points to a file
 	fi, err := f.lstat(f.root)
@@ -1382,7 +1404,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// If it is a translated link, just read in the contents, and
 	// then create a symlink
 	if !o.translatedLink {
-		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+		flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
+		var f *os.File
+		if o.fs.opt.DirectIO {
+			f, err = directIOOpenFile(o.path, flags, 0666)
+		} else {
+			f, err = file.OpenFile(o.path, flags, 0666)
+		}
 		if err != nil {
 			if runtime.GOOS == "windows" && os.IsPermission(err) {
 				// If permission denied on Windows might be trying to update a
@@ -1408,6 +1436,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			}
 		}
 		out = f
+		if o.fs.opt.DirectIO {
+			out, err = directio.NewSize(f, int(o.fs.opt.DirectIOBlockSize))
+			if err != nil {
+				_ = f.Close()
+				return err
+			}
+		}
 	} else {
 		out = nopWriterCloser{&symlinkData}
 	}

docs/content/local.md

@@ -588,6 +588,28 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --local-direct-io
+
+Using direct I/O to write files (unix only).
+
+Properties:
+
+- Config:      direct_io
+- Env Var:     RCLONE_LOCAL_DIRECT_IO
+- Type:        bool
+- Default:     false
+
+#### --local-direct-io-block-size
+
+If using direct I/O, this sets the block size to use.
+
+Properties:
+
+- Config:      direct_io_block_size
+- Env Var:     RCLONE_LOCAL_DIRECT_IO_BLOCK_SIZE
+- Type:        SizeSuffix
+- Default:     4Mi
+
 #### --local-time-type
 
 Set what kind of time is returned.

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3
 	github.com/aws/smithy-go v1.20.3
+	github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee
 	github.com/buengese/sgzip v0.1.1
 	github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891
 	github.com/colinmarc/hdfs/v2 v2.4.0

go.sum

@@ -148,6 +148,8 @@ github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW
 github.com/bradenaw/juniper v0.15.2/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
+github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee h1:gwVgfLo8dvQMauElELV1b1kwMuUh2ETzX6weEbdAKgA=
+github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee/go.mod h1:M/KA3XJG5PJaApPiv4gWNsgcSJquOQTqumZNLyYE0KM=
 github.com/buengese/sgzip v0.1.1 h1:ry+T8l1mlmiWEsDrH/YHZnCVWD2S3im1KLsyO+8ZmTU=
 github.com/buengese/sgzip v0.1.1/go.mod h1:i5ZiXGF3fhV7gL1xaRRL1nDnmpNj0X061FQzOS8VMas=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=