s3: empty directory markers - #3453

backend/s3/s3.go

@@ -2196,6 +2196,15 @@ See: https://github.com/rclone/rclone/issues/4673, https://github.com/rclone/rcl
 This is usually set to a CloudFront CDN URL as AWS S3 offers
 cheaper egress for data downloaded through the CloudFront network.`,
 			Advanced: true,
+		}, {
+			Name:     "directory_markers",
+			Default:  false,
+			Advanced: true,
+			Help: `Upload an empty object with a trailing slash in name when new directory is created
+
+Empty folders are unsupported for bucket based remotes, this option creates an empty 
+object named "/", to persist folder.
+`,
 		}, {
 			Name: "use_multipart_etag",
 			Help: `Whether to use ETag in multipart uploads for verification
@@ -2425,6 +2434,7 @@ type Options struct {
 	MemoryPoolUseMmap     bool                 `config:"memory_pool_use_mmap"`
 	DisableHTTP2          bool                 `config:"disable_http2"`
 	DownloadURL           string               `config:"download_url"`
+	DirectoryMarkers      bool                 `config:"directory_markers"`
 	UseMultipartEtag      fs.Tristate          `config:"use_multipart_etag"`
 	UsePresignedRequest   bool                 `config:"use_presigned_request"`
 	Versions              bool                 `config:"versions"`
@@ -3879,7 +3889,27 @@ func (f *Fs) bucketExists(ctx context.Context, bucket string) (bool, error) {
 // Mkdir creates the bucket if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	bucket, _ := f.split(dir)
-	return f.makeBucket(ctx, bucket)
+	e := f.makeBucket(ctx, bucket)
+	if e != nil {
+		return e
+	}
+	// Create directory marker file
+	if f.opt.DirectoryMarkers && bucket != "" && dir != "" {
+		markerFilePath := fmt.Sprintf("%s/", dir)
+		markerFileContent := io.Reader(strings.NewReader(""))
+		markerFileObject := &Object{
+			fs:     f,
+			remote: markerFilePath,
+			meta: map[string]string{
+				metaMtime: swift.TimeToFloatString(time.Now()),
+			},
+		}
+		_, e := f.Put(ctx, markerFileContent, markerFileObject)
+		if e != nil {
+			return e
+		}
+	}
+	return nil
 }
 
 // makeBucket creates the bucket if it doesn't exist
@@ -3920,6 +3950,15 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	bucket, directory := f.split(dir)
+	// Remove directory marker file
+	if f.opt.DirectoryMarkers && bucket != "" && dir != "" {
+		markerFilePath := fmt.Sprintf("%s/", dir)
+		markerFileObject := &Object{
+			fs:     f,
+			remote: markerFilePath,
+		}
+		_ = markerFileObject.Remove(ctx)
+	}
 	if bucket == "" || directory != "" {
 		return nil
 	}

backend/s3/s3_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
@@ -20,6 +21,24 @@ func TestIntegration(t *testing.T) {
 	})
 }
 
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	name := "TestS3"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:  name + ":",
+		NilObject:   (*Object)(nil),
+		TiersToTest: []string{"STANDARD", "STANDARD_IA"},
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: minChunkSize,
+		},
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "directory_markers", Value: "true"},
+		},
+	})
+}
+
 func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadChunkSize(cs)
 }