local: define OpenWriterAt interface and test and implement it #2252

This will enable multipart downloads in future commits

backend/cache/cache_test.go

@@ -17,7 +17,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   "TestCache:",
 		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs"},
+		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
 	})
 }

backend/crypt/crypt_test.go

@@ -23,6 +23,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   *fstest.RemoteName,
 		NilObject:                    (*crypt.Object)(nil),
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
@@ -43,6 +44,7 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
@@ -63,6 +65,7 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
@@ -84,6 +87,7 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
 		SkipBadWindowsCharacters:     true,
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }

backend/local/local.go

@@ -998,6 +998,36 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	return o.lstat()
 }
 
+// OpenWriterAt opens with a handle for random access writes
+//
+// Pass in the remote desired and the size if known.
+//
+// It truncates any existing object
+func (f *Fs) OpenWriterAt(remote string, size int64) (fs.WriterAtCloser, error) {
+	// Temporary Object under construction
+	o := f.newObject(remote, "")
+
+	err := o.mkdirAll()
+	if err != nil {
+		return nil, err
+	}
+
+	if o.translatedLink {
+		return nil, errors.New("can't open a symlink for random writing")
+	}
+
+	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return nil, err
+	}
+	// Pre-allocate the file for performance reasons
+	err = preAllocate(size, out)
+	if err != nil {
+		fs.Debugf(o, "Failed to pre-allocate: %v", err)
+	}
+	return out, nil
+}
+
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
 	// Don't overwrite the info if we don't need to
@@ -1144,5 +1174,6 @@ var (
 	_ fs.PutStreamer    = &Fs{}
 	_ fs.Mover          = &Fs{}
 	_ fs.DirMover       = &Fs{}
+	_ fs.OpenWriterAter = &Fs{}
 	_ fs.Object         = &Object{}
 )

fs/fs.go

@@ -427,6 +427,12 @@ type Usage struct {
 	Objects *int64 `json:"objects,omitempty"` // objects in the storage system
 }
 
+// WriterAtCloser wraps io.WriterAt and io.Closer
+type WriterAtCloser interface {
+	io.WriterAt
+	io.Closer
+}
+
 // Features describe the optional features of the Fs
 type Features struct {
 	// Feature flags, whether Fs
@@ -548,6 +554,13 @@ type Features struct {
 
 	// About gets quota information from the Fs
 	About func() (*Usage, error)
+
+	// OpenWriterAt opens with a handle for random access writes
+	//
+	// Pass in the remote desired and the size if known.
+	//
+	// It truncates any existing object
+	OpenWriterAt func(remote string, size int64) (WriterAtCloser, error)
 }
 
 // Disable nil's out the named feature.  If it isn't found then it
@@ -640,6 +653,9 @@ func (ft *Features) Fill(f Fs) *Features {
 	if do, ok := f.(Abouter); ok {
 		ft.About = do.About
 	}
+	if do, ok := f.(OpenWriterAter); ok {
+		ft.OpenWriterAt = do.OpenWriterAt
+	}
 	return ft.DisableList(Config.DisableFeatures)
 }
 
@@ -705,6 +721,9 @@ func (ft *Features) Mask(f Fs) *Features {
 	if mask.About == nil {
 		ft.About = nil
 	}
+	if mask.OpenWriterAt == nil {
+		ft.OpenWriterAt = nil
+	}
 	return ft.DisableList(Config.DisableFeatures)
 }
 
@@ -904,6 +923,16 @@ type Abouter interface {
 	About() (*Usage, error)
 }
 
+// OpenWriterAter is an optional interface for Fs
+type OpenWriterAter interface {
+	// OpenWriterAt opens with a handle for random access writes
+	//
+	// Pass in the remote desired and the size if known.
+	//
+	// It truncates any existing object
+	OpenWriterAt(remote string, size int64) (WriterAtCloser, error)
+}
+
 // ObjectsChan is a channel of Objects
 type ObjectsChan chan Object
 

fstest/fstests/fstests.go

@@ -674,6 +674,36 @@ func Run(t *testing.T, opt *Opt) {
 			}
 		})
 
+		t.Run("FsOpenWriterAt", func(t *testing.T) {
+			skipIfNotOk(t)
+			openWriterAt := remote.Features().OpenWriterAt
+			if openWriterAt == nil {
+				t.Skip("FS has no OpenWriterAt interface")
+			}
+			path := "writer-at-subdir/writer-at-file"
+			out, err := openWriterAt(path, -1)
+			require.NoError(t, err)
+
+			var n int
+			n, err = out.WriteAt([]byte("def"), 3)
+			assert.NoError(t, err)
+			assert.Equal(t, 3, n)
+			n, err = out.WriteAt([]byte("ghi"), 6)
+			assert.NoError(t, err)
+			assert.Equal(t, 3, n)
+			n, err = out.WriteAt([]byte("abc"), 0)
+			assert.NoError(t, err)
+			assert.Equal(t, 3, n)
+
+			assert.NoError(t, out.Close())
+
+			obj := findObject(t, remote, path)
+			assert.Equal(t, "abcdefghi", readObject(t, obj, -1), "contents of file differ")
+
+			assert.NoError(t, obj.Remove())
+			assert.NoError(t, remote.Rmdir("writer-at-subdir"))
+		})
+
 		// TestFsChangeNotify tests that changes are properly
 		// propagated
 		//