extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 16:34:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

serve s3: adjust to move of Mikubill/gofakes3 to rclone/gofakes3

Browse Source 
This also updates the interface which has gained a ctx parameter in
the mean time.
This commit is contained in:
Nick Craig-Wood 2024-04-13 17:50:11 +01:00
parent ae76498a38
commit c87097109b
8 changed files with 26 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/serve/s3/backend.go
View File

@ -10,8 +10,8 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/Mikubill/gofakes3"
"github.com/ncw/swift/v2" "github.com/ncw/swift/v2"
"github.com/rclone/gofakes3"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/vfs" "github.com/rclone/rclone/vfs"
) )
@ -39,7 +39,7 @@ func newBackend(vfs *vfs.VFS, opt *Options) gofakes3.Backend {
} }
// ListBuckets always returns the default bucket. // ListBuckets always returns the default bucket.
func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) { func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) {
dirEntries, err := getDirEntries("/", b.vfs) dirEntries, err := getDirEntries("/", b.vfs)
if err != nil { if err != nil {
return nil, err return nil, err
@ -59,7 +59,7 @@ func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) {
} }
// ListBucket lists the objects in the given bucket. // ListBucket lists the objects in the given bucket.
func (b *s3Backend) ListBucket(bucket string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { func (b *s3Backend) ListBucket(ctx context.Context, bucket string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
_, err := b.vfs.Stat(bucket) _, err := b.vfs.Stat(bucket)
if err != nil { if err != nil {
return nil, gofakes3.BucketNotFound(bucket) return nil, gofakes3.BucketNotFound(bucket)
@ -93,7 +93,7 @@ func (b *s3Backend) ListBucket(bucket string, prefix *gofakes3.Prefix, page gofa
// HeadObject returns the fileinfo for the given object name. // HeadObject returns the fileinfo for the given object name.
// //
// Note that the metadata is not supported yet. // Note that the metadata is not supported yet.
func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) { func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName string) (*gofakes3.Object, error) {
_, err := b.vfs.Stat(bucketName) _, err := b.vfs.Stat(bucketName)
if err != nil { if err != nil {
return nil, gofakes3.BucketNotFound(bucketName) return nil, gofakes3.BucketNotFound(bucketName)
@ -140,7 +140,7 @@ func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object,
} }
// GetObject fetchs the object from the filesystem. // GetObject fetchs the object from the filesystem.
func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) { func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
_, err = b.vfs.Stat(bucketName) _, err = b.vfs.Stat(bucketName)
if err != nil { if err != nil {
return nil, gofakes3.BucketNotFound(bucketName) return nil, gofakes3.BucketNotFound(bucketName)
@ -214,7 +214,7 @@ func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofak
} }
// TouchObject creates or updates meta on specified object. // TouchObject creates or updates meta on specified object.
func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) { func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
_, err = b.vfs.Stat(fp) _, err = b.vfs.Stat(fp)
if err == vfs.ENOENT { if err == vfs.ENOENT {
f, err := b.vfs.Create(fp) f, err := b.vfs.Create(fp)
@ -222,7 +222,7 @@ func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofak
return result, err return result, err
} }
_ = f.Close() _ = f.Close()
return b.TouchObject(fp, meta) return b.TouchObject(ctx, fp, meta)
} else if err != nil { } else if err != nil {
return result, err return result, err
} }
@ -255,6 +255,7 @@ func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofak
// PutObject creates or overwrites the object with the given name. // PutObject creates or overwrites the object with the given name.
func (b *s3Backend) PutObject( func (b *s3Backend) PutObject(
ctx context.Context,
bucketName, objectName string, bucketName, objectName string,
meta map[string]string, meta map[string]string,
input io.Reader, size int64, input io.Reader, size int64,
@ -323,7 +324,7 @@ func (b *s3Backend) PutObject(
} }
// DeleteMulti deletes multiple objects in a single request. // DeleteMulti deletes multiple objects in a single request.
func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
for _, object := range objects { for _, object := range objects {
if err := b.deleteObject(bucketName, object); err != nil { if err := b.deleteObject(bucketName, object); err != nil {
fs.Errorf("serve s3", "delete object failed: %v", err) fs.Errorf("serve s3", "delete object failed: %v", err)
@ -343,7 +344,7 @@ func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result go
} }
// DeleteObject deletes the object with the given name. // DeleteObject deletes the object with the given name.
func (b *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { func (b *s3Backend) DeleteObject(ctx context.Context, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
return result, b.deleteObject(bucketName, objectName) return result, b.deleteObject(bucketName, objectName)
} }
@ -367,7 +368,7 @@ func (b *s3Backend) deleteObject(bucketName, objectName string) error {
} }
// CreateBucket creates a new bucket. // CreateBucket creates a new bucket.
func (b *s3Backend) CreateBucket(name string) error { func (b *s3Backend) CreateBucket(ctx context.Context, name string) error {
_, err := b.vfs.Stat(name) _, err := b.vfs.Stat(name)
if err != nil && err != vfs.ENOENT { if err != nil && err != vfs.ENOENT {
return gofakes3.ErrInternal return gofakes3.ErrInternal
@ -384,7 +385,7 @@ func (b *s3Backend) CreateBucket(name string) error {
} }
// DeleteBucket deletes the bucket with the given name. // DeleteBucket deletes the bucket with the given name.
func (b *s3Backend) DeleteBucket(name string) error { func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error {
_, err := b.vfs.Stat(name) _, err := b.vfs.Stat(name)
if err != nil { if err != nil {
return gofakes3.BucketNotFound(name) return gofakes3.BucketNotFound(name)
@ -398,7 +399,7 @@ func (b *s3Backend) DeleteBucket(name string) error {
} }
// BucketExists checks if the bucket exists. // BucketExists checks if the bucket exists.
func (b *s3Backend) BucketExists(name string) (exists bool, err error) { func (b *s3Backend) BucketExists(ctx context.Context, name string) (exists bool, err error) {
_, err = b.vfs.Stat(name) _, err = b.vfs.Stat(name)
if err != nil { if err != nil {
return false, nil return false, nil
@ -408,7 +409,7 @@ func (b *s3Backend) BucketExists(name string) (exists bool, err error) {
} }
// CopyObject copy specified object from srcKey to dstKey. // CopyObject copy specified object from srcKey to dstKey.
func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
fp := path.Join(srcBucket, srcKey) fp := path.Join(srcBucket, srcKey)
if srcBucket == dstBucket && srcKey == dstKey { if srcBucket == dstBucket && srcKey == dstKey {
b.meta.Store(fp, meta) b.meta.Store(fp, meta)
@ -433,7 +434,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
return return
} }
c, err := b.GetObject(srcBucket, srcKey, nil) c, err := b.GetObject(ctx, srcBucket, srcKey, nil)
if err != nil { if err != nil {
return return
} }
@ -450,7 +451,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
meta["mtime"] = swift.TimeToFloatString(cStat.ModTime()) meta["mtime"] = swift.TimeToFloatString(cStat.ModTime())
} }
_, err = b.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) _, err = b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size)
if err != nil { if err != nil {
return return
} }

2
cmd/serve/s3/list.go
View File

@ -4,7 +4,7 @@ import (
"path" "path"
"strings" "strings"
"github.com/Mikubill/gofakes3" "github.com/rclone/gofakes3"
) )
func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {

2
cmd/serve/s3/logger.go
View File

@ -3,7 +3,7 @@ package s3
import ( import (
"fmt" "fmt"
"github.com/Mikubill/gofakes3" "github.com/rclone/gofakes3"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
) )

2
cmd/serve/s3/pager.go
View File

@ -4,7 +4,7 @@ package s3
import ( import (
"sort" "sort"
"github.com/Mikubill/gofakes3" "github.com/rclone/gofakes3"
) )
// pager splits the object list into smulitply pages. // pager splits the object list into smulitply pages.

2
cmd/serve/s3/server.go
View File

@ -7,8 +7,8 @@ import (
"math/rand" "math/rand"
"net/http" "net/http"
"github.com/Mikubill/gofakes3"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/rclone/gofakes3"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/hash"
httplib "github.com/rclone/rclone/lib/http" httplib "github.com/rclone/rclone/lib/http"

2
cmd/serve/s3/utils.go
View File

@ -9,7 +9,7 @@ import (
"path" "path"
"strings" "strings"
"github.com/Mikubill/gofakes3" "github.com/rclone/gofakes3"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/vfs" "github.com/rclone/rclone/vfs"

2
go.mod
View File

@ -10,7 +10,6 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700
github.com/Unknwon/goconfig v1.0.0 github.com/Unknwon/goconfig v1.0.0
github.com/a8m/tree v0.0.0-20240104212747-2c8764a5f17e github.com/a8m/tree v0.0.0-20240104212747-2c8764a5f17e
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3
@ -158,6 +157,7 @@ require (
github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
github.com/rclone/gofakes3 v0.0.3-0.20240413171058-b7a9fdb78ddb // indirect
github.com/relvacode/iso8601 v1.3.0 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect
github.com/rs/xid v1.5.0 // indirect github.com/rs/xid v1.5.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect

6
go.sum
View File

@ -60,8 +60,6 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIf
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q=
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo=
@ -458,6 +456,10 @@ github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5
github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q= github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/rclone/gofakes3 v0.0.3-0.20240413163749-372c3e746e45 h1:qrH/G7j3Ru2E6tuKaNvpA+KD9uQSujTTMyQoonCJ7kU=
github.com/rclone/gofakes3 v0.0.3-0.20240413163749-372c3e746e45/go.mod h1:L0VIBE0mT6ArN/5dfHsJm3UjqCpi5B/cdN+qWDNh7ko=
github.com/rclone/gofakes3 v0.0.3-0.20240413171058-b7a9fdb78ddb h1:HJJ7XgRBfXew3EosVk45aGPJRY5wSTSpmAJqz8Kiw0w=
github.com/rclone/gofakes3 v0.0.3-0.20240413171058-b7a9fdb78ddb/go.mod h1:L0VIBE0mT6ArN/5dfHsJm3UjqCpi5B/cdN+qWDNh7ko=
github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko=
github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=