From d3dcc6115444e8f238f31820c0273fbac05ced5d Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 12 Sep 2023 16:35:13 +0545 Subject: [PATCH] serve s3: fixes before merge - add context to log and fallthrough to error log level - test: use rclone random lib to generate random strings - calculate hash from vfs cache if file is uploading - add server started log with server url - remove md5 hasher --- cmd/serve/s3/backend.go | 4 +--- cmd/serve/s3/logger.go | 15 +++++++-------- cmd/serve/s3/s3.go | 5 ++++- cmd/serve/s3/s3_test.go | 18 +++--------------- cmd/serve/s3/server.go | 6 ++++++ cmd/serve/s3/utils.go | 29 ++++++++++++++++++++++++++--- go.mod | 7 ++++++- go.sum | 14 +++++++++----- 8 files changed, 62 insertions(+), 36 deletions(-) diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go index 227dc14d3..eacd91230 100644 --- a/cmd/serve/s3/backend.go +++ b/cmd/serve/s3/backend.go @@ -3,7 +3,6 @@ package s3 import ( "context" - "crypto/md5" "encoding/hex" "io" "log" @@ -306,8 +305,7 @@ func (db *s3Backend) PutObject( return result, err } - hasher := md5.New() - w := io.MultiWriter(f, hasher) + w := io.MultiWriter(f) if _, err := io.Copy(w, input); err != nil { // remove file when i/o error occurred (FsPutErr) _ = f.Close() diff --git a/cmd/serve/s3/logger.go b/cmd/serve/s3/logger.go index 3ef2e5f87..8c9b3067b 100644 --- a/cmd/serve/s3/logger.go +++ b/cmd/serve/s3/logger.go @@ -12,15 +12,14 @@ type logger struct{} // print log message func (l logger) Print(level gofakes3.LogLevel, v ...interface{}) { - // fs.Infof(nil, fmt.Sprintln(v...)) switch level { - case gofakes3.LogErr: - fs.Errorf(nil, fmt.Sprintln(v...)) - case gofakes3.LogWarn: - fs.Infof(nil, fmt.Sprintln(v...)) - case gofakes3.LogInfo: - fs.Debugf(nil, fmt.Sprintln(v...)) default: - panic("unknown level") + fallthrough + case gofakes3.LogErr: + fs.Errorf("serve s3", fmt.Sprintln(v...)) + case gofakes3.LogWarn: + fs.Infof("serve s3", fmt.Sprintln(v...)) + case gofakes3.LogInfo: + fs.Debugf("serve s3", fmt.Sprintln(v...)) } } diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go index 305b46dd7..b8546508d 100644 --- a/cmd/serve/s3/s3.go +++ b/cmd/serve/s3/s3.go @@ -61,7 +61,10 @@ var Command = &cobra.Command{ } router := s.Router() s.Bind(router) - s.Serve() + err = s.serve() + if err != nil { + return err + } s.Wait() return nil }) diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.go index dd5200299..92d727922 100644 --- a/cmd/serve/s3/s3_test.go +++ b/cmd/serve/s3/s3_test.go @@ -6,10 +6,8 @@ package s3 import ( "bytes" "context" - "encoding/hex" "fmt" "io" - "math/rand" "net/url" "os" "os/exec" @@ -29,6 +27,7 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fstest" httplib "github.com/rclone/rclone/lib/http" + "github.com/rclone/rclone/lib/random" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -39,8 +38,8 @@ const ( // Configure and serve the server func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) { - keyid = RandString(16) - keysec = RandString(16) + keyid = random.String(16) + keysec = random.String(16) serveropt := &Options{ HTTP: httplib.DefaultCfg(), pathBucketMode: true, @@ -60,17 +59,6 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) { return } -func RandString(n int) string { - src := rand.New(rand.NewSource(time.Now().UnixNano())) - b := make([]byte, (n+1)/2) - - if _, err := src.Read(b); err != nil { - panic(err) - } - - return hex.EncodeToString(b)[:n] -} - // TestS3 runs the s3 server then runs the unit tests for the // s3 remote against it. func TestS3(t *testing.T) { diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go index a273ea047..54e0be4e8 100644 --- a/cmd/serve/s3/server.go +++ b/cmd/serve/s3/server.go @@ -70,3 +70,9 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error func (w *Server) Bind(router chi.Router) { router.Handle("/*", w.handler) } + +func (w *Server) serve() error { + w.Serve() + fs.Logf(w.f, "Starting s3 server on %s", w.URLs()) + return nil +} diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go index ace79d8fe..547320b73 100644 --- a/cmd/serve/s3/utils.go +++ b/cmd/serve/s3/utils.go @@ -4,11 +4,14 @@ import ( "context" "encoding/hex" "fmt" + "io" + "os" "path" "strings" "github.com/Mikubill/gofakes3" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/vfs" ) @@ -47,9 +50,29 @@ func getFileHash(node interface{}) string { switch b := node.(type) { case vfs.Node: - o = b.DirEntry().(fs.Object) - case fs.DirEntry: - o = b.(fs.Object) + fsObj, ok := b.DirEntry().(fs.Object) + if !ok { + fs.Debugf("serve s3", "File uploading - reading hash from VFS cache") + in, err := b.Open(os.O_RDONLY) + if err != nil { + return "" + } + defer func() { + _ = in.Close() + }() + h, err := hash.NewMultiHasherTypes(hash.NewHashSet(Opt.hashType)) + if err != nil { + return "" + } + _, err = io.Copy(h, in) + if err != nil { + return "" + } + return h.Sums()[Opt.hashType] + } + o = fsObj + case fs.Object: + o = b } hash, err := o.Hash(context.Background(), Opt.hashType) diff --git a/go.mod b/go.mod index 8f8a06c02..d3dd28c46 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-runewidth v0.0.15 - github.com/minio/minio-go/v7 v7.0.57 + github.com/minio/minio-go/v7 v7.0.63 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/sys/mountinfo v0.6.2 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 @@ -131,6 +131,7 @@ require ( github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect @@ -140,6 +141,10 @@ require ( github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index e9328f329..cc1e76895 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIf github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e h1:gtBhC9D1R/uuoov9wO8IDx3E25Tqn8nW7xRTvgPDP2E= -github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= +github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q= +github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= @@ -355,6 +355,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -396,8 +397,8 @@ github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= -github.com/minio/minio-go/v7 v7.0.57 h1:xsFiOiWjpC1XAGbFEUOzj1/gMXGz7ljfxifwcb/5YXU= -github.com/minio/minio-go/v7 v7.0.57/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= @@ -469,10 +470,14 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E= github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -823,7 +828,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=