serve s3: fixes before merge

- add context to log and fallthrough to error log level
- test: use rclone random lib to generate random strings
- calculate hash from vfs cache if file is uploading
- add server started log with server url
- remove md5 hasher
This commit is contained in:
Saw-jan 2023-09-12 16:35:13 +05:45 committed by Nick Craig-Wood
parent 34ef5147aa
commit d3dcc61154
8 changed files with 62 additions and 36 deletions

View File

@ -3,7 +3,6 @@ package s3
import (
"context"
"crypto/md5"
"encoding/hex"
"io"
"log"
@ -306,8 +305,7 @@ func (db *s3Backend) PutObject(
return result, err
}
hasher := md5.New()
w := io.MultiWriter(f, hasher)
w := io.MultiWriter(f)
if _, err := io.Copy(w, input); err != nil {
// remove file when i/o error occurred (FsPutErr)
_ = f.Close()

View File

@ -12,15 +12,14 @@ type logger struct{}
// print log message
func (l logger) Print(level gofakes3.LogLevel, v ...interface{}) {
// fs.Infof(nil, fmt.Sprintln(v...))
switch level {
case gofakes3.LogErr:
fs.Errorf(nil, fmt.Sprintln(v...))
case gofakes3.LogWarn:
fs.Infof(nil, fmt.Sprintln(v...))
case gofakes3.LogInfo:
fs.Debugf(nil, fmt.Sprintln(v...))
default:
panic("unknown level")
fallthrough
case gofakes3.LogErr:
fs.Errorf("serve s3", fmt.Sprintln(v...))
case gofakes3.LogWarn:
fs.Infof("serve s3", fmt.Sprintln(v...))
case gofakes3.LogInfo:
fs.Debugf("serve s3", fmt.Sprintln(v...))
}
}

View File

@ -61,7 +61,10 @@ var Command = &cobra.Command{
}
router := s.Router()
s.Bind(router)
s.Serve()
err = s.serve()
if err != nil {
return err
}
s.Wait()
return nil
})

View File

@ -6,10 +6,8 @@ package s3
import (
"bytes"
"context"
"encoding/hex"
"fmt"
"io"
"math/rand"
"net/url"
"os"
"os/exec"
@ -29,6 +27,7 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fstest"
httplib "github.com/rclone/rclone/lib/http"
"github.com/rclone/rclone/lib/random"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -39,8 +38,8 @@ const (
// Configure and serve the server
func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
keyid = RandString(16)
keysec = RandString(16)
keyid = random.String(16)
keysec = random.String(16)
serveropt := &Options{
HTTP: httplib.DefaultCfg(),
pathBucketMode: true,
@ -60,17 +59,6 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
return
}
func RandString(n int) string {
src := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, (n+1)/2)
if _, err := src.Read(b); err != nil {
panic(err)
}
return hex.EncodeToString(b)[:n]
}
// TestS3 runs the s3 server then runs the unit tests for the
// s3 remote against it.
func TestS3(t *testing.T) {

View File

@ -70,3 +70,9 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
func (w *Server) Bind(router chi.Router) {
router.Handle("/*", w.handler)
}
func (w *Server) serve() error {
w.Serve()
fs.Logf(w.f, "Starting s3 server on %s", w.URLs())
return nil
}

View File

@ -4,11 +4,14 @@ import (
"context"
"encoding/hex"
"fmt"
"io"
"os"
"path"
"strings"
"github.com/Mikubill/gofakes3"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/vfs"
)
@ -47,9 +50,29 @@ func getFileHash(node interface{}) string {
switch b := node.(type) {
case vfs.Node:
o = b.DirEntry().(fs.Object)
case fs.DirEntry:
o = b.(fs.Object)
fsObj, ok := b.DirEntry().(fs.Object)
if !ok {
fs.Debugf("serve s3", "File uploading - reading hash from VFS cache")
in, err := b.Open(os.O_RDONLY)
if err != nil {
return ""
}
defer func() {
_ = in.Close()
}()
h, err := hash.NewMultiHasherTypes(hash.NewHashSet(Opt.hashType))
if err != nil {
return ""
}
_, err = io.Copy(h, in)
if err != nil {
return ""
}
return h.Sums()[Opt.hashType]
}
o = fsObj
case fs.Object:
o = b
}
hash, err := o.Hash(context.Background(), Opt.hashType)

7
go.mod
View File

@ -43,7 +43,7 @@ require (
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
github.com/mattn/go-colorable v0.1.13
github.com/mattn/go-runewidth v0.0.15
github.com/minio/minio-go/v7 v7.0.57
github.com/minio/minio-go/v7 v7.0.63
github.com/mitchellh/go-homedir v1.1.0
github.com/moby/sys/mountinfo v0.6.2
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1
@ -131,6 +131,7 @@ require (
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
@ -140,6 +141,10 @@ require (
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/errors v0.9.1 // indirect

14
go.sum
View File

@ -58,8 +58,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIf
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e h1:gtBhC9D1R/uuoov9wO8IDx3E25Tqn8nW7xRTvgPDP2E=
github.com/Mikubill/gofakes3 v0.0.3-0.20221030004050-725f2cf2bf5e/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q=
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug=
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo=
@ -355,6 +355,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@ -396,8 +397,8 @@ github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
github.com/minio/minio-go/v7 v7.0.57 h1:xsFiOiWjpC1XAGbFEUOzj1/gMXGz7ljfxifwcb/5YXU=
github.com/minio/minio-go/v7 v7.0.57/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
@ -469,10 +470,14 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI=
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0=
github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E=
github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
@ -823,7 +828,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=