diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go index bef41c022..e71f11275 100644 --- a/cmd/serve/s3/s3.go +++ b/cmd/serve/s3/s3.go @@ -10,9 +10,12 @@ import ( "github.com/rclone/rclone/cmd/serve/proxy" "github.com/rclone/rclone/cmd/serve/proxy/proxyflags" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/rc" httplib "github.com/rclone/rclone/lib/http" "github.com/rclone/rclone/vfs" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/rclone/rclone/vfs/vfsflags" "github.com/spf13/cobra" ) @@ -61,6 +64,28 @@ func init() { vfsflags.AddFlags(flagSet) proxyflags.AddFlags(flagSet) serve.Command.AddCommand(Command) + serve.AddRc("s3", func(ctx context.Context, f fs.Fs, in rc.Params) (serve.Handle, error) { + // Read VFS Opts + var vfsOpt = vfscommon.Opt // set default opts + err := configstruct.SetAny(in, &vfsOpt) + if err != nil { + return nil, err + } + // Read Proxy Opts + var proxyOpt = proxy.Opt // set default opts + err = configstruct.SetAny(in, &proxyOpt) + if err != nil { + return nil, err + } + // Read opts + var opt = Opt // set default opts + err = configstruct.SetAny(in, &opt) + if err != nil { + return nil, err + } + // Create server + return newServer(ctx, f, &opt, &vfsOpt, &proxyOpt) + }) } //go:embed serve_s3.md @@ -91,18 +116,11 @@ var Command = &cobra.Command{ } cmd.Run(false, false, command, func() error { - s, err := newServer(context.Background(), f, &Opt) + s, err := newServer(context.Background(), f, &Opt, &vfscommon.Opt, &proxy.Opt) if err != nil { return err } - router := s.server.Router() - s.Bind(router) - err = s.Serve() - if err != nil { - return err - } - s.server.Wait() - return nil + return s.Serve() }) return nil }, diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.go index 6bc73085f..15b19d36d 100644 --- a/cmd/serve/s3/s3_test.go +++ b/cmd/serve/s3/s3_test.go @@ -24,8 +24,10 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/lib/random" + "github.com/rclone/rclone/vfs/vfscommon" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -35,17 +37,16 @@ const ( ) // Configure and serve the server -func serveS3(f fs.Fs) (testURL string, keyid string, keysec string, w *Server) { +func serveS3(t *testing.T, f fs.Fs) (testURL string, keyid string, keysec string, w *Server) { keyid = random.String(16) keysec = random.String(16) opt := Opt // copy default options opt.AuthKey = []string{fmt.Sprintf("%s,%s", keyid, keysec)} opt.HTTP.ListenAddr = []string{endpoint} - w, _ = newServer(context.Background(), f, &opt) - router := w.server.Router() - - w.Bind(router) - _ = w.Serve() + w, _ = newServer(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt) + go func() { + require.NoError(t, w.Serve()) + }() testURL = w.server.URLs()[0] return @@ -55,7 +56,7 @@ func serveS3(f fs.Fs) (testURL string, keyid string, keysec string, w *Server) { // s3 remote against it. func TestS3(t *testing.T) { start := func(f fs.Fs) (configmap.Simple, func()) { - testURL, keyid, keysec, _ := serveS3(f) + testURL, keyid, keysec, _ := serveS3(t, f) // Config for the backend we'll use to connect to the server config := configmap.Simple{ "type": "s3", @@ -118,7 +119,7 @@ func TestEncodingWithMinioClient(t *testing.T) { _, err = f.Put(context.Background(), in, obji) assert.NoError(t, err) - endpoint, keyid, keysec, _ := serveS3(f) + endpoint, keyid, keysec, _ := serveS3(t, f) testURL, _ := url.Parse(endpoint) minioClient, err := minio.New(testURL.Host, &minio.Options{ Creds: credentials.NewStaticV4(keyid, keysec, ""), @@ -181,7 +182,7 @@ func testListBuckets(t *testing.T, cases []TestCase, useProxy bool) { for _, tt := range cases { t.Run(tt.description, func(t *testing.T) { - endpoint, keyid, keysec, s := serveS3(f) + endpoint, keyid, keysec, s := serveS3(t, f) defer func() { assert.NoError(t, s.server.Shutdown()) }() @@ -289,3 +290,10 @@ func TestListBucketsAuthProxy(t *testing.T) { testListBuckets(t, cases, true) } + +func TestRc(t *testing.T) { + servetest.TestRc(t, rc.Params{ + "type": "s3", + "vfs_cache_mode": "off", + }) +} diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go index 05ee994ac..6732eb5e0 100644 --- a/cmd/serve/s3/server.go +++ b/cmd/serve/s3/server.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "math/rand" + "net" "net/http" "strings" @@ -43,7 +44,7 @@ type Server struct { } // Make a new S3 Server to serve the remote -func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error) { +func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Options, proxyOpt *proxy.Options) (s *Server, err error) { w := &Server{ f: f, ctx: ctx, @@ -84,12 +85,12 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error w.handler = w.faker.Server() if proxy.Opt.AuthProxy != "" { - w.proxy = proxy.New(ctx, &proxy.Opt, &vfscommon.Opt) + w.proxy = proxy.New(ctx, proxyOpt, vfsOpt) // proxy auth middleware w.handler = proxyAuthMiddleware(w.handler, w) w.handler = authPairMiddleware(w.handler, w) } else { - w._vfs = vfs.New(f, &vfscommon.Opt) + w._vfs = vfs.New(f, vfsOpt) if len(opt.AuthKey) > 0 { w.faker.AddAuthKeys(authlistResolver(opt.AuthKey)) @@ -104,6 +105,9 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error return nil, fmt.Errorf("failed to init server: %w", err) } + router := w.server.Router() + w.Bind(router) + return w, nil } @@ -138,13 +142,24 @@ func (w *Server) Bind(router chi.Router) { router.Handle("/*", w.handler) } -// Serve serves the s3 server +// Serve serves the s3 server until the server is shutdown func (w *Server) Serve() error { w.server.Serve() fs.Logf(w.f, "Starting s3 server on %s", w.server.URLs()) + w.server.Wait() return nil } +// Addr returns the first address of the server +func (w *Server) Addr() net.Addr { + return w.server.Addr() +} + +// Shutdown the server +func (w *Server) Shutdown() error { + return w.server.Shutdown() +} + func authPairMiddleware(next http.Handler, ws *Server) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { accessKey, _ := parseAccessKeyID(r)