serve s3: implement --auth-proxy

This implements --auth-proxy for serve s3. In addition it:

* add listbuckets tests with and without authProxy
* use auth proxy test framework
* servetest: implement workaround for #7454
* update github.com/rclone/gofakes3 to fix race condition

cmd/serve/s3/server.go

@@ -3,12 +3,19 @@ package s3
 
 import (
 	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"math/rand"
 	"net/http"
+	"strings"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/rclone/gofakes3"
+	"github.com/rclone/gofakes3/signature"
+	"github.com/rclone/rclone/cmd/serve/proxy"
+	"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
 	httplib "github.com/rclone/rclone/lib/http"
@@ -16,6 +23,12 @@ import (
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
 
+type ctxKey int
+
+const (
+	ctxKeyID ctxKey = iota
+)
+
 // Options contains options for the http Server
 type Options struct {
 	//TODO add more options
@@ -24,17 +37,20 @@ type Options struct {
 	hashType       hash.Type
 	authPair       []string
 	noCleanup      bool
+	Auth           httplib.AuthConfig
 	HTTP           httplib.Config
 }
 
 // Server is a s3.FileSystem interface
 type Server struct {
-	*httplib.Server
-	f       fs.Fs
-	vfs     *vfs.VFS
-	faker   *gofakes3.GoFakeS3
-	handler http.Handler
-	ctx     context.Context // for global config
+	server   *httplib.Server
+	f        fs.Fs
+	_vfs     *vfs.VFS // don't use directly, use getVFS
+	faker    *gofakes3.GoFakeS3
+	handler  http.Handler
+	proxy    *proxy.Proxy
+	ctx      context.Context // for global config
+	s3Secret string
 }
 
 // Make a new S3 Server to serve the remote
@@ -42,16 +58,17 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
 	w := &Server{
 		f:   f,
 		ctx: ctx,
-		vfs: vfs.New(f, &vfscommon.Opt),
 	}
 
 	if len(opt.authPair) == 0 {
 		fs.Logf("serve s3", "No auth provided so allowing anonymous access")
+	} else {
+		w.s3Secret = getAuthSecret(opt.authPair)
 	}
 
 	var newLogger logger
 	w.faker = gofakes3.New(
-		newBackend(w.vfs, opt),
+		newBackend(w, opt),
 		gofakes3.WithHostBucket(!opt.pathBucketMode),
 		gofakes3.WithLogger(newLogger),
 		gofakes3.WithRequestID(rand.Uint64()),
@@ -60,24 +77,124 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
 		gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied
 	)
 
-	w.Server, err = httplib.NewServer(ctx,
+	w.handler = http.NewServeMux()
+	w.handler = w.faker.Server()
+
+	if proxyflags.Opt.AuthProxy != "" {
+		w.proxy = proxy.New(ctx, &proxyflags.Opt)
+		// proxy auth middleware
+		w.handler = proxyAuthMiddleware(w.handler, w)
+		w.handler = authPairMiddleware(w.handler, w)
+	} else {
+		w._vfs = vfs.New(f, &vfscommon.Opt)
+
+		if len(opt.authPair) > 0 {
+			w.faker.AddAuthKeys(authlistResolver(opt.authPair))
+		}
+	}
+
+	w.server, err = httplib.NewServer(ctx,
 		httplib.WithConfig(opt.HTTP),
+		httplib.WithAuth(opt.Auth),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init server: %w", err)
 	}
 
-	w.handler = w.faker.Server()
 	return w, nil
 }
 
+func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
+	if w._vfs != nil {
+		return w._vfs, nil
+	}
+
+	value := ctx.Value(ctxKeyID)
+	if value == nil {
+		return nil, errors.New("no VFS found in context")
+	}
+
+	VFS, ok := value.(*vfs.VFS)
+	if !ok {
+		return nil, fmt.Errorf("context value is not VFS: %#v", value)
+	}
+	return VFS, nil
+}
+
+// auth does proxy authorization
+func (w *Server) auth(accessKeyID string) (value interface{}, err error) {
+	VFS, _, err := w.proxy.Call(stringToMd5Hash(accessKeyID), accessKeyID, false)
+	if err != nil {
+		return nil, err
+	}
+	return VFS, err
+}
+
 // Bind register the handler to http.Router
 func (w *Server) Bind(router chi.Router) {
 	router.Handle("/*", w.handler)
 }
 
-func (w *Server) serve() error {
-	w.Serve()
-	fs.Logf(w.f, "Starting s3 server on %s", w.URLs())
+// Serve serves the s3 server
+func (w *Server) Serve() error {
+	w.server.Serve()
+	fs.Logf(w.f, "Starting s3 server on %s", w.server.URLs())
 	return nil
 }
+
+func authPairMiddleware(next http.Handler, ws *Server) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		accessKey, _ := parseAccessKeyID(r)
+		// set the auth pair
+		authPair := map[string]string{
+			accessKey: ws.s3Secret,
+		}
+		ws.faker.AddAuthKeys(authPair)
+		next.ServeHTTP(w, r)
+	})
+}
+
+func proxyAuthMiddleware(next http.Handler, ws *Server) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		accessKey, _ := parseAccessKeyID(r)
+		value, err := ws.auth(accessKey)
+		if err != nil {
+			fs.Infof(r.URL.Path, "%s: Auth failed: %v", r.RemoteAddr, err)
+		}
+		if value != nil {
+			r = r.WithContext(context.WithValue(r.Context(), ctxKeyID, value))
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+func parseAccessKeyID(r *http.Request) (accessKey string, error signature.ErrorCode) {
+	v4Auth := r.Header.Get("Authorization")
+	req, err := signature.ParseSignV4(v4Auth)
+	if err != signature.ErrNone {
+		return "", err
+	}
+
+	return req.Credential.GetAccessKey(), signature.ErrNone
+}
+
+func stringToMd5Hash(s string) string {
+	hasher := md5.New()
+	hasher.Write([]byte(s))
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+func getAuthSecret(authPair []string) string {
+	if len(authPair) == 0 {
+		return ""
+	}
+
+	splited := strings.Split(authPair[0], ",")
+	if len(splited) != 2 {
+		return ""
+	}
+
+	secret := strings.TrimSpace(splited[1])
+	return secret
+}