Merge branch 'rclone:master' into user-from-header

backend/sftp/sftp.go

@@ -99,6 +99,11 @@ Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
 in the new OpenSSH format can't be used.`,
 			IsPassword: true,
 			Sensitive:  true,
+		}, {
+			Name: "pubkey",
+			Help: `SSH public certificate for public certificate based authentication.
+Set this if you have a signed certificate you want to use for authentication.
+If specified will override pubkey_file.`,
 		}, {
 			Name: "pubkey_file",
 			Help: `Optional path to public key file.
@@ -511,6 +516,7 @@ type Options struct {
 	KeyPem                  string          `config:"key_pem"`
 	KeyFile                 string          `config:"key_file"`
 	KeyFilePass             string          `config:"key_file_pass"`
+	PubKey                  string          `config:"pubkey"`
 	PubKeyFile              string          `config:"pubkey_file"`
 	KnownHostsFile          string          `config:"known_hosts_file"`
 	KeyUseAgent             bool            `config:"key_use_agent"`
@@ -997,13 +1003,21 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 
 		// If a public key has been specified then use that
-		if pubkeyFile != "" {
-			certfile, err := os.ReadFile(pubkeyFile)
-			if err != nil {
-				return nil, fmt.Errorf("unable to read cert file: %w", err)
+		if pubkeyFile != "" || opt.PubKey != "" {
+			pubKeyRaw := []byte(opt.PubKey)
+			// Use this error if public key is provided inline and is not a certificate
+			// if public key file is provided instead, use the err in the if block
+			notACertError := errors.New("public key provided is not a certificate: " + opt.PubKey)
+			if opt.PubKey == "" {
+				notACertError = errors.New("public key file is not a certificate file: " + pubkeyFile)
+				err := error(nil)
+				pubKeyRaw, err = os.ReadFile(pubkeyFile)
+				if err != nil {
+					return nil, fmt.Errorf("unable to read cert file: %w", err)
+				}
 			}
 
-			pk, _, _, _, err := ssh.ParseAuthorizedKey(certfile)
+			pk, _, _, _, err := ssh.ParseAuthorizedKey(pubKeyRaw)
 			if err != nil {
 				return nil, fmt.Errorf("unable to parse cert file: %w", err)
 			}
@@ -1017,7 +1031,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			// knows everything it needs.
 			cert, ok := pk.(*ssh.Certificate)
 			if !ok {
-				return nil, errors.New("public key file is not a certificate file: " + pubkeyFile)
+				return nil, notACertError
 			}
 			pubsigner, err := ssh.NewCertSigner(cert, signer)
 			if err != nil {

docs/content/sftp.md

@@ -156,7 +156,7 @@ and the public key built into it will be used during the authentication process.
 If you have a certificate you may use it to sign your public key, creating a
 separate SSH user certificate that should be used instead of the plain public key
 extracted from the private key. Then you must provide the path to the
-user certificate public key file in `pubkey_file`.
+user certificate public key file in `pubkey_file` or the content of the file in `pubkey`.
 
 Note: This is not the traditional public key paired with your private key,
 typically saved as `/home/$USER/.ssh/id_rsa.pub`. Setting this path in
@@ -494,6 +494,19 @@ Properties:
 - Type:        string
 - Required:    false
 
+#### --sftp-pubkey
+
+SSH public certificate for public certificate based authentication.
+Set this if you have a signed certificate you want to use for authentication.
+If specified will override pubkey_file.
+
+Properties:
+
+- Config:      pubkey
+- Env Var:     RCLONE_SFTP_PUBKEY
+- Type:        string
+- Required:    false
+
 #### --sftp-pubkey-file
 
 Optional path to public key file.

go.mod

@@ -59,7 +59,7 @@ require (
 	github.com/prometheus/client_golang v1.19.1
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
-	github.com/rclone/gofakes3 v0.0.3-0.20240807151802-e80146f8de87
+	github.com/rclone/gofakes3 v0.0.3
 	github.com/rfjakob/eme v1.1.2
 	github.com/rivo/uniseg v0.4.7
 	github.com/rogpeppe/go-internal v1.12.0

go.sum

@@ -519,8 +519,8 @@ github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1
 github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
-github.com/rclone/gofakes3 v0.0.3-0.20240807151802-e80146f8de87 h1:0YRo2aYhE+SCZsjWYMFe8zLD18xieXy7wQ8M9Ywcr/g=
-github.com/rclone/gofakes3 v0.0.3-0.20240807151802-e80146f8de87/go.mod h1:z7+o2VUwitO0WuVHReQlOW9jZ03LpeJ0PUFSULyTIds=
+github.com/rclone/gofakes3 v0.0.3 h1:0sKCxJ8TUUAG5KXGuc/fcDKGnzB/j6IjNQui9ntIZPo=
+github.com/rclone/gofakes3 v0.0.3/go.mod h1:z7+o2VUwitO0WuVHReQlOW9jZ03LpeJ0PUFSULyTIds=
 github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko=
 github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
 github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=

lib/http/auth.go

@@ -19,7 +19,11 @@ By default this will serve files without needing a login.
 You can either use an htpasswd file which can take lots of users, or
 set a single username and password with the ` + "`--{{ .Prefix }}user` and `--{{ .Prefix }}pass`" + ` flags.
 
-If no static users are configured by either of the above methods, and client
+Alternatively, you can have the reverse proxy manage authentication and use the
+username provided in the configured header with ` + "`--user-from-header`" + `  (e.g., ` + "`--{{ .Prefix }}--user-from-header=x-remote-user`" + `).
+Ensure the proxy is trusted and headers cannot be spoofed, as misconfiguration may lead to unauthorized access.
+
+If either of the above authentication methods is not configured and client
 certificates are required by the ` + "`--client-ca`" + ` flag passed to the server, the
 client certificate common name will be considered as the username.
 

lib/http/middleware_test.go

@@ -14,11 +14,13 @@ import (
 
 func TestMiddlewareAuth(t *testing.T) {
 	servers := []struct {
-		name string
-		http Config
-		auth AuthConfig
-		user string
-		pass string
+		name         string
+		expectedUser string
+		remoteUser   string
+		http         Config
+		auth         AuthConfig
+		user         string
+		pass         string
 	}{
 		{
 			name: "Basic",
@@ -85,9 +87,32 @@ func TestMiddlewareAuth(t *testing.T) {
 			},
 			user: "custom",
 			pass: "custom",
+		}, {
+			name:         "UserFromHeader",
+			remoteUser:   "remoteUser",
+			expectedUser: "remoteUser",
+			http: Config{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				UserFromHeader: "X-Remote-User",
+			},
+		}, {
+			name:         "UserFromHeader/MixedWithHtPasswd",
+			remoteUser:   "remoteUser",
+			expectedUser: "md5",
+			http: Config{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				UserFromHeader: "X-Remote-User",
+				Realm:          "test",
+				HtPasswd:       "./testdata/.htpasswd",
+			},
+			user: "md5",
+			pass: "md5",
 		},
 	}
-
 	for _, ss := range servers {
 		t.Run(ss.name, func(t *testing.T) {
 			s, err := NewServer(context.Background(), WithConfig(ss.http), WithAuth(ss.auth))
@@ -97,7 +122,12 @@ func TestMiddlewareAuth(t *testing.T) {
 			}()
 
 			expected := []byte("secret-page")
-			s.Router().Mount("/", testEchoHandler(expected))
+			if ss.expectedUser != "" {
+				s.Router().Mount("/", testAuthUserHandler())
+			} else {
+				s.Router().Mount("/", testEchoHandler(expected))
+			}
+
 			s.Serve()
 
 			url := testGetServerURL(t, s)
@@ -114,18 +144,24 @@ func TestMiddlewareAuth(t *testing.T) {
 				}()
 
 				require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using no creds should return unauthorized")
-
-				wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
-				require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
-				require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+				if ss.auth.UserFromHeader == "" {
+					wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
+					require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
+					require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+				}
 			})
-
 			t.Run("BadCreds", func(t *testing.T) {
 				client := &http.Client{}
 				req, err := http.NewRequest("GET", url, nil)
 				require.NoError(t, err)
 
-				req.SetBasicAuth(ss.user+"BAD", ss.pass+"BAD")
+				if ss.user != "" {
+					req.SetBasicAuth(ss.user+"BAD", ss.pass+"BAD")
+				}
+
+				if ss.auth.UserFromHeader != "" {
+					req.Header.Set(ss.auth.UserFromHeader, "/test:")
+				}
 
 				resp, err := client.Do(req)
 				require.NoError(t, err)
@@ -134,10 +170,11 @@ func TestMiddlewareAuth(t *testing.T) {
 				}()
 
 				require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using bad creds should return unauthorized")
-
-				wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
-				require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
-				require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+				if ss.auth.UserFromHeader == "" {
+					wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
+					require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
+					require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+				}
 			})
 
 			t.Run("GoodCreds", func(t *testing.T) {
@@ -145,7 +182,13 @@ func TestMiddlewareAuth(t *testing.T) {
 				req, err := http.NewRequest("GET", url, nil)
 				require.NoError(t, err)
 
-				req.SetBasicAuth(ss.user, ss.pass)
+				if ss.user != "" {
+					req.SetBasicAuth(ss.user, ss.pass)
+				}
+
+				if ss.auth.UserFromHeader != "" {
+					req.Header.Set(ss.auth.UserFromHeader, ss.remoteUser)
+				}
 
 				resp, err := client.Do(req)
 				require.NoError(t, err)
@@ -155,7 +198,11 @@ func TestMiddlewareAuth(t *testing.T) {
 
 				require.Equal(t, http.StatusOK, resp.StatusCode, "using good creds should return ok")
 
-				testExpectRespBody(t, resp, expected)
+				if ss.expectedUser != "" {
+					testExpectRespBody(t, resp, []byte(ss.expectedUser))
+				} else {
+					testExpectRespBody(t, resp, expected)
+				}
 			})
 		})
 	}

lib/http/server.go

@@ -391,16 +391,23 @@ func NewServer(ctx context.Context, options ...Option) (*Server, error) {
 
 func (s *Server) initAuth() {
 	s.usingAuth = false
+	altUsernameEnabled := s.auth.HtPasswd == "" && s.auth.BasicUser == ""
 
-	authCertificateUserEnabled := s.tlsConfig != nil && s.tlsConfig.ClientAuth != tls.NoClientCert && s.auth.HtPasswd == "" && s.auth.BasicUser == ""
-	if authCertificateUserEnabled {
+	if altUsernameEnabled {
 		s.usingAuth = true
-		s.mux.Use(MiddlewareAuthCertificateUser())
+		if s.auth.UserFromHeader != "" {
+			s.mux.Use(MiddlewareAuthGetUserFromHeader(s.auth.UserFromHeader))
+		} else if s.tlsConfig != nil && s.tlsConfig.ClientAuth != tls.NoClientCert {
+			s.mux.Use(MiddlewareAuthCertificateUser())
+		} else {
+			s.usingAuth = false
+			altUsernameEnabled = false
+		}
 	}
 
 	if s.auth.CustomAuthFn != nil {
 		s.usingAuth = true
-		s.mux.Use(MiddlewareAuthCustom(s.auth.CustomAuthFn, s.auth.Realm, authCertificateUserEnabled))
+		s.mux.Use(MiddlewareAuthCustom(s.auth.CustomAuthFn, s.auth.Realm, altUsernameEnabled))
 		return
 	}
 
@@ -415,12 +422,6 @@ func (s *Server) initAuth() {
 		s.mux.Use(MiddlewareAuthBasic(s.auth.BasicUser, s.auth.BasicPass, s.auth.Realm, s.auth.Salt))
 		return
 	}
-
-	if s.auth.UserFromHeader != "" {
-		s.usingAuth = true
-		s.mux.Use(MiddlewareAuthGetUserFromHeader(s.auth.UserFromHeader))
-		return
-	}
 }
 
 func (s *Server) initTemplate() error {