From 16baa24964e1b937381195c055b598744e285b51 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 24 Oct 2024 11:20:37 +0100 Subject: [PATCH 1/2] serve s3: fix excess locking which was making serve s3 single threaded The fix for this was in the upstream library to narrow the locking window. See: https://forum.rclone.org/t/can-rclone-serve-s3-handle-more-than-one-client/48329/ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 954905cd1..fd6c96bc8 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/prometheus/client_golang v1.19.1 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 github.com/quasilyte/go-ruleguard/dsl v0.3.22 - github.com/rclone/gofakes3 v0.0.3-0.20240807151802-e80146f8de87 + github.com/rclone/gofakes3 v0.0.3 github.com/rfjakob/eme v1.1.2 github.com/rivo/uniseg v0.4.7 github.com/rogpeppe/go-internal v1.12.0 diff --git a/go.sum b/go.sum index 81ff97f66..e1dff78eb 100644 --- a/go.sum +++ b/go.sum @@ -519,8 +519,8 @@ github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1 github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= -github.com/rclone/gofakes3 v0.0.3-0.20240807151802-e80146f8de87 h1:0YRo2aYhE+SCZsjWYMFe8zLD18xieXy7wQ8M9Ywcr/g= -github.com/rclone/gofakes3 v0.0.3-0.20240807151802-e80146f8de87/go.mod h1:z7+o2VUwitO0WuVHReQlOW9jZ03LpeJ0PUFSULyTIds= +github.com/rclone/gofakes3 v0.0.3 h1:0sKCxJ8TUUAG5KXGuc/fcDKGnzB/j6IjNQui9ntIZPo= +github.com/rclone/gofakes3 v0.0.3/go.mod h1:z7+o2VUwitO0WuVHReQlOW9jZ03LpeJ0PUFSULyTIds= github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= From 40159e7a1615b4c48ed66a8e4fd1346aaf1c7138 Mon Sep 17 00:00:00 2001 From: Dimitar Ivanov Date: Fri, 25 Oct 2024 12:40:57 +0300 Subject: [PATCH 2/2] sftp: allow inline ssh public certificate for sftp Currently rclone allows us to specify the path to a public ssh certificate file. That works great for cases where we can specify key path, like local envs. If users are using rclone with [volsync](https://github.com/backube/volsync/tree/main/docs/usage/rclone) there currently is a limitation that users can specify only the rclone config file. With this change users can pass the public certificate in the same fashion as they can with `key_file`. --- backend/sftp/sftp.go | 26 ++++++++++++++++++++------ docs/content/sftp.md | 15 ++++++++++++++- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index e47e47db2..c3c60aade 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -99,6 +99,11 @@ Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys in the new OpenSSH format can't be used.`, IsPassword: true, Sensitive: true, + }, { + Name: "pubkey", + Help: `SSH public certificate for public certificate based authentication. +Set this if you have a signed certificate you want to use for authentication. +If specified will override pubkey_file.`, }, { Name: "pubkey_file", Help: `Optional path to public key file. @@ -511,6 +516,7 @@ type Options struct { KeyPem string `config:"key_pem"` KeyFile string `config:"key_file"` KeyFilePass string `config:"key_file_pass"` + PubKey string `config:"pubkey"` PubKeyFile string `config:"pubkey_file"` KnownHostsFile string `config:"known_hosts_file"` KeyUseAgent bool `config:"key_use_agent"` @@ -997,13 +1003,21 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } // If a public key has been specified then use that - if pubkeyFile != "" { - certfile, err := os.ReadFile(pubkeyFile) - if err != nil { - return nil, fmt.Errorf("unable to read cert file: %w", err) + if pubkeyFile != "" || opt.PubKey != "" { + pubKeyRaw := []byte(opt.PubKey) + // Use this error if public key is provided inline and is not a certificate + // if public key file is provided instead, use the err in the if block + notACertError := errors.New("public key provided is not a certificate: " + opt.PubKey) + if opt.PubKey == "" { + notACertError = errors.New("public key file is not a certificate file: " + pubkeyFile) + err := error(nil) + pubKeyRaw, err = os.ReadFile(pubkeyFile) + if err != nil { + return nil, fmt.Errorf("unable to read cert file: %w", err) + } } - pk, _, _, _, err := ssh.ParseAuthorizedKey(certfile) + pk, _, _, _, err := ssh.ParseAuthorizedKey(pubKeyRaw) if err != nil { return nil, fmt.Errorf("unable to parse cert file: %w", err) } @@ -1017,7 +1031,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // knows everything it needs. cert, ok := pk.(*ssh.Certificate) if !ok { - return nil, errors.New("public key file is not a certificate file: " + pubkeyFile) + return nil, notACertError } pubsigner, err := ssh.NewCertSigner(cert, signer) if err != nil { diff --git a/docs/content/sftp.md b/docs/content/sftp.md index 2ba4a48a7..56f4d98dc 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -156,7 +156,7 @@ and the public key built into it will be used during the authentication process. If you have a certificate you may use it to sign your public key, creating a separate SSH user certificate that should be used instead of the plain public key extracted from the private key. Then you must provide the path to the -user certificate public key file in `pubkey_file`. +user certificate public key file in `pubkey_file` or the content of the file in `pubkey`. Note: This is not the traditional public key paired with your private key, typically saved as `/home/$USER/.ssh/id_rsa.pub`. Setting this path in @@ -494,6 +494,19 @@ Properties: - Type: string - Required: false +#### --sftp-pubkey + +SSH public certificate for public certificate based authentication. +Set this if you have a signed certificate you want to use for authentication. +If specified will override pubkey_file. + +Properties: + +- Config: pubkey +- Env Var: RCLONE_SFTP_PUBKEY +- Type: string +- Required: false + #### --sftp-pubkey-file Optional path to public key file.