serve http: support --auth-proxy

This commit is contained in:
Matthias Baur 2023-02-22 15:55:24 +01:00 committed by GitHub
parent ce8b1cd861
commit a0b5d77427
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 159 additions and 22 deletions

View File

@ -3,6 +3,7 @@ package http
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"log" "log"
@ -15,6 +16,8 @@ import (
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
"github.com/rclone/rclone/cmd" "github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/accounting"
libhttp "github.com/rclone/rclone/lib/http" libhttp "github.com/rclone/rclone/lib/http"
@ -47,6 +50,7 @@ func init() {
libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP) libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template) libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template)
vfsflags.AddFlags(flagSet) vfsflags.AddFlags(flagSet)
proxyflags.AddFlags(flagSet)
} }
// Command definition for cobra // Command definition for cobra
@ -64,18 +68,21 @@ The server will log errors. Use ` + "`-v`" + ` to see access logs.
` + "`--bwlimit`" + ` will be respected for file transfers. Use ` + "`--stats`" + ` to ` + "`--bwlimit`" + ` will be respected for file transfers. Use ` + "`--stats`" + ` to
control the stats printing. control the stats printing.
` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help, ` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help + proxy.Help,
Annotations: map[string]string{ Annotations: map[string]string{
"versionIntroduced": "v1.39", "versionIntroduced": "v1.39",
}, },
Run: func(command *cobra.Command, args []string) { Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args) var f fs.Fs
f := cmd.NewFsSrc(args) if proxyflags.Opt.AuthProxy == "" {
cmd.CheckArgs(1, 1, command, args)
f = cmd.NewFsSrc(args)
} else {
cmd.CheckArgs(0, 0, command, args)
}
cmd.Run(false, true, command, func() error { cmd.Run(false, true, command, func() error {
ctx := context.Background() s, err := run(context.Background(), f, Opt)
s, err := run(ctx, f, Opt)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -86,25 +93,60 @@ control the stats printing.
}, },
} }
// server contains everything to run the server // HTTP contains everything to run the server
type serveCmd struct { type HTTP struct {
f fs.Fs f fs.Fs
vfs *vfs.VFS _vfs *vfs.VFS // don't use directly, use getVFS
server *libhttp.Server server *libhttp.Server
opt Options
proxy *proxy.Proxy
ctx context.Context // for global config
} }
func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) { // Gets the VFS in use for this request
var err error func (s *HTTP) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
if s._vfs != nil {
return s._vfs, nil
}
value := libhttp.CtxGetAuth(ctx)
if value == nil {
return nil, errors.New("no VFS found in context")
}
VFS, ok := value.(*vfs.VFS)
if !ok {
return nil, fmt.Errorf("context value is not VFS: %#v", value)
}
return VFS, nil
}
s := &serveCmd{ // auth does proxy authorization
func (s *HTTP) auth(user, pass string) (value interface{}, err error) {
VFS, _, err := s.proxy.Call(user, pass, false)
if err != nil {
return nil, err
}
return VFS, err
}
func run(ctx context.Context, f fs.Fs, opt Options) (s *HTTP, err error) {
s = &HTTP{
f: f, f: f,
vfs: vfs.New(f, &vfsflags.Opt), ctx: ctx,
opt: opt,
}
if proxyflags.Opt.AuthProxy != "" {
s.proxy = proxy.New(ctx, &proxyflags.Opt)
// override auth
s.opt.Auth.CustomAuthFn = s.auth
} else {
s._vfs = vfs.New(f, &vfsflags.Opt)
} }
s.server, err = libhttp.NewServer(ctx, s.server, err = libhttp.NewServer(ctx,
libhttp.WithConfig(opt.HTTP), libhttp.WithConfig(s.opt.HTTP),
libhttp.WithAuth(opt.Auth), libhttp.WithAuth(s.opt.Auth),
libhttp.WithTemplate(opt.Template), libhttp.WithTemplate(s.opt.Template),
) )
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to init server: %w", err) return nil, fmt.Errorf("failed to init server: %w", err)
@ -124,7 +166,7 @@ func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) {
} }
// handler reads incoming requests and dispatches them // handler reads incoming requests and dispatches them
func (s *serveCmd) handler(w http.ResponseWriter, r *http.Request) { func (s *HTTP) handler(w http.ResponseWriter, r *http.Request) {
isDir := strings.HasSuffix(r.URL.Path, "/") isDir := strings.HasSuffix(r.URL.Path, "/")
remote := strings.Trim(r.URL.Path, "/") remote := strings.Trim(r.URL.Path, "/")
if isDir { if isDir {
@ -135,9 +177,15 @@ func (s *serveCmd) handler(w http.ResponseWriter, r *http.Request) {
} }
// serveDir serves a directory index at dirRemote // serveDir serves a directory index at dirRemote
func (s *serveCmd) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) { func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
VFS, err := s.getVFS(r.Context())
if err != nil {
http.Error(w, "Root directory not found", http.StatusNotFound)
fs.Errorf(nil, "Failed to serve directory: %v", err)
return
}
// List the directory // List the directory
node, err := s.vfs.Stat(dirRemote) node, err := VFS.Stat(dirRemote)
if err == vfs.ENOENT { if err == vfs.ENOENT {
http.Error(w, "Directory not found", http.StatusNotFound) http.Error(w, "Directory not found", http.StatusNotFound)
return return
@ -177,8 +225,15 @@ func (s *serveCmd) serveDir(w http.ResponseWriter, r *http.Request, dirRemote st
} }
// serveFile serves a file object at remote // serveFile serves a file object at remote
func (s *serveCmd) serveFile(w http.ResponseWriter, r *http.Request, remote string) { func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
node, err := s.vfs.Stat(remote) VFS, err := s.getVFS(r.Context())
if err != nil {
http.Error(w, "File not found", http.StatusNotFound)
fs.Errorf(nil, "Failed to serve file: %v", err)
return
}
node, err := VFS.Stat(remote)
if err == vfs.ENOENT { if err == vfs.ENOENT {
fs.Infof(remote, "%s: File not found", r.RemoteAddr) fs.Infof(remote, "%s: File not found", r.RemoteAddr)
http.Error(w, "File not found", http.StatusNotFound) http.Error(w, "File not found", http.StatusNotFound)

View File

@ -21,7 +21,7 @@ import (
var ( var (
updateGolden = flag.Bool("updategolden", false, "update golden files for regression test") updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
sc *serveCmd sc *HTTP
testURL string testURL string
) )

View File

@ -437,6 +437,87 @@ _WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
result is accurate. However, this is very inefficient and may cost lots of API result is accurate. However, this is very inefficient and may cost lots of API
calls resulting in extra charges. Use it as a last resort and only with caching. calls resulting in extra charges. Use it as a last resort and only with caching.
## Auth Proxy
If you supply the parameter `--auth-proxy /path/to/program` then
rclone will use that program to generate backends on the fly which
then are used to authenticate incoming requests. This uses a simple
JSON based protocol with input on STDIN and output on STDOUT.
**PLEASE NOTE:** `--auth-proxy` and `--authorized-keys` cannot be used
together, if `--auth-proxy` is set the authorized keys option will be
ignored.
There is an example program
[bin/test_proxy.py](https://github.com/rclone/rclone/blob/master/test_proxy.py)
in the rclone source code.
The program's job is to take a `user` and `pass` on the input and turn
those into the config for a backend on STDOUT in JSON format. This
config will have any default parameters for the backend added, but it
won't use configuration from environment variables or command line
options - it is the job of the proxy program to make a complete
config.
This config generated must have this extra parameter
- `_root` - root to use for the backend
And it may have this parameter
- `_obscure` - comma separated strings for parameters to obscure
If password authentication was used by the client, input to the proxy
process (on STDIN) would look similar to this:
```
{
"user": "me",
"pass": "mypassword"
}
```
If public-key authentication was used by the client, input to the
proxy process (on STDIN) would look similar to this:
```
{
"user": "me",
"public_key": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDuwESFdAe14hVS6omeyX7edc...JQdf"
}
```
And as an example return this on STDOUT
```
{
"type": "sftp",
"_root": "",
"_obscure": "pass",
"user": "me",
"pass": "mypassword",
"host": "sftp.example.com"
}
```
This would mean that an SFTP backend would be created on the fly for
the `user` and `pass`/`public_key` returned in the output to the host given. Note
that since `_obscure` is set to `pass`, rclone will obscure the `pass`
parameter before creating the backend (which is required for sftp
backends).
The program can manipulate the supplied `user` in any way, for example
to make proxy to many different sftp backends, you could make the
`user` be `user@example.com` and then set the `host` to `example.com`
in the output and the user to `user`. For security you'd probably want
to restrict the `host` to a limited list.
Note that an internal cache is keyed on `user` so only use that for
configuration, don't use `pass` or `public_key`. This also means that if a user's
password or public-key is changed the cache will need to expire (which takes 5 mins)
before it takes effect.
This can be used to build general purpose proxies to any kind of
backend that rclone supports.
``` ```
rclone serve http remote:path [flags] rclone serve http remote:path [flags]
@ -446,6 +527,7 @@ rclone serve http remote:path [flags]
``` ```
--addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080])
--auth-proxy string A program to use to create the backend from the auth
--baseurl string Prefix for URLs - leave blank for root --baseurl string Prefix for URLs - leave blank for root
--cert string TLS PEM key (concatenation of certificate and CA certificate) --cert string TLS PEM key (concatenation of certificate and CA certificate)
--client-ca string Client certificate authority to verify clients with --client-ca string Client certificate authority to verify clients with