From 861c01caf51b91130a0d02907f4e0ddf84c2edd1 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Wed, 24 Apr 2024 15:37:07 +0300 Subject: [PATCH] http: support listening on passed FDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of the listening addresses specified above, rclone will listen to all FDs passed by the service manager, if any (and ignore any arguments passed by `--{{ .Prefix }}addr`. This allows rclone to be a socket-activated service. It can be configured as described in https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html It's possible to test this interactively through `systemd-socket-activate`, firing of a request in a second terminal: ``` ❯ systemd-socket-activate -l 8088 -l 8089 --fdname=foo:bar -- ./rclone serve webdav :local:test/ Listening on [::]:8088 as 3. Listening on [::]:8089 as 4. Communication attempt on fd 3. Execing ./rclone (./rclone serve webdav :local:test/) 2024/04/24 18:14:42 NOTICE: Local file system at /home/flokli/dev/flokli/rclone/test: WebDav Server started on [sd-listen:bar-0/ sd-listen:foo-0/] ``` --- lib/http/server.go | 130 +++++++++++++++++++++++++++++++-------------- 1 file changed, 89 insertions(+), 41 deletions(-) diff --git a/lib/http/server.go b/lib/http/server.go index e1e1501d6..518f07f7f 100644 --- a/lib/http/server.go +++ b/lib/http/server.go @@ -17,6 +17,7 @@ import ( "sync" "time" + sdActivation "github.com/coreos/go-systemd/v22/activation" "github.com/go-chi/chi/v5" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" @@ -42,6 +43,7 @@ or just by using an absolute path name. Note that unix sockets bypass the authentication - this is expected to be done with file system permissions. ` + "`--{{ .Prefix }}addr`" + ` may be repeated to listen on multiple IPs/ports/sockets. +Socket activation, described further below, can also be used to accomplish the same. ` + "`--{{ .Prefix }}server-read-timeout` and `--{{ .Prefix }}server-write-timeout`" + ` can be used to control the timeouts on the server. Note that this is the total time @@ -74,6 +76,21 @@ certificate authority certificate. values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0"). +### Socket activation + +Instead of the listening addresses specified above, rclone will listen to all +FDs passed by the service manager, if any (and ignore any arguments passed by ` + + "--{{ .Prefix }}addr`" + `). + +This allows rclone to be a socket-activated service. +It can be configured with .socket and .service unit files as described in +https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html + +Socket activation can be tested ad-hoc with the ` + "`systemd-socket-activate`" + `command + + systemd-socket-activate -l 8000 -- rclone serve + +This will socket-activate rclone on the first connection to port 8000 over TCP. ` tmpl, err := template.New("server help").Parse(help) if err != nil { @@ -240,6 +257,32 @@ func WithTemplate(cfg TemplateConfig) Option { } } +// For a given listener, and optional tlsConfig, construct a instance. +// The url string ends up in the `url` field of the `instance`. +// This unconditionally wraps the listener with the provided TLS config if one +// is specified, so all decision logic on whether to use TLS needs to live at +// the callsite. +func newInstance(ctx context.Context, s *Server, listener net.Listener, tlsCfg *tls.Config, url string) *instance { + if tlsCfg != nil { + listener = tls.NewListener(listener, tlsCfg) + } + + return &instance{ + url: url, + listener: listener, + httpServer: &http.Server{ + Handler: s.mux, + ReadTimeout: s.cfg.ServerReadTimeout, + WriteTimeout: s.cfg.ServerWriteTimeout, + MaxHeaderBytes: s.cfg.MaxHeaderBytes, + ReadHeaderTimeout: 10 * time.Second, // time to send the headers + IdleTimeout: 60 * time.Second, // time to keep idle connections open + TLSConfig: tlsCfg, + BaseContext: NewBaseContext(ctx, url), + }, + } +} + // NewServer instantiates a new http server using provided listeners and options // This function is provided if the default http server does not meet a services requirements and should not generally be used // A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443. @@ -288,55 +331,60 @@ func NewServer(ctx context.Context, options ...Option) (*Server, error) { s.initAuth() + // (Only) listen on FDs provided by the service manager, if any. + sdListeners, err := sdActivation.ListenersWithNames() + if err != nil { + return nil, fmt.Errorf("unable to acquire listeners: %w", err) + } + + if len(sdListeners) != 0 { + for listenerName, listeners := range sdListeners { + for i, listener := range listeners { + url := fmt.Sprintf("sd-listen:%s-%d/%s", listenerName, i, s.cfg.BaseURL) + if s.tlsConfig != nil { + url = fmt.Sprintf("sd-listen+tls:%s-%d/%s", listenerName, i, s.cfg.BaseURL) + } + + instance := newInstance(ctx, s, listener, s.tlsConfig, url) + + s.instances = append(s.instances, *instance) + } + } + + return s, nil + } + + // Process all listeners specified in the CLI Args. for _, addr := range s.cfg.ListenAddr { - var url string - var network = "tcp" - var tlsCfg *tls.Config + var instance *instance if strings.HasPrefix(addr, "unix://") || filepath.IsAbs(addr) { - network = "unix" addr = strings.TrimPrefix(addr, "unix://") - url = addr - } else if strings.HasPrefix(addr, "tls://") || (len(s.cfg.ListenAddr) == 1 && s.tlsConfig != nil) { - tlsCfg = s.tlsConfig - addr = strings.TrimPrefix(addr, "tls://") - } - - var listener net.Listener - if tlsCfg == nil { - listener, err = net.Listen(network, addr) - } else { - listener, err = tls.Listen(network, addr, tlsCfg) - } - if err != nil { - return nil, err - } - - if network == "tcp" { - var secure string - if tlsCfg != nil { - secure = "s" + listener, err := net.Listen("unix", addr) + if err != nil { + return nil, err } - url = fmt.Sprintf("http%s://%s%s/", secure, listener.Addr().String(), s.cfg.BaseURL) + instance = newInstance(ctx, s, listener, s.tlsConfig, addr) + } else if strings.HasPrefix(addr, "tls://") || (len(s.cfg.ListenAddr) == 1 && s.tlsConfig != nil) { + addr = strings.TrimPrefix(addr, "tls://") + listener, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + instance = newInstance(ctx, s, listener, s.tlsConfig, fmt.Sprintf("https://%s%s/", listener.Addr().String(), s.cfg.BaseURL)) + } else { + // HTTP case + addr = strings.TrimPrefix(addr, "http://") + listener, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + instance = newInstance(ctx, s, listener, nil, fmt.Sprintf("http://%s%s/", listener.Addr().String(), s.cfg.BaseURL)) + } - ii := instance{ - url: url, - listener: listener, - httpServer: &http.Server{ - Handler: s.mux, - ReadTimeout: s.cfg.ServerReadTimeout, - WriteTimeout: s.cfg.ServerWriteTimeout, - MaxHeaderBytes: s.cfg.MaxHeaderBytes, - ReadHeaderTimeout: 10 * time.Second, // time to send the headers - IdleTimeout: 60 * time.Second, // time to keep idle connections open - TLSConfig: tlsCfg, - BaseContext: NewBaseContext(ctx, url), - }, - } - - s.instances = append(s.instances, ii) + s.instances = append(s.instances, *instance) } return s, nil