diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go index b0454b867..ef032a384 100644 --- a/cmd/serve/http/http.go +++ b/cmd/serve/http/http.go @@ -11,6 +11,7 @@ import ( "github.com/ncw/rclone/cmd" "github.com/ncw/rclone/cmd/serve/httplib" + "github.com/ncw/rclone/cmd/serve/httplib/httpflags" "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/accounting" "github.com/ncw/rclone/lib/rest" @@ -20,7 +21,7 @@ import ( ) func init() { - httplib.AddFlags(Command.Flags()) + httpflags.AddFlags(Command.Flags()) vfsflags.AddFlags(Command.Flags()) } @@ -44,7 +45,7 @@ control the stats printing. cmd.CheckArgs(1, 1, command, args) f := cmd.NewFsSrc(args) cmd.Run(false, true, command, func() error { - s := newServer(f) + s := newServer(f, &httpflags.Opt) s.serve() return nil }) @@ -58,12 +59,12 @@ type server struct { srv *httplib.Server } -func newServer(f fs.Fs) *server { +func newServer(f fs.Fs, opt *httplib.Options) *server { mux := http.NewServeMux() s := &server{ f: f, vfs: vfs.New(f, &vfsflags.Opt), - srv: httplib.NewServer(mux), + srv: httplib.NewServer(mux, opt), } mux.HandleFunc("/", s.handler) return s diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go index 05da842db..a66511c4a 100644 --- a/cmd/serve/http/http_test.go +++ b/cmd/serve/http/http_test.go @@ -13,6 +13,7 @@ import ( "time" _ "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/cmd/serve/httplib" "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/config" "github.com/ncw/rclone/fs/filter" @@ -28,8 +29,9 @@ const ( ) func startServer(t *testing.T, f fs.Fs) { - s := newServer(f) - s.srv.SetBindAddress(testBindAddress) + opt := httplib.DefaultOpt + opt.ListenAddr = testBindAddress + s := newServer(f, &opt) go s.serve() // try to connect to the test server diff --git a/cmd/serve/httplib/httpflags/httpflags.go b/cmd/serve/httplib/httpflags/httpflags.go new file mode 100644 index 000000000..c1f0d8990 --- /dev/null +++ b/cmd/serve/httplib/httpflags/httpflags.go @@ -0,0 +1,27 @@ +package httpflags + +import ( + "github.com/ncw/rclone/cmd/serve/httplib" + "github.com/ncw/rclone/fs/config/flags" + "github.com/spf13/pflag" +) + +// Options set by command line flags +var ( + Opt = httplib.DefaultOpt +) + +// AddFlags adds flags for the httplib +func AddFlags(flagSet *pflag.FlagSet) { + flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.") + flags.DurationVarP(flagSet, &Opt.ServerReadTimeout, "server-read-timeout", "", Opt.ServerReadTimeout, "Timeout for server reading data") + flags.DurationVarP(flagSet, &Opt.ServerWriteTimeout, "server-write-timeout", "", Opt.ServerWriteTimeout, "Timeout for server writing data") + flags.IntVarP(flagSet, &Opt.MaxHeaderBytes, "max-header-bytes", "", Opt.MaxHeaderBytes, "Maximum size of request header") + flags.StringVarP(flagSet, &Opt.SslCert, "cert", "", Opt.SslCert, "SSL PEM key (concatenation of certificate and CA certificate)") + flags.StringVarP(flagSet, &Opt.SslKey, "key", "", Opt.SslKey, "SSL PEM Private key") + flags.StringVarP(flagSet, &Opt.ClientCA, "client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with") + flags.StringVarP(flagSet, &Opt.HtPasswd, "htpasswd", "", Opt.HtPasswd, "htpasswd file - if not provided no authentication is done") + flags.StringVarP(flagSet, &Opt.Realm, "realm", "", Opt.Realm, "realm for authentication") + flags.StringVarP(flagSet, &Opt.BasicUser, "user", "", Opt.BasicUser, "User name for authentication.") + flags.StringVarP(flagSet, &Opt.BasicPass, "pass", "", Opt.BasicPass, "Password for authentication.") +} diff --git a/cmd/serve/httplib/httplib.go b/cmd/serve/httplib/httplib.go index d051ef78a..54010c315 100644 --- a/cmd/serve/httplib/httplib.go +++ b/cmd/serve/httplib/httplib.go @@ -2,32 +2,20 @@ package httplib import ( + "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" "log" "net/http" + "time" auth "github.com/abbot/go-http-auth" "github.com/ncw/rclone/fs" - "github.com/spf13/pflag" ) // Globals -var ( - bindAddress = "localhost:8080" - htPasswdFile = "" - realm = "rclone" - basicUser = "" - basicPass = "" -) - -// AddFlags adds the http server specific flags -func AddFlags(flagSet *pflag.FlagSet) { - flagSet.StringVarP(&bindAddress, "addr", "", bindAddress, "IPaddress:Port to bind server to.") - flagSet.StringVarP(&htPasswdFile, "htpasswd", "", htPasswdFile, "File to use for htpasswd authentication.") - flagSet.StringVarP(&realm, "realm", "", realm, "Realm name for authentication.") - flagSet.StringVarP(&basicUser, "user", "", basicUser, "User name for authentication.") - flagSet.StringVarP(&basicPass, "pass", "", basicPass, "Password for authentication.") -} +var () // Help contains text describing the http server to add to the command // help. @@ -38,6 +26,13 @@ Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. +--server-read-timeout and --server-write-timeout can be used to +control the timeouts on the server. Note that this is the total time +for a transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + #### Authentication By default this will serve files without needing a login. @@ -55,69 +50,142 @@ To create an htpasswd file: htpasswd -B htpasswd user htpasswd -B htpasswd anotherUser -Use --realm to set the authentication realm.` +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +#### SSL/TLS + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --ssl along with --cert and --key. +If you wish to do client side certificate validation then you will +need to supply --client-ca also. +` + +// Options contains options for the http Server +type Options struct { + ListenAddr string // Port to listen on + ServerReadTimeout time.Duration // Timeout for server reading data + ServerWriteTimeout time.Duration // Timeout for server writing data + MaxHeaderBytes int // Maximum size of request header + SslCert string // SSL PEM key (concatenation of certificate and CA certificate) + SslKey string // SSL PEM Private key + ClientCA string // Client certificate authority to verify clients with + HtPasswd string // htpasswd file - if not provided no authentication is done + Realm string // realm for authentication + BasicUser string // single username for basic auth if not using Htpasswd + BasicPass string // password for BasicUser +} + +// DefaultOpt is the default values used for Options +var DefaultOpt = Options{ + ListenAddr: "localhost:8080", + Realm: "rclone", + ServerReadTimeout: 1 * time.Hour, + ServerWriteTimeout: 1 * time.Hour, + MaxHeaderBytes: 4096, +} // Server contains info about the running http server type Server struct { - bindAddress string + Opt Options + handler http.Handler // original handler httpServer *http.Server - basicUser string basicPassHashed string + useSSL bool // if server is configured for SSL/TLS } // singleUserProvider provides the encrypted password for a single user func (s *Server) singleUserProvider(user, realm string) string { - if user == s.basicUser { + if user == s.Opt.BasicUser { return s.basicPassHashed } return "" } -// NewServer creates an http server -func NewServer(handler http.Handler) *Server { +// NewServer creates an http server. The opt can be nil in which case +// the default options will be used. +func NewServer(handler http.Handler, opt *Options) *Server { s := &Server{ - bindAddress: bindAddress, + handler: handler, + } + + // Make a copy of the options + if opt != nil { + s.Opt = *opt + } else { + s.Opt = DefaultOpt } // Use htpasswd if required on everything - if htPasswdFile != "" || basicUser != "" { + if s.Opt.HtPasswd != "" || s.Opt.BasicUser != "" { var secretProvider auth.SecretProvider - if htPasswdFile != "" { - fs.Infof(nil, "Using %q as htpasswd storage", htPasswdFile) - secretProvider = auth.HtpasswdFileProvider(htPasswdFile) + if s.Opt.HtPasswd != "" { + fs.Infof(nil, "Using %q as htpasswd storage", s.Opt.HtPasswd) + secretProvider = auth.HtpasswdFileProvider(s.Opt.HtPasswd) } else { - fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", basicUser) - s.basicUser = basicUser - s.basicPassHashed = string(auth.MD5Crypt([]byte(basicPass), []byte("dlPL2MqE"), []byte("$1$"))) + fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", s.Opt.BasicUser) + s.basicPassHashed = string(auth.MD5Crypt([]byte(s.Opt.BasicPass), []byte("dlPL2MqE"), []byte("$1$"))) secretProvider = s.singleUserProvider } - authenticator := auth.NewBasicAuthenticator(realm, secretProvider) + authenticator := auth.NewBasicAuthenticator(s.Opt.Realm, secretProvider) handler = auth.JustCheck(authenticator, handler.ServeHTTP) } + s.useSSL = s.Opt.SslKey != "" + if (s.Opt.SslCert != "") != s.useSSL { + log.Fatalf("Need both -cert and -key to use SSL") + } + // FIXME make a transport? s.httpServer = &http.Server{ - Addr: s.bindAddress, + Addr: s.Opt.ListenAddr, Handler: handler, - MaxHeaderBytes: 1 << 20, + ReadTimeout: s.Opt.ServerReadTimeout, + WriteTimeout: s.Opt.ServerWriteTimeout, + MaxHeaderBytes: s.Opt.MaxHeaderBytes, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS10, // disable SSL v3.0 and earlier + }, } // go version specific initialisation initServer(s.httpServer) - return s -} -// SetBindAddress overrides the config flag -func (s *Server) SetBindAddress(addr string) { - s.bindAddress = addr - s.httpServer.Addr = addr + if s.Opt.ClientCA != "" { + if !s.useSSL { + log.Fatalf("Can't use --client-ca without --cert and --key") + } + certpool := x509.NewCertPool() + pem, err := ioutil.ReadFile(s.Opt.ClientCA) + if err != nil { + log.Fatalf("Failed to read client certificate authority: %v", err) + } + if !certpool.AppendCertsFromPEM(pem) { + log.Fatalf("Can't parse client certificate authority") + } + s.httpServer.TLSConfig.ClientCAs = certpool + s.httpServer.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert + } + + return s } // Serve runs the server - doesn't return func (s *Server) Serve() { - log.Fatal(s.httpServer.ListenAndServe()) + var err error + if s.useSSL { + err = s.httpServer.ListenAndServeTLS(s.Opt.SslCert, s.Opt.SslKey) + } else { + err = s.httpServer.ListenAndServe() + } + log.Fatalf("Fatal error while serving HTTP: %v", err) } // URL returns the serving address of this server func (s *Server) URL() string { - return fmt.Sprintf("http://%s/", s.bindAddress) + proto := "http" + if s.useSSL { + proto = "https" + } + return fmt.Sprintf("%s://%s/", proto, s.Opt.ListenAddr) }