From 86a66c6202da9007ef2ad8e255081b7962416ca3 Mon Sep 17 00:00:00 2001 From: Misha Bragin Date: Mon, 25 Jul 2022 19:55:38 +0200 Subject: [PATCH] Make Signal Service listen on a standard 443/80 port instead of 10000 (#396) Right now Signal Service runs the Let'sEncrypt manager on port 80 and a gRPC server on port 10000. There are two separate listeners. This PR combines these listeners into one with a cmux lib. The gRPC server runs on either 443 with TLS or 80 without TLS. Let's Encrypt manager always runs on port 80. --- encryption/letsencrypt.go | 8 +- go.mod | 3 +- go.sum | 3 + infrastructure_files/docker-compose.yml.tmpl | 2 +- management/cmd/management.go | 16 +- signal/client/grpc.go | 2 + signal/cmd/run.go | 149 +++++++++++++++---- 7 files changed, 148 insertions(+), 35 deletions(-) diff --git a/encryption/letsencrypt.go b/encryption/letsencrypt.go index 0540e190f..193eaed2f 100644 --- a/encryption/letsencrypt.go +++ b/encryption/letsencrypt.go @@ -8,17 +8,17 @@ import ( ) // CreateCertManager wraps common logic of generating Let's encrypt certificate. -func CreateCertManager(datadir string, letsencryptDomain string) *autocert.Manager { +func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Manager, error) { certDir := filepath.Join(datadir, "letsencrypt") if _, err := os.Stat(certDir); os.IsNotExist(err) { err = os.MkdirAll(certDir, os.ModeDir) if err != nil { - log.Fatalf("failed creating Let's encrypt certdir: %s: %v", certDir, err) + return nil, err } } - log.Infof("running with Let's encrypt with domain %s. Cert will be stored in %s", letsencryptDomain, certDir) + log.Infof("running with LetsEncrypt (%s). Cert will be stored in %s", letsencryptDomain, certDir) certManager := &autocert.Manager{ Prompt: autocert.AcceptTOS, @@ -26,5 +26,5 @@ func CreateCertManager(datadir string, letsencryptDomain string) *autocert.Manag HostPolicy: autocert.HostWhitelist(letsencryptDomain), } - return certManager + return certManager, nil } diff --git a/go.mod b/go.mod index 8cfd0e43e..e3e9cd111 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,9 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/rs/xid v1.3.0 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 + github.com/soheilhy/cmux v0.1.5 github.com/stretchr/testify v1.7.1 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 ) @@ -97,7 +99,6 @@ require ( golang.org/x/image v0.0.0-20200430140353-33d19683fad8 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/net v0.0.0-20220513224357-95641704303c // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2 // indirect golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect diff --git a/go.sum b/go.sum index 7cad67547..af870c2de 100644 --- a/go.sum +++ b/go.sum @@ -575,6 +575,8 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EE github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smartystreets/assertions v1.13.0 h1:Dx1kYM01xsSqKPno3aqLnrwac2LetPvN23diwyr69Qs= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -740,6 +742,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= diff --git a/infrastructure_files/docker-compose.yml.tmpl b/infrastructure_files/docker-compose.yml.tmpl index 38b08c4ae..797d23cd6 100644 --- a/infrastructure_files/docker-compose.yml.tmpl +++ b/infrastructure_files/docker-compose.yml.tmpl @@ -25,7 +25,7 @@ services: volumes: - $SIGNAL_VOLUMENAME:/var/lib/netbird ports: - - 10000:10000 + - 10000:80 # # port and command for Let's Encrypt validation # - 443:443 # command: ["--letsencrypt-domain", "$NETBIRD_DOMAIN", "--log-file", "console"] diff --git a/management/cmd/management.go b/management/cmd/management.go index 639ed86f6..45942db97 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -46,6 +46,17 @@ var ( Timeout: 2 * time.Second, } + // TLS enabled: + // - HTTP 80 for LetsEncrypt + // - if --port not specified gRPC and HTTP servers on 443 (with multiplexing) + // - if --port=X specified then run gRPC and HTTP servers on X (with multiplexing) + // - if --port=80 forbid this (throw error, otherwise we need to overcomplicate the logic with multiplexing) + // TLS disabled: + // - if --port not specified gRPC and HTTP servers on 443 on 80 (with multiplexing) + // - if --port=X specified then run gRPC and HTTP servers on 443 on X (with multiplexing) + // Always run gRPC on port 33073 regardless of TLS to be backward compatible + // Remove HTTP port 33071 from the configuration. + mgmtCmd = &cobra.Command{ Use: "management", Short: "start Netbird Management Server", @@ -97,7 +108,10 @@ var ( var httpServer *http.Server if config.HttpConfig.LetsEncryptDomain != "" { // automatically generate a new certificate with Let's Encrypt - certManager := encryption.CreateCertManager(config.Datadir, config.HttpConfig.LetsEncryptDomain) + certManager, err := encryption.CreateCertManager(config.Datadir, config.HttpConfig.LetsEncryptDomain) + if err != nil { + log.Fatalf("failed creating Let's Encrypt cert manager: %v", err) + } transportCredentials := credentials.NewTLS(certManager.TLSConfig()) opts = append(opts, grpc.Creds(transportCredentials)) diff --git a/signal/client/grpc.go b/signal/client/grpc.go index 4a634ebaa..f39624196 100644 --- a/signal/client/grpc.go +++ b/signal/client/grpc.go @@ -75,6 +75,8 @@ func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled boo return nil, err } + log.Debugf("connected to Signal Service: %v", conn.Target()) + return &GrpcClient{ realClient: proto.NewSignalExchangeClient(conn), ctx: ctx, diff --git a/signal/cmd/run.go b/signal/cmd/run.go index ad179ad47..e5f0bcd8a 100644 --- a/signal/cmd/run.go +++ b/signal/cmd/run.go @@ -4,6 +4,7 @@ import ( "errors" "flag" "fmt" + "golang.org/x/crypto/acme/autocert" "io" "io/fs" "io/ioutil" @@ -11,6 +12,7 @@ import ( "net/http" "os" "path" + "strings" "time" "github.com/netbirdio/netbird/encryption" @@ -29,6 +31,7 @@ var ( signalLetsencryptDomain string signalSSLDir string defaultSignalSSLDir string + tlsEnabled bool signalKaep = grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ MinTime: 5 * time.Second, @@ -44,9 +47,26 @@ var ( runCmd = &cobra.Command{ Use: "run", - Short: "start Netbird Signal Server daemon", - Run: func(cmd *cobra.Command, args []string) { + Short: "start NetBird Signal Server daemon", + PreRun: func(cmd *cobra.Command, args []string) { + // detect whether user specified a port + userPort := cmd.Flag("port").Changed + if signalLetsencryptDomain != "" { + tlsEnabled = true + } + + if !userPort { + // different defaults for signalPort + if tlsEnabled { + signalPort = 443 + } else { + signalPort = 80 + } + } + }, + RunE: func(cmd *cobra.Command, args []string) error { flag.Parse() + err := util.InitLog(logLevel, logFile) if err != nil { log.Fatalf("failed initializing log %v", err) @@ -62,47 +82,120 @@ var ( } var opts []grpc.ServerOption - if signalLetsencryptDomain != "" { - if _, err := os.Stat(signalSSLDir); os.IsNotExist(err) { - err = os.MkdirAll(signalSSLDir, os.ModeDir) - if err != nil { - log.Fatalf("failed creating datadir: %s: %v", signalSSLDir, err) - } + var certManager *autocert.Manager + if tlsEnabled { + // Let's encrypt enabled -> generate certificate automatically + certManager, err = encryption.CreateCertManager(signalSSLDir, signalLetsencryptDomain) + if err != nil { + return err } - certManager := encryption.CreateCertManager(signalSSLDir, signalLetsencryptDomain) transportCredentials := credentials.NewTLS(certManager.TLSConfig()) opts = append(opts, grpc.Creds(transportCredentials)) - - listener := certManager.Listener() - log.Infof("http server listening on %s", listener.Addr()) - go func() { - if err := http.Serve(listener, certManager.HTTPHandler(nil)); err != nil { - log.Errorf("failed to serve https server: %v", err) - } - }() } opts = append(opts, signalKaep, signalKasp) grpcServer := grpc.NewServer(opts...) - - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", signalPort)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - proto.RegisterSignalExchangeServer(grpcServer, server.NewServer()) - log.Printf("started server: localhost:%v", signalPort) - if err := grpcServer.Serve(lis); err != nil { - log.Fatalf("failed to serve: %v", err) + + var compatListener net.Listener + if signalPort != 10000 { + // The Signal gRPC server was running on port 10000 previously. Old agents that are already connected to Signal + // are using port 10000. For compatibility purposes we keep running a 2nd gRPC server on port 10000. + compatListener, err = serveGRPC(grpcServer, 10000) + if err != nil { + return err + } + log.Infof("running gRPC backward compatibility server: %s", compatListener.Addr().String()) } + var grpcListener net.Listener + var httpListener net.Listener + if tlsEnabled { + httpListener = certManager.Listener() + if signalPort == 443 { + // running gRPC and HTTP cert manager on the same port + serveHTTP(httpListener, certManager.HTTPHandler(grpcHandlerFunc(grpcServer))) + log.Infof("running HTTP server (LetsEncrypt challenge handler) and gRPC server on the same port: %s", httpListener.Addr().String()) + } else { + serveHTTP(httpListener, certManager.HTTPHandler(nil)) + log.Infof("running HTTP server (LetsEncrypt challenge handler): %s", httpListener.Addr().String()) + } + } + + if signalPort != 443 || !tlsEnabled { + grpcListener, err = serveGRPC(grpcServer, signalPort) + if err != nil { + return err + } + log.Infof("running gRPC server: %s", grpcListener.Addr().String()) + } + + log.Infof("started Signal Service") + SetupCloseHandler() + <-stopCh - log.Println("Receive signal to stop running the Signal server") + if grpcListener != nil { + _ = grpcListener.Close() + log.Infof("stopped gRPC server") + } + if httpListener != nil { + _ = httpListener.Close() + log.Infof("stopped HTTP server") + } + if compatListener != nil { + _ = compatListener.Close() + log.Infof("stopped gRPC backward compatibility server") + } + log.Infof("stopped Signal Service") + + return nil }, } ) +func grpcHandlerFunc(grpcServer *grpc.Server) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + grpcHeader := strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") || + strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc+proto") + if r.ProtoMajor == 2 && grpcHeader { + grpcServer.ServeHTTP(w, r) + } + }) +} + +func notifyStop(msg string) { + select { + case stopCh <- 1: + log.Error(msg) + default: + // stop has been already called, nothing to report + } +} + +func serveHTTP(httpListener net.Listener, handler http.Handler) { + go func() { + err := http.Serve(httpListener, handler) + if err != nil { + notifyStop(fmt.Sprintf("failed running HTTP server %v", err)) + } + }() +} + +func serveGRPC(grpcServer *grpc.Server, port int) (net.Listener, error) { + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return nil, err + } + go func() { + err := grpcServer.Serve(listener) + if err != nil { + notifyStop(fmt.Sprintf("failed running gRPC server on port %d: %v", port, err)) + } + }() + return listener, nil +} + func cpFile(src, dst string) error { var err error var srcfd *os.File @@ -191,7 +284,7 @@ func migrateToNetbird(oldPath, newPath string) bool { } func init() { - runCmd.PersistentFlags().IntVar(&signalPort, "port", 10000, "Server port to listen on (e.g. 10000)") + runCmd.PersistentFlags().IntVar(&signalPort, "port", 80, "Server port to listen on (defaults to 443 if TLS is enabled, 80 otherwise") runCmd.Flags().StringVar(&signalSSLDir, "ssl-dir", defaultSignalSSLDir, "server ssl directory location. *Required only for Let's Encrypt certificates.") runCmd.Flags().StringVar(&signalLetsencryptDomain, "letsencrypt-domain", "", "a domain to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") }