fix(web): Allow configuration of read-buffer-size (#675)

This fixes the `431 Request Header Fields Too Large` error

By default, the read-buffer-size is 8192, up from fiber's default of 4096.

Fixes #674

Fixes #636

Supersedes #637

Supersedes #663

README.md

@@ -245,6 +245,7 @@ If you want to test it locally, see [Docker](#docker).
 | `web`                                           | Web configuration.                                                                                                                          | `{}`                       |
 | `web.address`                                   | Address to listen on.                                                                                                                       | `0.0.0.0`                  |
 | `web.port`                                      | Port to listen on.                                                                                                                          | `8080`                     |
+| `web.read-buffer-size`                          | Buffer size for reading requests from a connection. Also limit for the maximum header size.                                                 | `8192`                     |
 | `web.tls.certificate-file`                      | Optional public certificate file for TLS in PEM format.                                                                                     | ``                         |
 | `web.tls.private-key-file`                      | Optional private key file for TLS in PEM format.                                                                                            | ``                         |
 | `ui`                                            | UI configuration.                                                                                                                           | `{}`                       |
@@ -1909,6 +1910,17 @@ endpoints:
 </details>
 
 
+### How to fix 431 Request Header Fields Too Large error
+Depending on where your environment is deployed and what kind of middleware or reverse proxy sits in front of Gatus,
+you may run into this issue. This could be because the request headers are too large, e.g. big cookies.
+
+By default, `web.read-buffer-size` is set to `8192`, but increasing this value like so will increase the read buffer size:
+```yaml
+web:
+  read-buffer-size: 32768
+```
+
+
 ### Badges
 #### Uptime
 ![Uptime 1h](https://status.twin.sh/api/v1/endpoints/core_blog-external/uptimes/1h/badge.svg)

api/api.go

@@ -7,6 +7,7 @@ import (
 	"os"
 
 	"github.com/TwiN/gatus/v5/config"
+	"github.com/TwiN/gatus/v5/config/web"
 	static "github.com/TwiN/gatus/v5/web"
 	"github.com/TwiN/health"
 	fiber "github.com/gofiber/fiber/v2"
@@ -26,6 +27,10 @@ type API struct {
 
 func New(cfg *config.Config) *API {
 	api := &API{}
+	if cfg.Web == nil {
+		log.Println("[api.New] nil web config passed as parameter. This should only happen in tests. Using default web configuration")
+		cfg.Web = web.GetDefaultConfig()
+	}
 	api.router = api.createRouter(cfg)
 	return api
 }
@@ -40,6 +45,7 @@ func (a *API) createRouter(cfg *config.Config) *fiber.App {
 			log.Printf("[api.ErrorHandler] %s", err.Error())
 			return fiber.DefaultErrorHandler(c, err)
 		},
+		ReadBufferSize: cfg.Web.ReadBufferSize,
 		Network:        fiber.NetworkTCP,
 	})
 	if os.Getenv("ENVIRONMENT") == "dev" {

config/web/web.go

@@ -13,10 +13,16 @@ const (
 
 	// DefaultPort is the default port the application will listen on
 	DefaultPort = 8080
+
+	// DefaultReadBufferSize is the default value for ReadBufferSize
+	DefaultReadBufferSize = 8192
+
+	// MinimumReadBufferSize is the minimum value for ReadBufferSize, and also the default value set
+	// for fiber.Config.ReadBufferSize
+	MinimumReadBufferSize = 4096
 )
 
-// Config is the structure which supports the configuration of the endpoint
+// Config is the structure which supports the configuration of the server listening to requests
-// which provides access to the web frontend
 type Config struct {
 	// Address to listen on (defaults to 0.0.0.0 specified by DefaultAddress)
 	Address string `yaml:"address"`
@@ -24,6 +30,14 @@ type Config struct {
 	// Port to listen on (default to 8080 specified by DefaultPort)
 	Port int `yaml:"port"`
 
+	// ReadBufferSize sets fiber.Config.ReadBufferSize, which is the buffer size for reading requests coming from a
+	// single connection and also acts as a limit for the maximum header size.
+	//
+	// If you're getting occasional "Request Header Fields Too Large", you may want to try increasing this value.
+	//
+	// Defaults to DefaultReadBufferSize
+	ReadBufferSize int `yaml:"read-buffer-size,omitempty"`
+
 	// TLS configuration (optional)
 	TLS *TLSConfig `yaml:"tls,omitempty"`
 }
@@ -38,7 +52,11 @@ type TLSConfig struct {
 
 // GetDefaultConfig returns a Config struct with the default values
 func GetDefaultConfig() *Config {
-	return &Config{Address: DefaultAddress, Port: DefaultPort}
+	return &Config{
+		Address:        DefaultAddress,
+		Port:           DefaultPort,
+		ReadBufferSize: DefaultReadBufferSize,
+	}
 }
 
 // ValidateAndSetDefaults validates the web configuration and sets the default values if necessary.
@@ -53,6 +71,12 @@ func (web *Config) ValidateAndSetDefaults() error {
 	} else if web.Port < 0 || web.Port > math.MaxUint16 {
 		return fmt.Errorf("invalid port: value should be between %d and %d", 0, math.MaxUint16)
 	}
+	// Validate ReadBufferSize
+	if web.ReadBufferSize == 0 {
+		web.ReadBufferSize = DefaultReadBufferSize // Not set? Use the default value.
+	} else if web.ReadBufferSize < MinimumReadBufferSize {
+		web.ReadBufferSize = MinimumReadBufferSize // Below the minimum? Use the minimum value.
+	}
 	// Try to load the TLS certificates
 	if web.TLS != nil {
 		if err := web.TLS.isValid(); err != nil {

config/web/web_test.go

@@ -12,6 +12,9 @@ func TestGetDefaultConfig(t *testing.T) {
 	if defaultConfig.Address != DefaultAddress {
 		t.Error("expected default config to have the default address")
 	}
+	if defaultConfig.ReadBufferSize != DefaultReadBufferSize {
+		t.Error("expected default config to have the default read buffer size")
+	}
 	if defaultConfig.TLS != nil {
 		t.Error("expected default config to have TLS disabled")
 	}
@@ -23,6 +26,7 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
 		cfg                    *Config
 		expectedAddress        string
 		expectedPort           int
+		expectedReadBufferSize int
 		expectedErr            bool
 	}{
 		{
@@ -30,6 +34,7 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
 			cfg:                    &Config{},
 			expectedAddress:        "0.0.0.0",
 			expectedPort:           8080,
+			expectedReadBufferSize: 8192,
 			expectedErr:            false,
 		},
 		{
@@ -37,11 +42,36 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
 			cfg:         &Config{Port: 100000000},
 			expectedErr: true,
 		},
+		{
+			name:                   "read-buffer-size-below-minimum",
+			cfg:                    &Config{ReadBufferSize: 1024},
+			expectedAddress:        "0.0.0.0",
+			expectedPort:           8080,
+			expectedReadBufferSize: MinimumReadBufferSize, // minimum is 4096, default is 8192.
+			expectedErr:            false,
+		},
+		{
+			name:                   "read-buffer-size-at-minimum",
+			cfg:                    &Config{ReadBufferSize: MinimumReadBufferSize},
+			expectedAddress:        "0.0.0.0",
+			expectedPort:           8080,
+			expectedReadBufferSize: 4096,
+			expectedErr:            false,
+		},
+		{
+			name:                   "custom-read-buffer-size",
+			cfg:                    &Config{ReadBufferSize: 65536},
+			expectedAddress:        "0.0.0.0",
+			expectedPort:           8080,
+			expectedReadBufferSize: 65536,
+			expectedErr:            false,
+		},
 		{
 			name:                   "with-good-tls-config",
 			cfg:                    &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/cert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
 			expectedAddress:        "0.0.0.0",
 			expectedPort:           443,
+			expectedReadBufferSize: 8192,
 			expectedErr:            false,
 		},
 		{
@@ -49,6 +79,7 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
 			cfg:                    &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/badcert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
 			expectedAddress:        "0.0.0.0",
 			expectedPort:           443,
+			expectedReadBufferSize: 8192,
 			expectedErr:            true,
 		},
 		{
@@ -56,6 +87,7 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
 			cfg:                    &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "", PrivateKeyFile: "../../testdata/cert.key"}},
 			expectedAddress:        "0.0.0.0",
 			expectedPort:           443,
+			expectedReadBufferSize: 8192,
 			expectedErr:            true,
 		},
 	}
@@ -68,10 +100,13 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
 			}
 			if !scenario.expectedErr {
 				if scenario.cfg.Port != scenario.expectedPort {
-					t.Errorf("expected port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port)
+					t.Errorf("expected Port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port)
+				}
+				if scenario.cfg.ReadBufferSize != scenario.expectedReadBufferSize {
+					t.Errorf("expected ReadBufferSize to be %d, got %d", scenario.expectedReadBufferSize, scenario.cfg.ReadBufferSize)
 				}
 				if scenario.cfg.Address != scenario.expectedAddress {
-					t.Errorf("expected address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address)
+					t.Errorf("expected Address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address)
 				}
 			}
 		})