fix(web): Allow configuration of read-buffer-size (#675)

This fixes the `431 Request Header Fields Too Large` error

By default, the read-buffer-size is 8192, up from fiber's default of 4096.

Fixes #674

Fixes #636

Supersedes #637

Supersedes #663
This commit is contained in:
TwiN 2024-02-07 18:54:30 -05:00 committed by GitHub
parent 3d1b4e566d
commit 2a623a59d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 157 additions and 80 deletions

110
README.md
View File

@ -206,57 +206,58 @@ If you want to test it locally, see [Docker](#docker).
## Configuration
| Parameter | Description | Default |
|:------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------|
| `debug` | Whether to enable debug logs. | `false` |
| `metrics` | Whether to expose metrics at /metrics. | `false` |
| `storage` | [Storage configuration](#storage) | `{}` |
| `endpoints` | List of endpoints to monitor. | Required `[]` |
| `endpoints[].enabled` | Whether to monitor the endpoint. | `true` |
| `endpoints[].name` | Name of the endpoint. Can be anything. | Required `""` |
| `endpoints[].group` | Group name. Used to group multiple endpoints together on the dashboard. <br />See [Endpoint groups](#endpoint-groups). | `""` |
| `endpoints[].url` | URL to send the request to. | Required `""` |
| `endpoints[].method` | Request method. | `GET` |
| `endpoints[].conditions` | Conditions used to determine the health of the endpoint. <br />See [Conditions](#conditions). | `[]` |
| `endpoints[].interval` | Duration to wait between every status check. | `60s` |
| `endpoints[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`). | `false` |
| `endpoints[].body` | Request body. | `""` |
| `endpoints[].headers` | Request headers. | `{}` |
| `endpoints[].dns` | Configuration for an endpoint of type DNS. <br />See [Monitoring an endpoint using DNS queries](#monitoring-an-endpoint-using-dns-queries). | `""` |
| `endpoints[].dns.query-type` | Query type (e.g. MX) | `""` |
| `endpoints[].dns.query-name` | Query name (e.g. example.com) | `""` |
| `endpoints[].ssh` | Configuration for an endpoint of type SSH. <br />See [Monitoring an endpoint using SSH](#monitoring-an-endpoint-using-ssh). | `""` |
| `debug` | Whether to enable debug logs. | `false` |
| `metrics` | Whether to expose metrics at /metrics. | `false` |
| `storage` | [Storage configuration](#storage) | `{}` |
| `endpoints` | List of endpoints to monitor. | Required `[]` |
| `endpoints[].enabled` | Whether to monitor the endpoint. | `true` |
| `endpoints[].name` | Name of the endpoint. Can be anything. | Required `""` |
| `endpoints[].group` | Group name. Used to group multiple endpoints together on the dashboard. <br />See [Endpoint groups](#endpoint-groups). | `""` |
| `endpoints[].url` | URL to send the request to. | Required `""` |
| `endpoints[].method` | Request method. | `GET` |
| `endpoints[].conditions` | Conditions used to determine the health of the endpoint. <br />See [Conditions](#conditions). | `[]` |
| `endpoints[].interval` | Duration to wait between every status check. | `60s` |
| `endpoints[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`). | `false` |
| `endpoints[].body` | Request body. | `""` |
| `endpoints[].headers` | Request headers. | `{}` |
| `endpoints[].dns` | Configuration for an endpoint of type DNS. <br />See [Monitoring an endpoint using DNS queries](#monitoring-an-endpoint-using-dns-queries). | `""` |
| `endpoints[].dns.query-type` | Query type (e.g. MX) | `""` |
| `endpoints[].dns.query-name` | Query name (e.g. example.com) | `""` |
| `endpoints[].ssh` | Configuration for an endpoint of type SSH. <br />See [Monitoring an endpoint using SSH](#monitoring-an-endpoint-using-ssh). | `""` |
| `endpoints[].ssh.username` | SSH username (e.g. example) | Required `""` |
| `endpoints[].ssh.password` | SSH password (e.g. password) | Required `""` |
| `endpoints[].alerts[].type` | Type of alert. <br />See [Alerting](#alerting) for all valid types. | Required `""` |
| `endpoints[].alerts[].enabled` | Whether to enable the alert. | `true` |
| `endpoints[].alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` |
| `endpoints[].alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` |
| `endpoints[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` |
| `endpoints[].alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` |
| `endpoints[].client` | [Client configuration](#client-configuration). | `{}` |
| `endpoints[].ui` | UI configuration at the endpoint level. | `{}` |
| `endpoints[].ui.hide-hostname` | Whether to hide the hostname in the result. | `false` |
| `endpoints[].ui.hide-url` | Whether to ensure the URL is not displayed in the results. Useful if the URL contains a token. | `false` |
| `endpoints[].ui.dont-resolve-failed-conditions` | Whether to resolve failed conditions for the UI. | `false` |
| `endpoints[].ui.badge.reponse-time` | List of response time thresholds. Each time a threshold is reached, the badge has a different color. | `[50, 200, 300, 500, 750]` |
| `alerting` | [Alerting configuration](#alerting). | `{}` |
| `security` | [Security configuration](#security). | `{}` |
| `disable-monitoring-lock` | Whether to [disable the monitoring lock](#disable-monitoring-lock). | `false` |
| `skip-invalid-config-update` | Whether to ignore invalid configuration update. <br />See [Reloading configuration on the fly](#reloading-configuration-on-the-fly). | `false` |
| `web` | Web configuration. | `{}` |
| `web.address` | Address to listen on. | `0.0.0.0` |
| `web.port` | Port to listen on. | `8080` |
| `web.tls.certificate-file` | Optional public certificate file for TLS in PEM format. | `` |
| `web.tls.private-key-file` | Optional private key file for TLS in PEM format. | `` |
| `ui` | UI configuration. | `{}` |
| `ui.title` | [Title of the document](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/title). | `Health Dashboard ǀ Gatus` |
| `ui.description` | Meta description for the page. | `Gatus is an advanced...`. |
| `ui.header` | Header at the top of the dashboard. | `Health Status` |
| `ui.logo` | URL to the logo to display. | `""` |
| `ui.link` | Link to open when the logo is clicked. | `""` |
| `ui.buttons` | List of buttons to display below the header. | `[]` |
| `ui.buttons[].name` | Text to display on the button. | Required `""` |
| `ui.buttons[].link` | Link to open when the button is clicked. | Required `""` |
| `maintenance` | [Maintenance configuration](#maintenance). | `{}` |
| `endpoints[].alerts[].type` | Type of alert. <br />See [Alerting](#alerting) for all valid types. | Required `""` |
| `endpoints[].alerts[].enabled` | Whether to enable the alert. | `true` |
| `endpoints[].alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` |
| `endpoints[].alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` |
| `endpoints[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` |
| `endpoints[].alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` |
| `endpoints[].client` | [Client configuration](#client-configuration). | `{}` |
| `endpoints[].ui` | UI configuration at the endpoint level. | `{}` |
| `endpoints[].ui.hide-hostname` | Whether to hide the hostname in the result. | `false` |
| `endpoints[].ui.hide-url` | Whether to ensure the URL is not displayed in the results. Useful if the URL contains a token. | `false` |
| `endpoints[].ui.dont-resolve-failed-conditions` | Whether to resolve failed conditions for the UI. | `false` |
| `endpoints[].ui.badge.reponse-time` | List of response time thresholds. Each time a threshold is reached, the badge has a different color. | `[50, 200, 300, 500, 750]` |
| `alerting` | [Alerting configuration](#alerting). | `{}` |
| `security` | [Security configuration](#security). | `{}` |
| `disable-monitoring-lock` | Whether to [disable the monitoring lock](#disable-monitoring-lock). | `false` |
| `skip-invalid-config-update` | Whether to ignore invalid configuration update. <br />See [Reloading configuration on the fly](#reloading-configuration-on-the-fly). | `false` |
| `web` | Web configuration. | `{}` |
| `web.address` | Address to listen on. | `0.0.0.0` |
| `web.port` | Port to listen on. | `8080` |
| `web.read-buffer-size` | Buffer size for reading requests from a connection. Also limit for the maximum header size. | `8192` |
| `web.tls.certificate-file` | Optional public certificate file for TLS in PEM format. | `` |
| `web.tls.private-key-file` | Optional private key file for TLS in PEM format. | `` |
| `ui` | UI configuration. | `{}` |
| `ui.title` | [Title of the document](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/title). | `Health Dashboard ǀ Gatus` |
| `ui.description` | Meta description for the page. | `Gatus is an advanced...`. |
| `ui.header` | Header at the top of the dashboard. | `Health Status` |
| `ui.logo` | URL to the logo to display. | `""` |
| `ui.link` | Link to open when the logo is clicked. | `""` |
| `ui.buttons` | List of buttons to display below the header. | `[]` |
| `ui.buttons[].name` | Text to display on the button. | Required `""` |
| `ui.buttons[].link` | Link to open when the button is clicked. | Required `""` |
| `maintenance` | [Maintenance configuration](#maintenance). | `{}` |
### Conditions
@ -1909,6 +1910,17 @@ endpoints:
</details>
### How to fix 431 Request Header Fields Too Large error
Depending on where your environment is deployed and what kind of middleware or reverse proxy sits in front of Gatus,
you may run into this issue. This could be because the request headers are too large, e.g. big cookies.
By default, `web.read-buffer-size` is set to `8192`, but increasing this value like so will increase the read buffer size:
```yaml
web:
read-buffer-size: 32768
```
### Badges
#### Uptime
![Uptime 1h](https://status.twin.sh/api/v1/endpoints/core_blog-external/uptimes/1h/badge.svg)

View File

@ -7,6 +7,7 @@ import (
"os"
"github.com/TwiN/gatus/v5/config"
"github.com/TwiN/gatus/v5/config/web"
static "github.com/TwiN/gatus/v5/web"
"github.com/TwiN/health"
fiber "github.com/gofiber/fiber/v2"
@ -26,6 +27,10 @@ type API struct {
func New(cfg *config.Config) *API {
api := &API{}
if cfg.Web == nil {
log.Println("[api.New] nil web config passed as parameter. This should only happen in tests. Using default web configuration")
cfg.Web = web.GetDefaultConfig()
}
api.router = api.createRouter(cfg)
return api
}
@ -40,7 +45,8 @@ func (a *API) createRouter(cfg *config.Config) *fiber.App {
log.Printf("[api.ErrorHandler] %s", err.Error())
return fiber.DefaultErrorHandler(c, err)
},
Network: fiber.NetworkTCP,
ReadBufferSize: cfg.Web.ReadBufferSize,
Network: fiber.NetworkTCP,
})
if os.Getenv("ENVIRONMENT") == "dev" {
app.Use(cors.New(cors.Config{

View File

@ -13,10 +13,16 @@ const (
// DefaultPort is the default port the application will listen on
DefaultPort = 8080
// DefaultReadBufferSize is the default value for ReadBufferSize
DefaultReadBufferSize = 8192
// MinimumReadBufferSize is the minimum value for ReadBufferSize, and also the default value set
// for fiber.Config.ReadBufferSize
MinimumReadBufferSize = 4096
)
// Config is the structure which supports the configuration of the endpoint
// which provides access to the web frontend
// Config is the structure which supports the configuration of the server listening to requests
type Config struct {
// Address to listen on (defaults to 0.0.0.0 specified by DefaultAddress)
Address string `yaml:"address"`
@ -24,6 +30,14 @@ type Config struct {
// Port to listen on (default to 8080 specified by DefaultPort)
Port int `yaml:"port"`
// ReadBufferSize sets fiber.Config.ReadBufferSize, which is the buffer size for reading requests coming from a
// single connection and also acts as a limit for the maximum header size.
//
// If you're getting occasional "Request Header Fields Too Large", you may want to try increasing this value.
//
// Defaults to DefaultReadBufferSize
ReadBufferSize int `yaml:"read-buffer-size,omitempty"`
// TLS configuration (optional)
TLS *TLSConfig `yaml:"tls,omitempty"`
}
@ -38,7 +52,11 @@ type TLSConfig struct {
// GetDefaultConfig returns a Config struct with the default values
func GetDefaultConfig() *Config {
return &Config{Address: DefaultAddress, Port: DefaultPort}
return &Config{
Address: DefaultAddress,
Port: DefaultPort,
ReadBufferSize: DefaultReadBufferSize,
}
}
// ValidateAndSetDefaults validates the web configuration and sets the default values if necessary.
@ -53,6 +71,12 @@ func (web *Config) ValidateAndSetDefaults() error {
} else if web.Port < 0 || web.Port > math.MaxUint16 {
return fmt.Errorf("invalid port: value should be between %d and %d", 0, math.MaxUint16)
}
// Validate ReadBufferSize
if web.ReadBufferSize == 0 {
web.ReadBufferSize = DefaultReadBufferSize // Not set? Use the default value.
} else if web.ReadBufferSize < MinimumReadBufferSize {
web.ReadBufferSize = MinimumReadBufferSize // Below the minimum? Use the minimum value.
}
// Try to load the TLS certificates
if web.TLS != nil {
if err := web.TLS.isValid(); err != nil {

View File

@ -12,6 +12,9 @@ func TestGetDefaultConfig(t *testing.T) {
if defaultConfig.Address != DefaultAddress {
t.Error("expected default config to have the default address")
}
if defaultConfig.ReadBufferSize != DefaultReadBufferSize {
t.Error("expected default config to have the default read buffer size")
}
if defaultConfig.TLS != nil {
t.Error("expected default config to have TLS disabled")
}
@ -19,18 +22,20 @@ func TestGetDefaultConfig(t *testing.T) {
func TestConfig_ValidateAndSetDefaults(t *testing.T) {
scenarios := []struct {
name string
cfg *Config
expectedAddress string
expectedPort int
expectedErr bool
name string
cfg *Config
expectedAddress string
expectedPort int
expectedReadBufferSize int
expectedErr bool
}{
{
name: "no-explicit-config",
cfg: &Config{},
expectedAddress: "0.0.0.0",
expectedPort: 8080,
expectedErr: false,
name: "no-explicit-config",
cfg: &Config{},
expectedAddress: "0.0.0.0",
expectedPort: 8080,
expectedReadBufferSize: 8192,
expectedErr: false,
},
{
name: "invalid-port",
@ -38,25 +43,52 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
expectedErr: true,
},
{
name: "with-good-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/cert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedErr: false,
name: "read-buffer-size-below-minimum",
cfg: &Config{ReadBufferSize: 1024},
expectedAddress: "0.0.0.0",
expectedPort: 8080,
expectedReadBufferSize: MinimumReadBufferSize, // minimum is 4096, default is 8192.
expectedErr: false,
},
{
name: "with-bad-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/badcert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedErr: true,
name: "read-buffer-size-at-minimum",
cfg: &Config{ReadBufferSize: MinimumReadBufferSize},
expectedAddress: "0.0.0.0",
expectedPort: 8080,
expectedReadBufferSize: 4096,
expectedErr: false,
},
{
name: "with-partial-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedErr: true,
name: "custom-read-buffer-size",
cfg: &Config{ReadBufferSize: 65536},
expectedAddress: "0.0.0.0",
expectedPort: 8080,
expectedReadBufferSize: 65536,
expectedErr: false,
},
{
name: "with-good-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/cert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedReadBufferSize: 8192,
expectedErr: false,
},
{
name: "with-bad-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/badcert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedReadBufferSize: 8192,
expectedErr: true,
},
{
name: "with-partial-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedReadBufferSize: 8192,
expectedErr: true,
},
}
for _, scenario := range scenarios {
@ -68,10 +100,13 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
}
if !scenario.expectedErr {
if scenario.cfg.Port != scenario.expectedPort {
t.Errorf("expected port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port)
t.Errorf("expected Port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port)
}
if scenario.cfg.ReadBufferSize != scenario.expectedReadBufferSize {
t.Errorf("expected ReadBufferSize to be %d, got %d", scenario.expectedReadBufferSize, scenario.cfg.ReadBufferSize)
}
if scenario.cfg.Address != scenario.expectedAddress {
t.Errorf("expected address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address)
t.Errorf("expected Address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address)
}
}
})