diff --git a/config/config.go b/config/config.go index 6990cbd6..33b90a55 100644 --- a/config/config.go +++ b/config/config.go @@ -195,19 +195,10 @@ func validateStorageConfig(config *Config) error { config.Storage = &storage.Config{ Type: storage.TypeMemory, } - } - err := storage.Initialize(config.Storage) - if err != nil { - return err - } - // Remove all EndpointStatus that represent endpoints which no longer exist in the configuration - var keys []string - for _, endpoint := range config.Endpoints { - keys = append(keys, endpoint.Key()) - } - numberOfEndpointStatusesDeleted := storage.Get().DeleteAllEndpointStatusesNotInKeys(keys) - if numberOfEndpointStatusesDeleted > 0 { - log.Printf("[config][validateStorageConfig] Deleted %d endpoint statuses because their matching endpoints no longer existed", numberOfEndpointStatusesDeleted) + } else { + if err := config.Storage.ValidateAndSetDefaults(); err != nil { + return err + } } return nil } diff --git a/controller/handler/badge.go b/controller/handler/badge.go index 5d87df4b..3b7ec6b4 100644 --- a/controller/handler/badge.go +++ b/controller/handler/badge.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/storage/store/common" "github.com/gorilla/mux" ) @@ -40,7 +40,7 @@ func UptimeBadge(writer http.ResponseWriter, request *http.Request) { return } key := variables["key"] - uptime, err := storage.Get().GetUptimeByKey(key, from, time.Now()) + uptime, err := store.Get().GetUptimeByKey(key, from, time.Now()) if err != nil { if err == common.ErrEndpointNotFound { writer.WriteHeader(http.StatusNotFound) @@ -79,7 +79,7 @@ func ResponseTimeBadge(writer http.ResponseWriter, request *http.Request) { return } key := variables["key"] - averageResponseTime, err := storage.Get().GetAverageResponseTimeByKey(key, from, time.Now()) + averageResponseTime, err := store.Get().GetAverageResponseTimeByKey(key, from, time.Now()) if err != nil { if err == common.ErrEndpointNotFound { writer.WriteHeader(http.StatusNotFound) diff --git a/controller/handler/badge_test.go b/controller/handler/badge_test.go index 33eed962..1f16f5ac 100644 --- a/controller/handler/badge_test.go +++ b/controller/handler/badge_test.go @@ -9,12 +9,12 @@ import ( "github.com/TwiN/gatus/v3/config" "github.com/TwiN/gatus/v3/core" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/watchdog" ) func TestUptimeBadge(t *testing.T) { - defer storage.Get().Clear() + defer store.Get().Clear() defer cache.Clear() cfg := &config.Config{ Metrics: true, diff --git a/controller/handler/chart.go b/controller/handler/chart.go index f4a30229..f945501a 100644 --- a/controller/handler/chart.go +++ b/controller/handler/chart.go @@ -7,7 +7,7 @@ import ( "sort" "time" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/storage/store/common" "github.com/gorilla/mux" "github.com/wcharczuk/go-chart/v2" @@ -42,7 +42,7 @@ func ResponseTimeChart(writer http.ResponseWriter, r *http.Request) { http.Error(writer, "Durations supported: 7d, 24h", http.StatusBadRequest) return } - hourlyAverageResponseTime, err := storage.Get().GetHourlyAverageResponseTimeByKey(vars["key"], from, time.Now()) + hourlyAverageResponseTime, err := store.Get().GetHourlyAverageResponseTimeByKey(vars["key"], from, time.Now()) if err != nil { if err == common.ErrEndpointNotFound { writer.WriteHeader(http.StatusNotFound) diff --git a/controller/handler/chart_test.go b/controller/handler/chart_test.go index 32edd01b..dd950f18 100644 --- a/controller/handler/chart_test.go +++ b/controller/handler/chart_test.go @@ -8,12 +8,12 @@ import ( "github.com/TwiN/gatus/v3/config" "github.com/TwiN/gatus/v3/core" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/watchdog" ) func TestResponseTimeChart(t *testing.T) { - defer storage.Get().Clear() + defer store.Get().Clear() defer cache.Clear() cfg := &config.Config{ Metrics: true, diff --git a/controller/handler/endpoint_status.go b/controller/handler/endpoint_status.go index 260825b4..06c9813e 100644 --- a/controller/handler/endpoint_status.go +++ b/controller/handler/endpoint_status.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/storage/store/common" "github.com/TwiN/gatus/v3/storage/store/common/paging" "github.com/TwiN/gocache" @@ -44,7 +44,7 @@ func EndpointStatuses(writer http.ResponseWriter, r *http.Request) { var err error buffer := &bytes.Buffer{} gzipWriter := gzip.NewWriter(buffer) - endpointStatuses, err := storage.Get().GetAllEndpointStatuses(paging.NewEndpointStatusParams().WithResults(page, pageSize)) + endpointStatuses, err := store.Get().GetAllEndpointStatuses(paging.NewEndpointStatusParams().WithResults(page, pageSize)) if err != nil { log.Printf("[handler][EndpointStatuses] Failed to retrieve endpoint statuses: %s", err.Error()) http.Error(writer, err.Error(), http.StatusInternalServerError) @@ -76,7 +76,7 @@ func EndpointStatuses(writer http.ResponseWriter, r *http.Request) { func EndpointStatus(writer http.ResponseWriter, r *http.Request) { page, pageSize := extractPageAndPageSizeFromRequest(r) vars := mux.Vars(r) - endpointStatus, err := storage.Get().GetEndpointStatusByKey(vars["key"], paging.NewEndpointStatusParams().WithResults(page, pageSize).WithEvents(1, common.MaximumNumberOfEvents)) + endpointStatus, err := store.Get().GetEndpointStatusByKey(vars["key"], paging.NewEndpointStatusParams().WithResults(page, pageSize).WithEvents(1, common.MaximumNumberOfEvents)) if err != nil { if err == common.ErrEndpointNotFound { http.Error(writer, err.Error(), http.StatusNotFound) diff --git a/controller/handler/endpoint_status_test.go b/controller/handler/endpoint_status_test.go index b4b6fcb3..8cb86b16 100644 --- a/controller/handler/endpoint_status_test.go +++ b/controller/handler/endpoint_status_test.go @@ -8,7 +8,7 @@ import ( "github.com/TwiN/gatus/v3/config" "github.com/TwiN/gatus/v3/core" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/watchdog" ) @@ -84,7 +84,7 @@ var ( ) func TestEndpointStatus(t *testing.T) { - defer storage.Get().Clear() + defer store.Get().Clear() defer cache.Clear() cfg := &config.Config{ Metrics: true, @@ -153,12 +153,12 @@ func TestEndpointStatus(t *testing.T) { } func TestEndpointStatuses(t *testing.T) { - defer storage.Get().Clear() + defer store.Get().Clear() defer cache.Clear() firstResult := &testSuccessfulResult secondResult := &testUnsuccessfulResult - storage.Get().Insert(&testEndpoint, firstResult) - storage.Get().Insert(&testEndpoint, secondResult) + store.Get().Insert(&testEndpoint, firstResult) + store.Get().Insert(&testEndpoint, secondResult) // Can't be bothered dealing with timezone issues on the worker that runs the automated tests firstResult.Timestamp = time.Time{} secondResult.Timestamp = time.Time{} diff --git a/controller/handler/spa_test.go b/controller/handler/spa_test.go index a4404667..19142ddb 100644 --- a/controller/handler/spa_test.go +++ b/controller/handler/spa_test.go @@ -8,12 +8,12 @@ import ( "github.com/TwiN/gatus/v3/config" "github.com/TwiN/gatus/v3/core" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/watchdog" ) func TestSinglePageApplication(t *testing.T) { - defer storage.Get().Clear() + defer store.Get().Clear() defer cache.Clear() cfg := &config.Config{ Metrics: true, diff --git a/main.go b/main.go index c6e9c74a..90345148 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,7 @@ import ( "github.com/TwiN/gatus/v3/config" "github.com/TwiN/gatus/v3/controller" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" "github.com/TwiN/gatus/v3/watchdog" ) @@ -18,6 +18,7 @@ func main() { if err != nil { panic(err) } + initializeStorage(cfg) start(cfg) // Wait for termination signal signalChannel := make(chan os.Signal, 1) @@ -46,8 +47,7 @@ func stop() { } func save() { - err := storage.Get().Save() - if err != nil { + if err := store.Get().Save(); err != nil { log.Println("Failed to save storage provider:", err.Error()) } } @@ -62,6 +62,27 @@ func loadConfiguration() (cfg *config.Config, err error) { return } +// initializeStorage initializes the storage provider +// +// Q: "TwiN, why are you putting this here? Wouldn't it make more sense to have this in the config?!" +// A: Yes. Yes it would make more sense to have it in the config package. But I don't want to import +// the massive SQL dependencies just because I want to import the config, so here we are. +func initializeStorage(cfg *config.Config) { + err := store.Initialize(cfg.Storage) + if err != nil { + panic(err) + } + // Remove all EndpointStatus that represent endpoints which no longer exist in the configuration + var keys []string + for _, endpoint := range cfg.Endpoints { + keys = append(keys, endpoint.Key()) + } + numberOfEndpointStatusesDeleted := store.Get().DeleteAllEndpointStatusesNotInKeys(keys) + if numberOfEndpointStatusesDeleted > 0 { + log.Printf("[config][validateStorageConfig] Deleted %d endpoint statuses because their matching endpoints no longer existed", numberOfEndpointStatusesDeleted) + } +} + func listenToConfigurationFileChanges(cfg *config.Config) { for { time.Sleep(30 * time.Second) diff --git a/storage/config.go b/storage/config.go index e3cde2f9..83358b6d 100644 --- a/storage/config.go +++ b/storage/config.go @@ -1,5 +1,11 @@ package storage +import "errors" + +var ( + ErrSQLStorageRequiresFile = errors.New("sql storage requires a non-empty file to be defined") +) + // Config is the configuration for storage type Config struct { // File is the path of the file to use for persistence @@ -12,3 +18,11 @@ type Config struct { // If blank, uses the default in-memory store Type Type `yaml:"type"` } + +// ValidateAndSetDefaults validates the configuration and sets the default values (if applicable) +func (c *Config) ValidateAndSetDefaults() error { + if (c.Type == TypePostgres || c.Type == TypeSQLite) && len(c.File) == 0 { + return ErrSQLStorageRequiresFile + } + return nil +} diff --git a/storage/storage.go b/storage/storage.go deleted file mode 100644 index d4bda379..00000000 --- a/storage/storage.go +++ /dev/null @@ -1,91 +0,0 @@ -package storage - -import ( - "context" - "log" - "time" - - "github.com/TwiN/gatus/v3/storage/store" - "github.com/TwiN/gatus/v3/storage/store/memory" - "github.com/TwiN/gatus/v3/storage/store/sql" -) - -var ( - provider store.Store - - // initialized keeps track of whether the storage provider was initialized - // Because store.Store is an interface, a nil check wouldn't be sufficient, so instead of doing reflection - // every single time Get is called, we'll just lazily keep track of its existence through this variable - initialized bool - - ctx context.Context - cancelFunc context.CancelFunc -) - -// Get retrieves the storage provider -func Get() store.Store { - if !initialized { - log.Println("[storage][Get] Provider requested before it was initialized, automatically initializing") - err := Initialize(nil) - if err != nil { - panic("failed to automatically initialize store: " + err.Error()) - } - } - return provider -} - -// Initialize instantiates the storage provider based on the Config provider -func Initialize(cfg *Config) error { - initialized = true - var err error - if cancelFunc != nil { - // Stop the active autoSaveStore task, if there's already one - cancelFunc() - } - if cfg == nil { - cfg = &Config{} - } - if len(cfg.File) == 0 && cfg.Type != TypePostgres { - log.Printf("[storage][Initialize] Creating storage provider with type=%s and file=%s", cfg.Type, cfg.File) - } else { - log.Printf("[storage][Initialize] Creating storage provider with type=%s", cfg.Type) - } - ctx, cancelFunc = context.WithCancel(context.Background()) - switch cfg.Type { - case TypeSQLite, TypePostgres: - provider, err = sql.NewStore(string(cfg.Type), cfg.File) - if err != nil { - return err - } - case TypeMemory: - fallthrough - default: - if len(cfg.File) > 0 { - provider, err = memory.NewStore(cfg.File) - if err != nil { - return err - } - go autoSaveStore(ctx, provider, 7*time.Minute) - } else { - provider, _ = memory.NewStore("") - } - } - return nil -} - -// autoSaveStore automatically calls the Save function of the provider at every interval -func autoSaveStore(ctx context.Context, provider store.Store, interval time.Duration) { - for { - select { - case <-ctx.Done(): - log.Printf("[storage][autoSaveStore] Stopping active job") - return - case <-time.After(interval): - log.Printf("[storage][autoSaveStore] Saving") - err := provider.Save() - if err != nil { - log.Println("[storage][autoSaveStore] Save failed:", err.Error()) - } - } - } -} diff --git a/storage/storage_test.go b/storage/storage_test.go deleted file mode 100644 index 504b6af9..00000000 --- a/storage/storage_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package storage - -import ( - "testing" - "time" - - "github.com/TwiN/gatus/v3/storage/store/sql" -) - -func TestGet(t *testing.T) { - store := Get() - if store == nil { - t.Error("store should've been automatically initialized") - } -} - -func TestInitialize(t *testing.T) { - type Scenario struct { - Name string - Cfg *Config - ExpectedErr error - } - scenarios := []Scenario{ - { - Name: "nil", - Cfg: nil, - ExpectedErr: nil, - }, - { - Name: "blank", - Cfg: &Config{}, - ExpectedErr: nil, - }, - { - Name: "memory-no-file", - Cfg: &Config{Type: TypeMemory}, - ExpectedErr: nil, - }, - { - Name: "memory-with-file", - Cfg: &Config{Type: TypeMemory, File: t.TempDir() + "/TestInitialize_memory-with-file.db"}, - ExpectedErr: nil, - }, - { - Name: "sqlite-no-file", - Cfg: &Config{Type: TypeSQLite}, - ExpectedErr: sql.ErrFilePathNotSpecified, - }, - { - Name: "sqlite-with-file", - Cfg: &Config{Type: TypeSQLite, File: t.TempDir() + "/TestInitialize_sqlite-with-file.db"}, - ExpectedErr: nil, - }, - } - for _, scenario := range scenarios { - t.Run(scenario.Name, func(t *testing.T) { - err := Initialize(scenario.Cfg) - if err != scenario.ExpectedErr { - t.Errorf("expected %v, got %v", scenario.ExpectedErr, err) - } - if err != nil { - return - } - if cancelFunc == nil { - t.Error("cancelFunc shouldn't have been nil") - } - if ctx == nil { - t.Error("ctx shouldn't have been nil") - } - if provider == nil { - t.Fatal("provider shouldn't have been nit") - } - provider.Close() - // Try to initialize it again - err = Initialize(scenario.Cfg) - if err != scenario.ExpectedErr { - t.Errorf("expected %v, got %v", scenario.ExpectedErr, err) - return - } - provider.Close() - }) - } -} - -func TestAutoSave(t *testing.T) { - file := t.TempDir() + "/TestAutoSave.db" - if err := Initialize(&Config{File: file}); err != nil { - t.Fatal("shouldn't have returned an error") - } - go autoSaveStore(ctx, provider, 3*time.Millisecond) - time.Sleep(15 * time.Millisecond) - cancelFunc() - time.Sleep(50 * time.Millisecond) -} diff --git a/storage/store/store.go b/storage/store/store.go index 7220c58a..2bb24365 100644 --- a/storage/store/store.go +++ b/storage/store/store.go @@ -1,9 +1,12 @@ package store import ( + "context" + "log" "time" "github.com/TwiN/gatus/v3/core" + "github.com/TwiN/gatus/v3/storage" "github.com/TwiN/gatus/v3/storage/store/common/paging" "github.com/TwiN/gatus/v3/storage/store/memory" "github.com/TwiN/gatus/v3/storage/store/sql" @@ -56,3 +59,82 @@ var ( _ Store = (*memory.Store)(nil) _ Store = (*sql.Store)(nil) ) + +var ( + store Store + + // initialized keeps track of whether the storage provider was initialized + // Because store.Store is an interface, a nil check wouldn't be sufficient, so instead of doing reflection + // every single time Get is called, we'll just lazily keep track of its existence through this variable + initialized bool + + ctx context.Context + cancelFunc context.CancelFunc +) + +func Get() Store { + if !initialized { + log.Println("[store][Get] Provider requested before it was initialized, automatically initializing") + err := Initialize(nil) + if err != nil { + panic("failed to automatically initialize store: " + err.Error()) + } + } + return store +} + +// Initialize instantiates the storage provider based on the Config provider +func Initialize(cfg *storage.Config) error { + initialized = true + var err error + if cancelFunc != nil { + // Stop the active autoSave task, if there's already one + cancelFunc() + } + if cfg == nil { + cfg = &storage.Config{} + } + if len(cfg.File) == 0 && cfg.Type != storage.TypePostgres { + log.Printf("[store][Initialize] Creating storage provider with type=%s and file=%s", cfg.Type, cfg.File) + } else { + log.Printf("[store][Initialize] Creating storage provider with type=%s", cfg.Type) + } + ctx, cancelFunc = context.WithCancel(context.Background()) + switch cfg.Type { + case storage.TypeSQLite, storage.TypePostgres: + store, err = sql.NewStore(string(cfg.Type), cfg.File) + if err != nil { + return err + } + case storage.TypeMemory: + fallthrough + default: + if len(cfg.File) > 0 { + store, err = memory.NewStore(cfg.File) + if err != nil { + return err + } + go autoSave(ctx, store, 7*time.Minute) + } else { + store, _ = memory.NewStore("") + } + } + return nil +} + +// autoSave automatically calls the Save function of the provider at every interval +func autoSave(ctx context.Context, store Store, interval time.Duration) { + for { + select { + case <-ctx.Done(): + log.Printf("[store][autoSave] Stopping active job") + return + case <-time.After(interval): + log.Printf("[store][autoSave] Saving") + err := store.Save() + if err != nil { + log.Println("[store][autoSave] Save failed:", err.Error()) + } + } + } +} diff --git a/storage/store/store_test.go b/storage/store/store_test.go index 55f0183c..fa93bdba 100644 --- a/storage/store/store_test.go +++ b/storage/store/store_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/TwiN/gatus/v3/core" + "github.com/TwiN/gatus/v3/storage" "github.com/TwiN/gatus/v3/storage/store/common" "github.com/TwiN/gatus/v3/storage/store/common/paging" "github.com/TwiN/gatus/v3/storage/store/memory" @@ -520,3 +521,89 @@ func TestStore_DeleteAllEndpointStatusesNotInKeys(t *testing.T) { }) } } + +func TestGet(t *testing.T) { + store := Get() + if store == nil { + t.Error("store should've been automatically initialized") + } +} + +func TestInitialize(t *testing.T) { + type Scenario struct { + Name string + Cfg *storage.Config + ExpectedErr error + } + scenarios := []Scenario{ + { + Name: "nil", + Cfg: nil, + ExpectedErr: nil, + }, + { + Name: "blank", + Cfg: &storage.Config{}, + ExpectedErr: nil, + }, + { + Name: "memory-no-file", + Cfg: &storage.Config{Type: storage.TypeMemory}, + ExpectedErr: nil, + }, + { + Name: "memory-with-file", + Cfg: &storage.Config{Type: storage.TypeMemory, File: t.TempDir() + "/TestInitialize_memory-with-file.db"}, + ExpectedErr: nil, + }, + { + Name: "sqlite-no-file", + Cfg: &storage.Config{Type: storage.TypeSQLite}, + ExpectedErr: sql.ErrFilePathNotSpecified, + }, + { + Name: "sqlite-with-file", + Cfg: &storage.Config{Type: storage.TypeSQLite, File: t.TempDir() + "/TestInitialize_sqlite-with-file.db"}, + ExpectedErr: nil, + }, + } + for _, scenario := range scenarios { + t.Run(scenario.Name, func(t *testing.T) { + err := Initialize(scenario.Cfg) + if err != scenario.ExpectedErr { + t.Errorf("expected %v, got %v", scenario.ExpectedErr, err) + } + if err != nil { + return + } + if cancelFunc == nil { + t.Error("cancelFunc shouldn't have been nil") + } + if ctx == nil { + t.Error("ctx shouldn't have been nil") + } + if store == nil { + t.Fatal("provider shouldn't have been nit") + } + store.Close() + // Try to initialize it again + err = Initialize(scenario.Cfg) + if err != scenario.ExpectedErr { + t.Errorf("expected %v, got %v", scenario.ExpectedErr, err) + return + } + store.Close() + }) + } +} + +func TestAutoSave(t *testing.T) { + file := t.TempDir() + "/TestAutoSave.db" + if err := Initialize(&storage.Config{File: file}); err != nil { + t.Fatal("shouldn't have returned an error") + } + go autoSave(ctx, store, 3*time.Millisecond) + time.Sleep(15 * time.Millisecond) + cancelFunc() + time.Sleep(50 * time.Millisecond) +} diff --git a/watchdog/watchdog.go b/watchdog/watchdog.go index a67a071e..eb96b0b4 100644 --- a/watchdog/watchdog.go +++ b/watchdog/watchdog.go @@ -11,7 +11,7 @@ import ( "github.com/TwiN/gatus/v3/config/maintenance" "github.com/TwiN/gatus/v3/core" "github.com/TwiN/gatus/v3/metric" - "github.com/TwiN/gatus/v3/storage" + "github.com/TwiN/gatus/v3/storage/store" ) var ( @@ -29,13 +29,13 @@ func Monitor(cfg *config.Config) { for _, endpoint := range cfg.Endpoints { if endpoint.IsEnabled() { // To prevent multiple requests from running at the same time, we'll wait for a little before each iteration - time.Sleep(1111 * time.Millisecond) + time.Sleep(777 * time.Millisecond) go monitor(endpoint, cfg.Alerting, cfg.Maintenance, cfg.DisableMonitoringLock, cfg.Metrics, cfg.Debug, ctx) } } } -// monitor monitors a single endpoint in a loop +// monitor a single endpoint in a loop func monitor(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, disableMonitoringLock, enabledMetrics, debug bool, ctx context.Context) { // Run it immediately on start execute(endpoint, alertingConfig, maintenanceConfig, disableMonitoringLock, enabledMetrics, debug) @@ -88,7 +88,7 @@ func execute(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenan // UpdateEndpointStatuses updates the slice of endpoint statuses func UpdateEndpointStatuses(endpoint *core.Endpoint, result *core.Result) { - if err := storage.Get().Insert(endpoint, result); err != nil { + if err := store.Get().Insert(endpoint, result); err != nil { log.Println("[watchdog][UpdateEndpointStatuses] Failed to insert data in storage:", err.Error()) } }