Move store initialization to store package

This will allow importing storage.Config without importing every SQL drivers in the known universe
This commit is contained in:
TwiN 2021-10-28 19:35:46 -04:00
parent 257f859825
commit 9287e2f9e2
15 changed files with 234 additions and 224 deletions

View File

@ -195,19 +195,10 @@ func validateStorageConfig(config *Config) error {
config.Storage = &storage.Config{
Type: storage.TypeMemory,
}
}
err := storage.Initialize(config.Storage)
if err != nil {
return err
}
// Remove all EndpointStatus that represent endpoints which no longer exist in the configuration
var keys []string
for _, endpoint := range config.Endpoints {
keys = append(keys, endpoint.Key())
}
numberOfEndpointStatusesDeleted := storage.Get().DeleteAllEndpointStatusesNotInKeys(keys)
if numberOfEndpointStatusesDeleted > 0 {
log.Printf("[config][validateStorageConfig] Deleted %d endpoint statuses because their matching endpoints no longer existed", numberOfEndpointStatusesDeleted)
} else {
if err := config.Storage.ValidateAndSetDefaults(); err != nil {
return err
}
}
return nil
}

View File

@ -7,7 +7,7 @@ import (
"strings"
"time"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/storage/store/common"
"github.com/gorilla/mux"
)
@ -40,7 +40,7 @@ func UptimeBadge(writer http.ResponseWriter, request *http.Request) {
return
}
key := variables["key"]
uptime, err := storage.Get().GetUptimeByKey(key, from, time.Now())
uptime, err := store.Get().GetUptimeByKey(key, from, time.Now())
if err != nil {
if err == common.ErrEndpointNotFound {
writer.WriteHeader(http.StatusNotFound)
@ -79,7 +79,7 @@ func ResponseTimeBadge(writer http.ResponseWriter, request *http.Request) {
return
}
key := variables["key"]
averageResponseTime, err := storage.Get().GetAverageResponseTimeByKey(key, from, time.Now())
averageResponseTime, err := store.Get().GetAverageResponseTimeByKey(key, from, time.Now())
if err != nil {
if err == common.ErrEndpointNotFound {
writer.WriteHeader(http.StatusNotFound)

View File

@ -9,12 +9,12 @@ import (
"github.com/TwiN/gatus/v3/config"
"github.com/TwiN/gatus/v3/core"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/watchdog"
)
func TestUptimeBadge(t *testing.T) {
defer storage.Get().Clear()
defer store.Get().Clear()
defer cache.Clear()
cfg := &config.Config{
Metrics: true,

View File

@ -7,7 +7,7 @@ import (
"sort"
"time"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/storage/store/common"
"github.com/gorilla/mux"
"github.com/wcharczuk/go-chart/v2"
@ -42,7 +42,7 @@ func ResponseTimeChart(writer http.ResponseWriter, r *http.Request) {
http.Error(writer, "Durations supported: 7d, 24h", http.StatusBadRequest)
return
}
hourlyAverageResponseTime, err := storage.Get().GetHourlyAverageResponseTimeByKey(vars["key"], from, time.Now())
hourlyAverageResponseTime, err := store.Get().GetHourlyAverageResponseTimeByKey(vars["key"], from, time.Now())
if err != nil {
if err == common.ErrEndpointNotFound {
writer.WriteHeader(http.StatusNotFound)

View File

@ -8,12 +8,12 @@ import (
"github.com/TwiN/gatus/v3/config"
"github.com/TwiN/gatus/v3/core"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/watchdog"
)
func TestResponseTimeChart(t *testing.T) {
defer storage.Get().Clear()
defer store.Get().Clear()
defer cache.Clear()
cfg := &config.Config{
Metrics: true,

View File

@ -10,7 +10,7 @@ import (
"strings"
"time"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/storage/store/common"
"github.com/TwiN/gatus/v3/storage/store/common/paging"
"github.com/TwiN/gocache"
@ -44,7 +44,7 @@ func EndpointStatuses(writer http.ResponseWriter, r *http.Request) {
var err error
buffer := &bytes.Buffer{}
gzipWriter := gzip.NewWriter(buffer)
endpointStatuses, err := storage.Get().GetAllEndpointStatuses(paging.NewEndpointStatusParams().WithResults(page, pageSize))
endpointStatuses, err := store.Get().GetAllEndpointStatuses(paging.NewEndpointStatusParams().WithResults(page, pageSize))
if err != nil {
log.Printf("[handler][EndpointStatuses] Failed to retrieve endpoint statuses: %s", err.Error())
http.Error(writer, err.Error(), http.StatusInternalServerError)
@ -76,7 +76,7 @@ func EndpointStatuses(writer http.ResponseWriter, r *http.Request) {
func EndpointStatus(writer http.ResponseWriter, r *http.Request) {
page, pageSize := extractPageAndPageSizeFromRequest(r)
vars := mux.Vars(r)
endpointStatus, err := storage.Get().GetEndpointStatusByKey(vars["key"], paging.NewEndpointStatusParams().WithResults(page, pageSize).WithEvents(1, common.MaximumNumberOfEvents))
endpointStatus, err := store.Get().GetEndpointStatusByKey(vars["key"], paging.NewEndpointStatusParams().WithResults(page, pageSize).WithEvents(1, common.MaximumNumberOfEvents))
if err != nil {
if err == common.ErrEndpointNotFound {
http.Error(writer, err.Error(), http.StatusNotFound)

View File

@ -8,7 +8,7 @@ import (
"github.com/TwiN/gatus/v3/config"
"github.com/TwiN/gatus/v3/core"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/watchdog"
)
@ -84,7 +84,7 @@ var (
)
func TestEndpointStatus(t *testing.T) {
defer storage.Get().Clear()
defer store.Get().Clear()
defer cache.Clear()
cfg := &config.Config{
Metrics: true,
@ -153,12 +153,12 @@ func TestEndpointStatus(t *testing.T) {
}
func TestEndpointStatuses(t *testing.T) {
defer storage.Get().Clear()
defer store.Get().Clear()
defer cache.Clear()
firstResult := &testSuccessfulResult
secondResult := &testUnsuccessfulResult
storage.Get().Insert(&testEndpoint, firstResult)
storage.Get().Insert(&testEndpoint, secondResult)
store.Get().Insert(&testEndpoint, firstResult)
store.Get().Insert(&testEndpoint, secondResult)
// Can't be bothered dealing with timezone issues on the worker that runs the automated tests
firstResult.Timestamp = time.Time{}
secondResult.Timestamp = time.Time{}

View File

@ -8,12 +8,12 @@ import (
"github.com/TwiN/gatus/v3/config"
"github.com/TwiN/gatus/v3/core"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/watchdog"
)
func TestSinglePageApplication(t *testing.T) {
defer storage.Get().Clear()
defer store.Get().Clear()
defer cache.Clear()
cfg := &config.Config{
Metrics: true,

27
main.go
View File

@ -9,7 +9,7 @@ import (
"github.com/TwiN/gatus/v3/config"
"github.com/TwiN/gatus/v3/controller"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/watchdog"
)
@ -18,6 +18,7 @@ func main() {
if err != nil {
panic(err)
}
initializeStorage(cfg)
start(cfg)
// Wait for termination signal
signalChannel := make(chan os.Signal, 1)
@ -46,8 +47,7 @@ func stop() {
}
func save() {
err := storage.Get().Save()
if err != nil {
if err := store.Get().Save(); err != nil {
log.Println("Failed to save storage provider:", err.Error())
}
}
@ -62,6 +62,27 @@ func loadConfiguration() (cfg *config.Config, err error) {
return
}
// initializeStorage initializes the storage provider
//
// Q: "TwiN, why are you putting this here? Wouldn't it make more sense to have this in the config?!"
// A: Yes. Yes it would make more sense to have it in the config package. But I don't want to import
// the massive SQL dependencies just because I want to import the config, so here we are.
func initializeStorage(cfg *config.Config) {
err := store.Initialize(cfg.Storage)
if err != nil {
panic(err)
}
// Remove all EndpointStatus that represent endpoints which no longer exist in the configuration
var keys []string
for _, endpoint := range cfg.Endpoints {
keys = append(keys, endpoint.Key())
}
numberOfEndpointStatusesDeleted := store.Get().DeleteAllEndpointStatusesNotInKeys(keys)
if numberOfEndpointStatusesDeleted > 0 {
log.Printf("[config][validateStorageConfig] Deleted %d endpoint statuses because their matching endpoints no longer existed", numberOfEndpointStatusesDeleted)
}
}
func listenToConfigurationFileChanges(cfg *config.Config) {
for {
time.Sleep(30 * time.Second)

View File

@ -1,5 +1,11 @@
package storage
import "errors"
var (
ErrSQLStorageRequiresFile = errors.New("sql storage requires a non-empty file to be defined")
)
// Config is the configuration for storage
type Config struct {
// File is the path of the file to use for persistence
@ -12,3 +18,11 @@ type Config struct {
// If blank, uses the default in-memory store
Type Type `yaml:"type"`
}
// ValidateAndSetDefaults validates the configuration and sets the default values (if applicable)
func (c *Config) ValidateAndSetDefaults() error {
if (c.Type == TypePostgres || c.Type == TypeSQLite) && len(c.File) == 0 {
return ErrSQLStorageRequiresFile
}
return nil
}

View File

@ -1,91 +0,0 @@
package storage
import (
"context"
"log"
"time"
"github.com/TwiN/gatus/v3/storage/store"
"github.com/TwiN/gatus/v3/storage/store/memory"
"github.com/TwiN/gatus/v3/storage/store/sql"
)
var (
provider store.Store
// initialized keeps track of whether the storage provider was initialized
// Because store.Store is an interface, a nil check wouldn't be sufficient, so instead of doing reflection
// every single time Get is called, we'll just lazily keep track of its existence through this variable
initialized bool
ctx context.Context
cancelFunc context.CancelFunc
)
// Get retrieves the storage provider
func Get() store.Store {
if !initialized {
log.Println("[storage][Get] Provider requested before it was initialized, automatically initializing")
err := Initialize(nil)
if err != nil {
panic("failed to automatically initialize store: " + err.Error())
}
}
return provider
}
// Initialize instantiates the storage provider based on the Config provider
func Initialize(cfg *Config) error {
initialized = true
var err error
if cancelFunc != nil {
// Stop the active autoSaveStore task, if there's already one
cancelFunc()
}
if cfg == nil {
cfg = &Config{}
}
if len(cfg.File) == 0 && cfg.Type != TypePostgres {
log.Printf("[storage][Initialize] Creating storage provider with type=%s and file=%s", cfg.Type, cfg.File)
} else {
log.Printf("[storage][Initialize] Creating storage provider with type=%s", cfg.Type)
}
ctx, cancelFunc = context.WithCancel(context.Background())
switch cfg.Type {
case TypeSQLite, TypePostgres:
provider, err = sql.NewStore(string(cfg.Type), cfg.File)
if err != nil {
return err
}
case TypeMemory:
fallthrough
default:
if len(cfg.File) > 0 {
provider, err = memory.NewStore(cfg.File)
if err != nil {
return err
}
go autoSaveStore(ctx, provider, 7*time.Minute)
} else {
provider, _ = memory.NewStore("")
}
}
return nil
}
// autoSaveStore automatically calls the Save function of the provider at every interval
func autoSaveStore(ctx context.Context, provider store.Store, interval time.Duration) {
for {
select {
case <-ctx.Done():
log.Printf("[storage][autoSaveStore] Stopping active job")
return
case <-time.After(interval):
log.Printf("[storage][autoSaveStore] Saving")
err := provider.Save()
if err != nil {
log.Println("[storage][autoSaveStore] Save failed:", err.Error())
}
}
}
}

View File

@ -1,94 +0,0 @@
package storage
import (
"testing"
"time"
"github.com/TwiN/gatus/v3/storage/store/sql"
)
func TestGet(t *testing.T) {
store := Get()
if store == nil {
t.Error("store should've been automatically initialized")
}
}
func TestInitialize(t *testing.T) {
type Scenario struct {
Name string
Cfg *Config
ExpectedErr error
}
scenarios := []Scenario{
{
Name: "nil",
Cfg: nil,
ExpectedErr: nil,
},
{
Name: "blank",
Cfg: &Config{},
ExpectedErr: nil,
},
{
Name: "memory-no-file",
Cfg: &Config{Type: TypeMemory},
ExpectedErr: nil,
},
{
Name: "memory-with-file",
Cfg: &Config{Type: TypeMemory, File: t.TempDir() + "/TestInitialize_memory-with-file.db"},
ExpectedErr: nil,
},
{
Name: "sqlite-no-file",
Cfg: &Config{Type: TypeSQLite},
ExpectedErr: sql.ErrFilePathNotSpecified,
},
{
Name: "sqlite-with-file",
Cfg: &Config{Type: TypeSQLite, File: t.TempDir() + "/TestInitialize_sqlite-with-file.db"},
ExpectedErr: nil,
},
}
for _, scenario := range scenarios {
t.Run(scenario.Name, func(t *testing.T) {
err := Initialize(scenario.Cfg)
if err != scenario.ExpectedErr {
t.Errorf("expected %v, got %v", scenario.ExpectedErr, err)
}
if err != nil {
return
}
if cancelFunc == nil {
t.Error("cancelFunc shouldn't have been nil")
}
if ctx == nil {
t.Error("ctx shouldn't have been nil")
}
if provider == nil {
t.Fatal("provider shouldn't have been nit")
}
provider.Close()
// Try to initialize it again
err = Initialize(scenario.Cfg)
if err != scenario.ExpectedErr {
t.Errorf("expected %v, got %v", scenario.ExpectedErr, err)
return
}
provider.Close()
})
}
}
func TestAutoSave(t *testing.T) {
file := t.TempDir() + "/TestAutoSave.db"
if err := Initialize(&Config{File: file}); err != nil {
t.Fatal("shouldn't have returned an error")
}
go autoSaveStore(ctx, provider, 3*time.Millisecond)
time.Sleep(15 * time.Millisecond)
cancelFunc()
time.Sleep(50 * time.Millisecond)
}

View File

@ -1,9 +1,12 @@
package store
import (
"context"
"log"
"time"
"github.com/TwiN/gatus/v3/core"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store/common/paging"
"github.com/TwiN/gatus/v3/storage/store/memory"
"github.com/TwiN/gatus/v3/storage/store/sql"
@ -56,3 +59,82 @@ var (
_ Store = (*memory.Store)(nil)
_ Store = (*sql.Store)(nil)
)
var (
store Store
// initialized keeps track of whether the storage provider was initialized
// Because store.Store is an interface, a nil check wouldn't be sufficient, so instead of doing reflection
// every single time Get is called, we'll just lazily keep track of its existence through this variable
initialized bool
ctx context.Context
cancelFunc context.CancelFunc
)
func Get() Store {
if !initialized {
log.Println("[store][Get] Provider requested before it was initialized, automatically initializing")
err := Initialize(nil)
if err != nil {
panic("failed to automatically initialize store: " + err.Error())
}
}
return store
}
// Initialize instantiates the storage provider based on the Config provider
func Initialize(cfg *storage.Config) error {
initialized = true
var err error
if cancelFunc != nil {
// Stop the active autoSave task, if there's already one
cancelFunc()
}
if cfg == nil {
cfg = &storage.Config{}
}
if len(cfg.File) == 0 && cfg.Type != storage.TypePostgres {
log.Printf("[store][Initialize] Creating storage provider with type=%s and file=%s", cfg.Type, cfg.File)
} else {
log.Printf("[store][Initialize] Creating storage provider with type=%s", cfg.Type)
}
ctx, cancelFunc = context.WithCancel(context.Background())
switch cfg.Type {
case storage.TypeSQLite, storage.TypePostgres:
store, err = sql.NewStore(string(cfg.Type), cfg.File)
if err != nil {
return err
}
case storage.TypeMemory:
fallthrough
default:
if len(cfg.File) > 0 {
store, err = memory.NewStore(cfg.File)
if err != nil {
return err
}
go autoSave(ctx, store, 7*time.Minute)
} else {
store, _ = memory.NewStore("")
}
}
return nil
}
// autoSave automatically calls the Save function of the provider at every interval
func autoSave(ctx context.Context, store Store, interval time.Duration) {
for {
select {
case <-ctx.Done():
log.Printf("[store][autoSave] Stopping active job")
return
case <-time.After(interval):
log.Printf("[store][autoSave] Saving")
err := store.Save()
if err != nil {
log.Println("[store][autoSave] Save failed:", err.Error())
}
}
}
}

View File

@ -5,6 +5,7 @@ import (
"time"
"github.com/TwiN/gatus/v3/core"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store/common"
"github.com/TwiN/gatus/v3/storage/store/common/paging"
"github.com/TwiN/gatus/v3/storage/store/memory"
@ -520,3 +521,89 @@ func TestStore_DeleteAllEndpointStatusesNotInKeys(t *testing.T) {
})
}
}
func TestGet(t *testing.T) {
store := Get()
if store == nil {
t.Error("store should've been automatically initialized")
}
}
func TestInitialize(t *testing.T) {
type Scenario struct {
Name string
Cfg *storage.Config
ExpectedErr error
}
scenarios := []Scenario{
{
Name: "nil",
Cfg: nil,
ExpectedErr: nil,
},
{
Name: "blank",
Cfg: &storage.Config{},
ExpectedErr: nil,
},
{
Name: "memory-no-file",
Cfg: &storage.Config{Type: storage.TypeMemory},
ExpectedErr: nil,
},
{
Name: "memory-with-file",
Cfg: &storage.Config{Type: storage.TypeMemory, File: t.TempDir() + "/TestInitialize_memory-with-file.db"},
ExpectedErr: nil,
},
{
Name: "sqlite-no-file",
Cfg: &storage.Config{Type: storage.TypeSQLite},
ExpectedErr: sql.ErrFilePathNotSpecified,
},
{
Name: "sqlite-with-file",
Cfg: &storage.Config{Type: storage.TypeSQLite, File: t.TempDir() + "/TestInitialize_sqlite-with-file.db"},
ExpectedErr: nil,
},
}
for _, scenario := range scenarios {
t.Run(scenario.Name, func(t *testing.T) {
err := Initialize(scenario.Cfg)
if err != scenario.ExpectedErr {
t.Errorf("expected %v, got %v", scenario.ExpectedErr, err)
}
if err != nil {
return
}
if cancelFunc == nil {
t.Error("cancelFunc shouldn't have been nil")
}
if ctx == nil {
t.Error("ctx shouldn't have been nil")
}
if store == nil {
t.Fatal("provider shouldn't have been nit")
}
store.Close()
// Try to initialize it again
err = Initialize(scenario.Cfg)
if err != scenario.ExpectedErr {
t.Errorf("expected %v, got %v", scenario.ExpectedErr, err)
return
}
store.Close()
})
}
}
func TestAutoSave(t *testing.T) {
file := t.TempDir() + "/TestAutoSave.db"
if err := Initialize(&storage.Config{File: file}); err != nil {
t.Fatal("shouldn't have returned an error")
}
go autoSave(ctx, store, 3*time.Millisecond)
time.Sleep(15 * time.Millisecond)
cancelFunc()
time.Sleep(50 * time.Millisecond)
}

View File

@ -11,7 +11,7 @@ import (
"github.com/TwiN/gatus/v3/config/maintenance"
"github.com/TwiN/gatus/v3/core"
"github.com/TwiN/gatus/v3/metric"
"github.com/TwiN/gatus/v3/storage"
"github.com/TwiN/gatus/v3/storage/store"
)
var (
@ -29,13 +29,13 @@ func Monitor(cfg *config.Config) {
for _, endpoint := range cfg.Endpoints {
if endpoint.IsEnabled() {
// To prevent multiple requests from running at the same time, we'll wait for a little before each iteration
time.Sleep(1111 * time.Millisecond)
time.Sleep(777 * time.Millisecond)
go monitor(endpoint, cfg.Alerting, cfg.Maintenance, cfg.DisableMonitoringLock, cfg.Metrics, cfg.Debug, ctx)
}
}
}
// monitor monitors a single endpoint in a loop
// monitor a single endpoint in a loop
func monitor(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, disableMonitoringLock, enabledMetrics, debug bool, ctx context.Context) {
// Run it immediately on start
execute(endpoint, alertingConfig, maintenanceConfig, disableMonitoringLock, enabledMetrics, debug)
@ -88,7 +88,7 @@ func execute(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenan
// UpdateEndpointStatuses updates the slice of endpoint statuses
func UpdateEndpointStatuses(endpoint *core.Endpoint, result *core.Result) {
if err := storage.Get().Insert(endpoint, result); err != nil {
if err := store.Get().Insert(endpoint, result); err != nil {
log.Println("[watchdog][UpdateEndpointStatuses] Failed to insert data in storage:", err.Error())
}
}