Merge pull request #67 from cjheppell/split-memory-map-out-of-watchdog

Break watchdog into watchdog and storage (in-memory store)
This commit is contained in:
Chris C 2020-12-31 17:40:52 -05:00 committed by GitHub
commit 2b4fc7138a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 560 additions and 26 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
bin
.idea
.vscode
gatus

View File

@ -79,7 +79,7 @@ func serviceStatusesHandler(writer http.ResponseWriter, r *http.Request) {
var err error
buffer := &bytes.Buffer{}
gzipWriter := gzip.NewWriter(buffer)
data, err = watchdog.GetJSONEncodedServiceStatuses()
data, err = watchdog.GetServiceStatusesAsJSON()
if err != nil {
log.Printf("[main][serviceStatusesHandler] Unable to marshal object to JSON: %s", err.Error())
writer.WriteHeader(http.StatusInternalServerError)

113
storage/memory.go Normal file
View File

@ -0,0 +1,113 @@
package storage
import (
"fmt"
"sync"
"github.com/TwinProduction/gatus/core"
)
// InMemoryStore implements an in-memory store
type InMemoryStore struct {
serviceStatuses map[string]*core.ServiceStatus
serviceResultsMutex sync.RWMutex
}
// NewInMemoryStore returns an in-memory store. Note that the store acts as a singleton, so although new-ing
// up in-memory stores will give you a unique reference to a struct each time, all structs returned
// by this function will act on the same in-memory store.
func NewInMemoryStore() *InMemoryStore {
return &InMemoryStore{
serviceStatuses: make(map[string]*core.ServiceStatus),
}
}
// GetAll returns all the observed results for all services from the in memory store
func (ims *InMemoryStore) GetAll() map[string]*core.ServiceStatus {
results := make(map[string]*core.ServiceStatus, len(ims.serviceStatuses))
ims.serviceResultsMutex.RLock()
for key, svcStatus := range ims.serviceStatuses {
copiedResults := copyResults(svcStatus.Results)
results[key] = &core.ServiceStatus{
Name: svcStatus.Name,
Group: svcStatus.Group,
Results: copiedResults,
}
}
ims.serviceResultsMutex.RUnlock()
return results
}
// GetServiceStatus returns the service status for a given service name in the given group
func (ims *InMemoryStore) GetServiceStatus(group, name string) *core.ServiceStatus {
key := fmt.Sprintf("%s_%s", group, name)
ims.serviceResultsMutex.RLock()
serviceStatus := ims.serviceStatuses[key]
ims.serviceResultsMutex.RUnlock()
return serviceStatus
}
// Insert inserts the observed result for the specified service into the in memory store
func (ims *InMemoryStore) Insert(service *core.Service, result *core.Result) {
key := fmt.Sprintf("%s_%s", service.Group, service.Name)
ims.serviceResultsMutex.Lock()
serviceStatus, exists := ims.serviceStatuses[key]
if !exists {
serviceStatus = core.NewServiceStatus(service)
ims.serviceStatuses[key] = serviceStatus
}
serviceStatus.AddResult(result)
ims.serviceResultsMutex.Unlock()
}
func copyResults(results []*core.Result) []*core.Result {
copiedResults := []*core.Result{}
for _, result := range results {
copiedErrors := copyErrors(result.Errors)
copiedConditionResults := copyConditionResults(result.ConditionResults)
copiedResults = append(copiedResults, &core.Result{
HTTPStatus: result.HTTPStatus,
DNSRCode: result.DNSRCode,
Body: result.Body,
Hostname: result.Hostname,
IP: result.IP,
Connected: result.Connected,
Duration: result.Duration,
Errors: copiedErrors,
ConditionResults: copiedConditionResults,
Success: result.Connected,
Timestamp: result.Timestamp,
CertificateExpiration: result.CertificateExpiration,
})
}
return copiedResults
}
func copyConditionResults(crs []*core.ConditionResult) []*core.ConditionResult {
copiedConditionResults := []*core.ConditionResult{}
for _, conditionResult := range crs {
copiedConditionResults = append(copiedConditionResults, &core.ConditionResult{
Condition: conditionResult.Condition,
Success: conditionResult.Success,
})
}
return copiedConditionResults
}
func copyErrors(errors []string) []string {
copiedErrors := []string{}
for _, error := range errors {
copiedErrors = append(copiedErrors, error)
}
return copiedErrors
}
// Clear will empty all the results from the in memory store
func (ims *InMemoryStore) Clear() {
ims.serviceResultsMutex.Lock()
ims.serviceStatuses = make(map[string]*core.ServiceStatus)
ims.serviceResultsMutex.Unlock()
}

436
storage/memory_test.go Normal file
View File

@ -0,0 +1,436 @@
package storage
import (
"fmt"
"testing"
"time"
"github.com/TwinProduction/gatus/core"
)
var testService = core.Service{
Name: "Name",
Group: "Group",
URL: "URL",
DNS: &core.DNS{QueryType: "QueryType", QueryName: "QueryName"},
Method: "Method",
Body: "Body",
GraphQL: false,
Headers: nil,
Interval: time.Second * 2,
Conditions: nil,
Alerts: nil,
Insecure: false,
NumberOfFailuresInARow: 0,
NumberOfSuccessesInARow: 0,
}
var memoryStore = NewInMemoryStore()
func TestStorage_GetAllFromEmptyMemoryStoreReturnsNothing(t *testing.T) {
memoryStore.Clear()
results := memoryStore.GetAll()
if len(results) != 0 {
t.Errorf("MemoryStore should've returned 0 results, but actually returned %d", len(results))
}
}
func TestStorage_InsertIntoEmptyMemoryStoreThenGetAllReturnsOneResult(t *testing.T) {
memoryStore.Clear()
result := core.Result{
HTTPStatus: 200,
DNSRCode: "DNSRCode",
Body: nil,
Hostname: "Hostname",
IP: "IP",
Connected: false,
Duration: time.Second * 2,
Errors: nil,
ConditionResults: nil,
Success: false,
Timestamp: time.Now(),
CertificateExpiration: time.Second * 2,
}
memoryStore.Insert(&testService, &result)
results := memoryStore.GetAll()
if len(results) != 1 {
t.Errorf("MemoryStore should've returned 0 results, but actually returned %d", len(results))
}
key := fmt.Sprintf("%s_%s", testService.Group, testService.Name)
storedResult, exists := results[key]
if !exists {
t.Fatalf("In Memory Store should've contained key '%s', but didn't", key)
}
if storedResult.Name != testService.Name {
t.Errorf("Stored Results Name should've been %s, but was %s", testService.Name, storedResult.Name)
}
if storedResult.Group != testService.Group {
t.Errorf("Stored Results Group should've been %s, but was %s", testService.Group, storedResult.Group)
}
if len(storedResult.Results) != 1 {
t.Errorf("Stored Results for service %s should've had 1 result, but actually had %d", storedResult.Name, len(storedResult.Results))
}
if storedResult.Results[0] == &result {
t.Errorf("Returned result is the same reference as result passed to insert. Returned result should be copies only")
}
}
func TestStorage_InsertTwoResultsForSingleServiceIntoEmptyMemoryStore_ThenGetAllReturnsTwoResults(t *testing.T) {
memoryStore.Clear()
result1 := core.Result{
HTTPStatus: 404,
DNSRCode: "DNSRCode",
Body: nil,
Hostname: "Hostname",
IP: "IP",
Connected: false,
Duration: time.Second * 2,
Errors: nil,
ConditionResults: nil,
Success: false,
Timestamp: time.Now(),
CertificateExpiration: time.Second * 2,
}
result2 := core.Result{
HTTPStatus: 200,
DNSRCode: "DNSRCode",
Body: nil,
Hostname: "Hostname",
IP: "IP",
Connected: true,
Duration: time.Second * 2,
Errors: nil,
ConditionResults: nil,
Success: true,
Timestamp: time.Now(),
CertificateExpiration: time.Second * 2,
}
resultsToInsert := []core.Result{result1, result2}
memoryStore.Insert(&testService, &result1)
memoryStore.Insert(&testService, &result2)
results := memoryStore.GetAll()
if len(results) != 1 {
t.Fatalf("MemoryStore should've returned 1 results, but actually returned %d", len(results))
}
key := fmt.Sprintf("%s_%s", testService.Group, testService.Name)
serviceResults, exists := results[key]
if !exists {
t.Fatalf("In Memory Store should've contained key '%s', but didn't", key)
}
if len(serviceResults.Results) != 2 {
t.Fatalf("Service '%s' should've had 2 results, but actually returned %d", serviceResults.Name, len(serviceResults.Results))
}
for i, r := range serviceResults.Results {
expectedResult := resultsToInsert[i]
if r.HTTPStatus != expectedResult.HTTPStatus {
t.Errorf("Result at index %d should've had a HTTPStatus of %d, but was actually %d", i, expectedResult.HTTPStatus, r.HTTPStatus)
}
if r.DNSRCode != expectedResult.DNSRCode {
t.Errorf("Result at index %d should've had a DNSRCode of %s, but was actually %s", i, expectedResult.DNSRCode, r.DNSRCode)
}
if len(r.Body) != len(expectedResult.Body) {
t.Errorf("Result at index %d should've had a body of length %d, but was actually %d", i, len(expectedResult.Body), len(r.Body))
}
if r.Hostname != expectedResult.Hostname {
t.Errorf("Result at index %d should've had a Hostname of %s, but was actually %s", i, expectedResult.Hostname, r.Hostname)
}
if r.IP != expectedResult.IP {
t.Errorf("Result at index %d should've had a IP of %s, but was actually %s", i, expectedResult.IP, r.IP)
}
if r.Connected != expectedResult.Connected {
t.Errorf("Result at index %d should've had a Connected value of %t, but was actually %t", i, expectedResult.Connected, r.Connected)
}
if r.Duration != expectedResult.Duration {
t.Errorf("Result at index %d should've had a Duration of %s, but was actually %s", i, expectedResult.Duration.String(), r.Duration.String())
}
if len(r.Errors) != len(expectedResult.Errors) {
t.Errorf("Result at index %d should've had %d errors, but actually had %d errors", i, len(expectedResult.Errors), len(r.Errors))
}
if len(r.ConditionResults) != len(expectedResult.ConditionResults) {
t.Errorf("Result at index %d should've had %d ConditionResults, but actually had %d ConditionResults", i, len(expectedResult.ConditionResults), len(r.ConditionResults))
}
if r.Success != expectedResult.Success {
t.Errorf("Result at index %d should've had a Success of %t, but was actually %t", i, expectedResult.Success, r.Success)
}
if r.Timestamp != expectedResult.Timestamp {
t.Errorf("Result at index %d should've had a Timestamp of %s, but was actually %s", i, expectedResult.Timestamp.String(), r.Timestamp.String())
}
if r.CertificateExpiration != expectedResult.CertificateExpiration {
t.Errorf("Result at index %d should've had a CertificateExpiration of %s, but was actually %s", i, expectedResult.CertificateExpiration.String(), r.CertificateExpiration.String())
}
}
}
func TestStorage_InsertTwoResultsTwoServicesIntoEmptyMemoryStore_ThenGetAllReturnsTwoServicesWithOneResultEach(t *testing.T) {
memoryStore.Clear()
result1 := core.Result{
HTTPStatus: 404,
DNSRCode: "DNSRCode",
Body: nil,
Hostname: "Hostname",
IP: "IP",
Connected: false,
Duration: time.Second * 2,
Errors: nil,
ConditionResults: nil,
Success: false,
Timestamp: time.Now(),
CertificateExpiration: time.Second * 2,
}
result2 := core.Result{
HTTPStatus: 200,
DNSRCode: "DNSRCode",
Body: nil,
Hostname: "Hostname",
IP: "IP",
Connected: true,
Duration: time.Second * 2,
Errors: nil,
ConditionResults: nil,
Success: true,
Timestamp: time.Now(),
CertificateExpiration: time.Second * 2,
}
testService2 := core.Service{
Name: "Name2",
Group: "Group",
URL: "URL",
DNS: &core.DNS{QueryType: "QueryType", QueryName: "QueryName"},
Method: "Method",
Body: "Body",
GraphQL: false,
Headers: nil,
Interval: time.Second * 2,
Conditions: nil,
Alerts: nil,
Insecure: false,
NumberOfFailuresInARow: 0,
NumberOfSuccessesInARow: 0,
}
memoryStore.Insert(&testService, &result1)
memoryStore.Insert(&testService2, &result2)
results := memoryStore.GetAll()
if len(results) != 2 {
t.Fatalf("MemoryStore should've returned 2 results, but actually returned %d", len(results))
}
key := fmt.Sprintf("%s_%s", testService.Group, testService.Name)
serviceResults1, exists := results[key]
if !exists {
t.Fatalf("In Memory Store should've contained key '%s', but didn't", key)
}
if len(serviceResults1.Results) != 1 {
t.Fatalf("Service '%s' should've had 1 results, but actually returned %d", serviceResults1.Name, len(serviceResults1.Results))
}
key = fmt.Sprintf("%s_%s", testService2.Group, testService2.Name)
serviceResults2, exists := results[key]
if !exists {
t.Fatalf("In Memory Store should've contained key '%s', but didn't", key)
}
if len(serviceResults2.Results) != 1 {
t.Fatalf("Service '%s' should've had 1 results, but actually returned %d", serviceResults1.Name, len(serviceResults1.Results))
}
}
func TestStorage_InsertResultForServiceWithErrorsIntoEmptyMemoryStore_ThenGetAllReturnsOneResultWithErrors(t *testing.T) {
memoryStore.Clear()
errors := []string{
"error1",
"error2",
}
result1 := core.Result{
HTTPStatus: 404,
DNSRCode: "DNSRCode",
Body: nil,
Hostname: "Hostname",
IP: "IP",
Connected: false,
Duration: time.Second * 2,
Errors: errors,
ConditionResults: nil,
Success: false,
Timestamp: time.Now(),
CertificateExpiration: time.Second * 2,
}
memoryStore.Insert(&testService, &result1)
results := memoryStore.GetAll()
if len(results) != 1 {
t.Fatalf("MemoryStore should've returned 1 results, but actually returned %d", len(results))
}
key := fmt.Sprintf("%s_%s", testService.Group, testService.Name)
serviceResults, exists := results[key]
if !exists {
t.Fatalf("In Memory Store should've contained key '%s', but didn't", key)
}
if len(serviceResults.Results) != 1 {
t.Fatalf("Service '%s' should've had 2 results, but actually returned %d", serviceResults.Name, len(serviceResults.Results))
}
actualResult := serviceResults.Results[0]
if len(actualResult.Errors) != len(errors) {
t.Errorf("Service result should've had 2 errors, but actually had %d errors", len(actualResult.Errors))
}
for i, err := range actualResult.Errors {
if err != errors[i] {
t.Errorf("Error at index %d should've been %s, but was actually %s", i, errors[i], err)
}
}
}
func TestStorage_InsertResultForServiceWithConditionResultsIntoEmptyMemoryStore_ThenGetAllReturnsOneResultWithConditionResults(t *testing.T) {
memoryStore.Clear()
crs := []*core.ConditionResult{
{
Condition: "condition1",
Success: true,
},
{
Condition: "condition2",
Success: false,
},
}
result := core.Result{
HTTPStatus: 404,
DNSRCode: "DNSRCode",
Body: nil,
Hostname: "Hostname",
IP: "IP",
Connected: false,
Duration: time.Second * 2,
Errors: nil,
ConditionResults: crs,
Success: false,
Timestamp: time.Now(),
CertificateExpiration: time.Second * 2,
}
memoryStore.Insert(&testService, &result)
results := memoryStore.GetAll()
if len(results) != 1 {
t.Fatalf("MemoryStore should've returned 1 results, but actually returned %d", len(results))
}
key := fmt.Sprintf("%s_%s", testService.Group, testService.Name)
serviceResults, exists := results[key]
if !exists {
t.Fatalf("In Memory Store should've contained key '%s', but didn't", key)
}
if len(serviceResults.Results) != 1 {
t.Fatalf("Service '%s' should've had 2 results, but actually returned %d", serviceResults.Name, len(serviceResults.Results))
}
actualResult := serviceResults.Results[0]
if len(actualResult.ConditionResults) != len(crs) {
t.Errorf("Service result should've had 2 ConditionResults, but actually had %d ConditionResults", len(actualResult.Errors))
}
for i, cr := range actualResult.ConditionResults {
if cr.Condition != crs[i].Condition {
t.Errorf("ConditionResult at index %d should've had condition %s, but was actually %s", i, crs[i].Condition, cr.Condition)
}
if cr.Success != crs[i].Success {
t.Errorf("ConditionResult at index %d should've had success value of %t, but was actually %t", i, crs[i].Success, cr.Success)
}
}
}
func TestStorage_MultipleMemoryStoreInstancesReferToDifferentInternalMaps(t *testing.T) {
memoryStore.Clear()
currentMap := memoryStore.GetAll()
otherMemoryStore := NewInMemoryStore()
otherMemoryStoresMap := otherMemoryStore.GetAll()
if len(currentMap) != len(otherMemoryStoresMap) {
t.Errorf("Multiple memory stores should refer to the different internal maps, but 'memoryStore' returned %d results, and 'otherMemoryStore' returned %d results", len(currentMap), len(otherMemoryStoresMap))
}
memoryStore.Insert(&testService, &core.Result{})
currentMap = memoryStore.GetAll()
otherMemoryStoresMap = otherMemoryStore.GetAll()
if len(currentMap) == len(otherMemoryStoresMap) {
t.Errorf("Multiple memory stores should refer to different internal maps, but 'memoryStore' returned %d results after inserting, and 'otherMemoryStore' returned %d results after inserting", len(currentMap), len(otherMemoryStoresMap))
}
otherMemoryStore.Clear()
currentMap = memoryStore.GetAll()
otherMemoryStoresMap = otherMemoryStore.GetAll()
if len(currentMap) == len(otherMemoryStoresMap) {
t.Errorf("Multiple memory stores should refer to different internal maps, but 'memoryStore' returned %d results after clearing, and 'otherMemoryStore' returned %d results after clearing", len(currentMap), len(otherMemoryStoresMap))
}
}
func TestStorage_ModificationsToReturnedMapDoNotAffectInternalMap(t *testing.T) {
memoryStore.Clear()
memoryStore.Insert(&testService, &core.Result{})
modifiedResults := memoryStore.GetAll()
for k := range modifiedResults {
delete(modifiedResults, k)
}
results := memoryStore.GetAll()
if len(modifiedResults) == len(results) {
t.Errorf("Returned map from GetAll should be free to modify by the caller without affecting internal in-memory map, but length of results from in-memory map (%d) was equal to the length of results in modified map (%d)", len(results), len(modifiedResults))
}
}
func TestStorage_GetServiceStatusForExistingStatusReturnsThatServiceStatus(t *testing.T) {
memoryStore.Clear()
memoryStore.Insert(&testService, &core.Result{})
serviceStatus := memoryStore.GetServiceStatus(testService.Group, testService.Name)
if serviceStatus == nil {
t.Errorf("Returned service status for group '%s' and name '%s' was nil after inserting the service into the store", testService.Group, testService.Name)
}
}
func TestStorage_GetServiceStatusForMissingStatusReturnsNil(t *testing.T) {
memoryStore.Clear()
memoryStore.Insert(&testService, &core.Result{})
serviceStatus := memoryStore.GetServiceStatus("nonexistantgroup", "nonexistantname")
if serviceStatus != nil {
t.Errorf("Returned service status for group '%s' and name '%s' not nil after inserting the service into the store", testService.Group, testService.Name)
}
serviceStatus = memoryStore.GetServiceStatus(testService.Group, "nonexistantname")
if serviceStatus != nil {
t.Errorf("Returned service status for group '%s' and name '%s' not nil after inserting the service into the store", testService.Group, "nonexistantname")
}
serviceStatus = memoryStore.GetServiceStatus("nonexistantgroup", testService.Name)
if serviceStatus != nil {
t.Errorf("Returned service status for group '%s' and name '%s' not nil after inserting the service into the store", "nonexistantgroup", testService.Name)
}
}

View File

@ -10,35 +10,27 @@ import (
"github.com/TwinProduction/gatus/config"
"github.com/TwinProduction/gatus/core"
"github.com/TwinProduction/gatus/metric"
"github.com/TwinProduction/gatus/storage"
)
var (
serviceStatuses = make(map[string]*core.ServiceStatus)
// serviceStatusesMutex is used to prevent concurrent map access
serviceStatusesMutex sync.RWMutex
store = storage.NewInMemoryStore()
// monitoringMutex is used to prevent multiple services from being evaluated at the same time.
// Without this, conditions using response time may become inaccurate.
monitoringMutex sync.Mutex
)
// GetJSONEncodedServiceStatuses returns a list of core.ServiceStatus for each services encoded using json.Marshal.
// The reason why the encoding is done here is because we use a mutex to prevent concurrent map access.
func GetJSONEncodedServiceStatuses() ([]byte, error) {
serviceStatusesMutex.RLock()
data, err := json.Marshal(serviceStatuses)
serviceStatusesMutex.RUnlock()
return data, err
// GetServiceStatusesAsJSON returns a list of core.ServiceStatus for each services encoded using json.Marshal.
func GetServiceStatusesAsJSON() ([]byte, error) {
serviceStatuses := store.GetAll()
return json.Marshal(serviceStatuses)
}
// GetUptimeByServiceGroupAndName returns the uptime of a service based on its group and name
func GetUptimeByServiceGroupAndName(group, name string) *core.Uptime {
key := fmt.Sprintf("%s_%s", group, name)
serviceStatusesMutex.RLock()
serviceStatus, exists := serviceStatuses[key]
serviceStatusesMutex.RUnlock()
if !exists {
serviceStatus := store.GetServiceStatus(group, name)
if serviceStatus == nil {
return nil
}
return serviceStatus.Uptime
@ -93,13 +85,5 @@ func monitor(service *core.Service) {
// UpdateServiceStatuses updates the slice of service statuses
func UpdateServiceStatuses(service *core.Service, result *core.Result) {
key := fmt.Sprintf("%s_%s", service.Group, service.Name)
serviceStatusesMutex.Lock()
serviceStatus, exists := serviceStatuses[key]
if !exists {
serviceStatus = core.NewServiceStatus(service)
serviceStatuses[key] = serviceStatus
}
serviceStatus.AddResult(result)
serviceStatusesMutex.Unlock()
store.Insert(service, result)
}