From 38d3d1157102d90f20033355d63fc66b20e26ba7 Mon Sep 17 00:00:00 2001 From: Keith Carichner Jr Date: Wed, 19 Feb 2025 17:30:14 -0500 Subject: [PATCH 001/119] Attempting to add support for Pi-hole v6. --- docs/configuration.md | 10 +- internal/glance/widget-dns-stats.go | 142 ++++++++++++++++++++++++++-- 2 files changed, 144 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 01afc8e..58a20fd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1786,8 +1786,14 @@ Only required when using AdGuard Home. The username used to log into the admin d ##### `password` Only required when using AdGuard Home. The password used to log into the admin dashboard. Can be specified from an environment variable using the syntax `${VARIABLE_NAME}`. -##### `token` -Only required when using Pi-hole. The API token which can be found in `Settings -> API -> Show API token`. Can be specified from an environment variable using the syntax `${VARIABLE_NAME}`. +##### `token` (Deprecated) +Only required when using Pi-hole major version 5 or earlier. The API token which can be found in `Settings -> API -> Show API token`. Can be specified from an environment variable using the syntax `${VARIABLE_NAME}`. + +##### `app-password` +Only required when using Pi-hole. The App Password can be found in `Settings -> Web Interface / API -> Configure app password`. + +##### `version` +Only required if using an older version of PiHole (major version 5 or earlier). ##### `hide-graph` Whether to hide the graph showing the number of queries over time. diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index 833a80d..f32ab0d 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -1,17 +1,23 @@ package glance import ( + "bytes" "context" "encoding/json" "errors" "html/template" + "io" "log/slog" "net/http" + "os" "sort" "strings" "time" ) +// Global HTTP client for reuse +var httpClient = &http.Client{} + var dnsStatsWidgetTemplate = mustParseTemplate("dns-stats.html", "widget-base.html") type dnsStatsWidget struct { @@ -27,6 +33,8 @@ type dnsStatsWidget struct { AllowInsecure bool `yaml:"allow-insecure"` URL string `yaml:"url"` Token string `yaml:"token"` + AppPassword string `yaml:"app-password"` + PiHoleVersion string `yaml:"pihole-version"` Username string `yaml:"username"` Password string `yaml:"password"` } @@ -62,7 +70,7 @@ func (widget *dnsStatsWidget) update(ctx context.Context) { if widget.Service == "adguard" { stats, err = fetchAdguardStats(widget.URL, widget.AllowInsecure, widget.Username, widget.Password, widget.HideGraph) } else { - stats, err = fetchPiholeStats(widget.URL, widget.AllowInsecure, widget.Token, widget.HideGraph) + stats, err = fetchPiholeStats(widget.URL, widget.AllowInsecure, widget.Token, widget.HideGraph, widget.PiHoleVersion, widget.AppPassword) } if !widget.canContinueUpdateAfterHandlingErr(err) { @@ -275,13 +283,135 @@ func (p *piholeTopBlockedDomains) UnmarshalJSON(data []byte) error { return nil } -func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGraph bool) (*dnsStats, error) { - if token == "" { - return nil, errors.New("missing API token") +// piholeGetSID retrieves a new SID from Pi-hole using the app password. +func piholeGetSID(instanceURL, appPassword string) (string, error) { + requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth" + requestBody := []byte(`{"password":"` + appPassword + `"}`) + + request, err := http.NewRequest("POST", requestURL, bytes.NewBuffer(requestBody)) + if err != nil { + return "", errors.New("failed to create authentication request: " + err.Error()) + } + request.Header.Set("Content-Type", "application/json") + + response, err := httpClient.Do(request) + if err != nil { + return "", errors.New("failed to send authentication request: " + err.Error()) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return "", errors.New("authentication failed, received status: " + response.Status) } - requestURL := strings.TrimRight(instanceURL, "/") + - "/admin/api.php?summaryRaw&topItems&overTimeData10mins&auth=" + token + body, err := io.ReadAll(response.Body) + if err != nil { + return "", errors.New("failed to read authentication response: " + err.Error()) + } + + var jsonResponse struct { + Session struct { + SID string `json:"sid"` + } `json:"session"` + } + + if err := json.Unmarshal(body, &jsonResponse); err != nil { + return "", errors.New("failed to parse authentication response: " + err.Error()) + } + + if jsonResponse.Session.SID == "" { + return "", errors.New("authentication response did not contain a valid SID") + } + + return jsonResponse.Session.SID, nil +} + +// piholeCheckAndRefreshSID ensures the SID is valid, refreshing it if necessary. +func piholeCheckAndRefreshSID(instanceURL, appPassword string) (string, error) { + sid := os.Getenv("SID") + if sid == "" { + newSID, err := piholeGetSID(instanceURL, appPassword) + if err != nil { + return "", err + } + os.Setenv("SID", newSID) + return newSID, nil + } + + requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth?sid=" + sid + requestBody := []byte(`{"password":"` + appPassword + `"}`) + + request, err := http.NewRequest("GET", requestURL, bytes.NewBuffer(requestBody)) + if err != nil { + return "", errors.New("failed to create SID validation request: " + err.Error()) + } + request.Header.Set("Content-Type", "application/json") + + response, err := httpClient.Do(request) + if err != nil { + return "", errors.New("failed to send SID validation request: " + err.Error()) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + // Fetch a new SID if validation request fails + newSID, err := piholeGetSID(instanceURL, appPassword) + if err != nil { + return "", err + } + os.Setenv("SID", newSID) + return newSID, nil + } + + body, err := io.ReadAll(response.Body) + if err != nil { + return "", errors.New("failed to read SID validation response: " + err.Error()) + } + + var jsonResponse struct { + Session struct { + Valid bool `json:"valid"` + SID string `json:"sid"` + } `json:"session"` + } + + if err := json.Unmarshal(body, &jsonResponse); err != nil { + return "", errors.New("failed to parse SID validation response: " + err.Error()) + } + + if !jsonResponse.Session.Valid { + newSID, err := piholeGetSID(instanceURL, appPassword) + if err != nil { + return "", err + } + os.Setenv("SID", newSID) + return newSID, nil + } + + return sid, nil +} + +func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGraph bool, version, appPassword string) (*dnsStats, error) { + var requestURL string + + // Handle Pi-hole v6 authentication + if version == "" || version == "6" { + if appPassword == "" { + return nil, errors.New("missing app password") + } + + sid, err := piholeCheckAndRefreshSID(instanceURL, appPassword) + if err != nil { + return nil, err + } + + requestURL = strings.TrimRight(instanceURL, "/") + "/api/stats/summary?sid=" + sid + } else { + if token == "" { + return nil, errors.New("missing API token") + } + requestURL = strings.TrimRight(instanceURL, "/") + "/admin/api.php?summaryRaw&topItems&overTimeData10mins&auth=" + token + } request, err := http.NewRequest("GET", requestURL, nil) if err != nil { From 008c4a3ab7a3472abb186cb437fe178aa7e0cde4 Mon Sep 17 00:00:00 2001 From: Keith Carichner Jr Date: Wed, 19 Feb 2025 23:04:45 -0500 Subject: [PATCH 002/119] Removing SID check func. --- internal/glance/widget-dns-stats.go | 77 +++-------------------------- 1 file changed, 8 insertions(+), 69 deletions(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index f32ab0d..8944603 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -326,71 +326,6 @@ func piholeGetSID(instanceURL, appPassword string) (string, error) { return jsonResponse.Session.SID, nil } -// piholeCheckAndRefreshSID ensures the SID is valid, refreshing it if necessary. -func piholeCheckAndRefreshSID(instanceURL, appPassword string) (string, error) { - sid := os.Getenv("SID") - if sid == "" { - newSID, err := piholeGetSID(instanceURL, appPassword) - if err != nil { - return "", err - } - os.Setenv("SID", newSID) - return newSID, nil - } - - requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth?sid=" + sid - requestBody := []byte(`{"password":"` + appPassword + `"}`) - - request, err := http.NewRequest("GET", requestURL, bytes.NewBuffer(requestBody)) - if err != nil { - return "", errors.New("failed to create SID validation request: " + err.Error()) - } - request.Header.Set("Content-Type", "application/json") - - response, err := httpClient.Do(request) - if err != nil { - return "", errors.New("failed to send SID validation request: " + err.Error()) - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - // Fetch a new SID if validation request fails - newSID, err := piholeGetSID(instanceURL, appPassword) - if err != nil { - return "", err - } - os.Setenv("SID", newSID) - return newSID, nil - } - - body, err := io.ReadAll(response.Body) - if err != nil { - return "", errors.New("failed to read SID validation response: " + err.Error()) - } - - var jsonResponse struct { - Session struct { - Valid bool `json:"valid"` - SID string `json:"sid"` - } `json:"session"` - } - - if err := json.Unmarshal(body, &jsonResponse); err != nil { - return "", errors.New("failed to parse SID validation response: " + err.Error()) - } - - if !jsonResponse.Session.Valid { - newSID, err := piholeGetSID(instanceURL, appPassword) - if err != nil { - return "", err - } - os.Setenv("SID", newSID) - return newSID, nil - } - - return sid, nil -} - func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGraph bool, version, appPassword string) (*dnsStats, error) { var requestURL string @@ -399,11 +334,15 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr if appPassword == "" { return nil, errors.New("missing app password") } - - sid, err := piholeCheckAndRefreshSID(instanceURL, appPassword) - if err != nil { - return nil, err + // If SID env var is not set, get a new SID + if os.Getenv("SID") == "" { + sid, err := piholeGetSID(instanceURL, appPassword) + os.Setenv("SID", sid) + if err != nil { + return nil, err + } } + sid := os.Getenv("SID") requestURL = strings.TrimRight(instanceURL, "/") + "/api/stats/summary?sid=" + sid } else { From 72c1ebf66db960f185ca97d06340c590f713ce41 Mon Sep 17 00:00:00 2001 From: Keith Carichner Jr Date: Wed, 19 Feb 2025 23:23:22 -0500 Subject: [PATCH 003/119] Minor README adjustment. --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 58a20fd..a5404d7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1792,8 +1792,8 @@ Only required when using Pi-hole major version 5 or earlier. The API token which ##### `app-password` Only required when using Pi-hole. The App Password can be found in `Settings -> Web Interface / API -> Configure app password`. -##### `version` -Only required if using an older version of PiHole (major version 5 or earlier). +##### `pihole-version` +Only required if using an older version of Pi-hole (major version 5 or earlier). ##### `hide-graph` Whether to hide the graph showing the number of queries over time. From e643a44b590479fa1cf4c8217112886d11714a98 Mon Sep 17 00:00:00 2001 From: Keith Carichner Jr Date: Thu, 20 Feb 2025 14:54:11 -0500 Subject: [PATCH 004/119] Pushing latest fixes to handle the two different JSON responses. --- internal/glance/widget-dns-stats.go | 253 ++++++++++++++++++---------- 1 file changed, 166 insertions(+), 87 deletions(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index 8944603..73de4d9 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -7,7 +7,6 @@ import ( "errors" "html/template" "io" - "log/slog" "net/http" "os" "sort" @@ -15,9 +14,6 @@ import ( "time" ) -// Global HTTP client for reuse -var httpClient = &http.Client{} - var dnsStatsWidgetTemplate = mustParseTemplate("dns-stats.html", "widget-base.html") type dnsStatsWidget struct { @@ -235,7 +231,8 @@ func fetchAdguardStats(instanceURL string, allowInsecure bool, username, passwor return stats, nil } -type piholeStatsResponse struct { +// Legacy Pi-hole stats response (before v6) +type legacyPiholeStatsResponse struct { TotalQueries int `json:"dns_queries_today"` QueriesSeries piholeQueriesSeries `json:"domains_over_time"` BlockedQueries int `json:"ads_blocked_today"` @@ -245,6 +242,24 @@ type piholeStatsResponse struct { DomainsBlocked int `json:"domains_being_blocked"` } +// Pi-hole v6+ response format +type piholeStatsResponse struct { + Queries struct { + Total int `json:"total"` + Blocked int `json:"blocked"` + PercentBlocked float64 `json:"percent_blocked"` + } `json:"queries"` + Gravity struct { + DomainsBlocked int `json:"domains_being_blocked"` + } `json:"gravity"` + //Note we do not need the full structure. We extract the values needed + //Adding dummy fields to allow easier json parsing. + QueriesSeries piholeQueriesSeries `json:"domains_over_time"` // Will always be empty + BlockedSeries map[int64]int `json:"ads_over_time"` // Will always be empty. +} + +type piholeTopDomainsResponse map[string]int + // If the user has query logging disabled it's possible for domains_over_time to be returned as an // empty array rather than a map which will prevent unmashalling the rest of the data so we use // custom unmarshal behavior to fallback to an empty map. @@ -284,7 +299,14 @@ func (p *piholeTopBlockedDomains) UnmarshalJSON(data []byte) error { } // piholeGetSID retrieves a new SID from Pi-hole using the app password. -func piholeGetSID(instanceURL, appPassword string) (string, error) { +func piholeGetSID(instanceURL, appPassword string, allowInsecure bool) (string, error) { + var client requestDoer + if !allowInsecure { + client = defaultHTTPClient + } else { + client = defaultInsecureHTTPClient + } + requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth" requestBody := []byte(`{"password":"` + appPassword + `"}`) @@ -294,7 +316,7 @@ func piholeGetSID(instanceURL, appPassword string) (string, error) { } request.Header.Set("Content-Type", "application/json") - response, err := httpClient.Do(request) + response, err := client.Do(request) if err != nil { return "", errors.New("failed to send authentication request: " + err.Error()) } @@ -326,8 +348,121 @@ func piholeGetSID(instanceURL, appPassword string) (string, error) { return jsonResponse.Session.SID, nil } +// fetchPiholeTopDomains fetches the top blocked domains for Pi-hole v6+. +func fetchPiholeTopDomains(instanceURL string, sid string, allowInsecure bool) (piholeTopDomainsResponse, error) { + requestURL := strings.TrimRight(instanceURL, "/") + "/api/stats/top_domains?blocked=true&sid=" + sid + + request, err := http.NewRequest("GET", requestURL, nil) + if err != nil { + return nil, err + } + + var client requestDoer + if !allowInsecure { + client = defaultHTTPClient + } else { + client = defaultInsecureHTTPClient + } + + return decodeJsonFromRequest[piholeTopDomainsResponse](client, request) +} + +// Helper functions to process the responses +func parsePiholeStats(r *piholeStatsResponse, topDomains piholeTopDomainsResponse) *dnsStats { + + stats := &dnsStats{ + TotalQueries: r.Queries.Total, + BlockedQueries: r.Queries.Blocked, + BlockedPercent: int(r.Queries.PercentBlocked), + DomainsBlocked: r.Gravity.DomainsBlocked, + } + + if len(topDomains) > 0 { + domains := make([]dnsStatsBlockedDomain, 0, len(topDomains)) + for domain, count := range topDomains { + domains = append(domains, dnsStatsBlockedDomain{ + Domain: domain, + PercentBlocked: int(float64(count) / float64(r.Queries.Blocked) * 100), // Calculate percentage here + }) + } + + sort.Slice(domains, func(a, b int) bool { + return domains[a].PercentBlocked > domains[b].PercentBlocked + }) + stats.TopBlockedDomains = domains[:min(len(domains), 5)] + } + + return stats +} +func parsePiholeStatsLegacy(r *legacyPiholeStatsResponse, noGraph bool) *dnsStats { + + stats := &dnsStats{ + TotalQueries: r.TotalQueries, + BlockedQueries: r.BlockedQueries, + BlockedPercent: int(r.BlockedPercentage), + DomainsBlocked: r.DomainsBlocked, + } + if len(r.TopBlockedDomains) > 0 { + domains := make([]dnsStatsBlockedDomain, 0, len(r.TopBlockedDomains)) + + for domain, count := range r.TopBlockedDomains { + domains = append(domains, dnsStatsBlockedDomain{ + Domain: domain, + PercentBlocked: int(float64(count) / float64(r.BlockedQueries) * 100), + }) + } + + sort.Slice(domains, func(a, b int) bool { + return domains[a].PercentBlocked > domains[b].PercentBlocked + }) + + stats.TopBlockedDomains = domains[:min(len(domains), 5)] + } + if noGraph { + return stats + } + + // Pihole _should_ return data for the last 24 hours in a 10 minute interval, 6*24 = 144 + if len(r.QueriesSeries) != 144 || len(r.BlockedSeries) != 144 { + return stats + } + + var lowestTimestamp int64 = 0 + for timestamp := range r.QueriesSeries { + if lowestTimestamp == 0 || timestamp < lowestTimestamp { + lowestTimestamp = timestamp + } + } + maxQueriesInSeries := 0 + + for i := 0; i < 8; i++ { + queries := 0 + blocked := 0 + for j := 0; j < 18; j++ { + index := lowestTimestamp + int64(i*10800+j*600) + queries += r.QueriesSeries[index] + blocked += r.BlockedSeries[index] + } + if queries > maxQueriesInSeries { + maxQueriesInSeries = queries + } + stats.Series[i] = dnsStatsSeries{ + Queries: queries, + Blocked: blocked, + } + if queries > 0 { + stats.Series[i].PercentBlocked = int(float64(blocked) / float64(queries) * 100) + } + } + for i := 0; i < 8; i++ { + stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) + } + return stats +} + func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGraph bool, version, appPassword string) (*dnsStats, error) { var requestURL string + var sid string // Handle Pi-hole v6 authentication if version == "" || version == "6" { @@ -336,20 +471,22 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr } // If SID env var is not set, get a new SID if os.Getenv("SID") == "" { - sid, err := piholeGetSID(instanceURL, appPassword) - os.Setenv("SID", sid) + sid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) if err != nil { return nil, err } + os.Setenv("SID", sid) + } sid := os.Getenv("SID") - requestURL = strings.TrimRight(instanceURL, "/") + "/api/stats/summary?sid=" + sid + } else { if token == "" { return nil, errors.New("missing API token") } requestURL = strings.TrimRight(instanceURL, "/") + "/admin/api.php?summaryRaw&topItems&overTimeData10mins&auth=" + token + } request, err := http.NewRequest("GET", requestURL, nil) @@ -364,87 +501,29 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr client = defaultInsecureHTTPClient } - responseJson, err := decodeJsonFromRequest[piholeStatsResponse](client, request) + var responseJson interface{} + + if version == "" || version == "6" { + responseJson, err = decodeJsonFromRequest[piholeStatsResponse](client, request) + + } else { + responseJson, err = decodeJsonFromRequest[legacyPiholeStatsResponse](client, request) + } if err != nil { return nil, err } - stats := &dnsStats{ - TotalQueries: responseJson.TotalQueries, - BlockedQueries: responseJson.BlockedQueries, - BlockedPercent: int(responseJson.BlockedPercentage), - DomainsBlocked: responseJson.DomainsBlocked, - } - - if len(responseJson.TopBlockedDomains) > 0 { - domains := make([]dnsStatsBlockedDomain, 0, len(responseJson.TopBlockedDomains)) - - for domain, count := range responseJson.TopBlockedDomains { - domains = append(domains, dnsStatsBlockedDomain{ - Domain: domain, - PercentBlocked: int(float64(count) / float64(responseJson.BlockedQueries) * 100), - }) + switch r := responseJson.(type) { + case *piholeStatsResponse: + // Fetch top domains separately for v6 + topDomains, err := fetchPiholeTopDomains(instanceURL, sid, allowInsecure) + if err != nil { + return nil, err } - - sort.Slice(domains, func(a, b int) bool { - return domains[a].PercentBlocked > domains[b].PercentBlocked - }) - - stats.TopBlockedDomains = domains[:min(len(domains), 5)] + return parsePiholeStats(r, topDomains), nil + case *legacyPiholeStatsResponse: + return parsePiholeStatsLegacy(r, noGraph), nil + default: + return nil, errors.New("unexpected response type") } - - if noGraph { - return stats, nil - } - - // Pihole _should_ return data for the last 24 hours in a 10 minute interval, 6*24 = 144 - if len(responseJson.QueriesSeries) != 144 || len(responseJson.BlockedSeries) != 144 { - slog.Warn( - "DNS stats for pihole: did not get expected 144 data points", - "len(queries)", len(responseJson.QueriesSeries), - "len(blocked)", len(responseJson.BlockedSeries), - ) - return stats, nil - } - - var lowestTimestamp int64 = 0 - - for timestamp := range responseJson.QueriesSeries { - if lowestTimestamp == 0 || timestamp < lowestTimestamp { - lowestTimestamp = timestamp - } - } - - maxQueriesInSeries := 0 - - for i := 0; i < 8; i++ { - queries := 0 - blocked := 0 - - for j := 0; j < 18; j++ { - index := lowestTimestamp + int64(i*10800+j*600) - - queries += responseJson.QueriesSeries[index] - blocked += responseJson.BlockedSeries[index] - } - - if queries > maxQueriesInSeries { - maxQueriesInSeries = queries - } - - stats.Series[i] = dnsStatsSeries{ - Queries: queries, - Blocked: blocked, - } - - if queries > 0 { - stats.Series[i].PercentBlocked = int(float64(blocked) / float64(queries) * 100) - } - } - - for i := 0; i < 8; i++ { - stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) - } - - return stats, nil } From 2002ed1c9c3ca11e22e26dc83992bc0d2161b78c Mon Sep 17 00:00:00 2001 From: Keith Carichner Jr Date: Mon, 24 Feb 2025 13:52:27 -0500 Subject: [PATCH 005/119] Pushing latest changes. --- internal/glance/widget-dns-stats.go | 116 ++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 23 deletions(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index 73de4d9..cbb9b11 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "html/template" "io" "net/http" @@ -348,6 +349,35 @@ func piholeGetSID(instanceURL, appPassword string, allowInsecure bool) (string, return jsonResponse.Session.SID, nil } +// checkPiholeSID checks if the SID is valid by checking HTTP response status code from /api/auth. +func checkPiholeSID(instanceURL string, appPassword, sid string, allowInsecure bool) error { + requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth?sid=" + sid + + request, err := http.NewRequest("GET", requestURL, nil) + if err != nil { + return err + } + + var client requestDoer + if !allowInsecure { + client = defaultHTTPClient + } else { + client = defaultInsecureHTTPClient + } + + response, err := client.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return errors.New("SID is invalid, received status: " + response.Status) + } + + return nil +} + // fetchPiholeTopDomains fetches the top blocked domains for Pi-hole v6+. func fetchPiholeTopDomains(instanceURL string, sid string, allowInsecure bool) (piholeTopDomainsResponse, error) { requestURL := strings.TrimRight(instanceURL, "/") + "/api/stats/top_domains?blocked=true&sid=" + sid @@ -367,6 +397,35 @@ func fetchPiholeTopDomains(instanceURL string, sid string, allowInsecure bool) ( return decodeJsonFromRequest[piholeTopDomainsResponse](client, request) } +// fetchPiholeSeries fetches the series data for Pi-hole v6+ (QueriesSeries and BlockedSeries). +func fetchPiholeSeries(instanceURL string, sid string, allowInsecure bool) (piholeQueriesSeries, map[int64]int, error) { + requestURL := strings.TrimRight(instanceURL, "/") + "/api/stats/over_time_data?sid=" + sid + + request, err := http.NewRequest("GET", requestURL, nil) + if err != nil { + return nil, nil, err + } + + var client requestDoer + if !allowInsecure { + client = defaultHTTPClient + } else { + client = defaultInsecureHTTPClient + } + + var responseJson struct { + QueriesSeries piholeQueriesSeries `json:"queries_over_time"` + BlockedSeries map[int64]int `json:"blocked_over_time"` + } + + err = decodeJsonFromRequest[&responseJson](client, request) + if err != nil { + return nil, nil, err + } + + return responseJson.QueriesSeries, responseJson.BlockedSeries, nil +} + // Helper functions to process the responses func parsePiholeStats(r *piholeStatsResponse, topDomains piholeTopDomainsResponse) *dnsStats { @@ -461,64 +520,75 @@ func parsePiholeStatsLegacy(r *legacyPiholeStatsResponse, noGraph bool) *dnsStat } func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGraph bool, version, appPassword string) (*dnsStats, error) { + instanceURL = strings.TrimRight(instanceURL, "/") var requestURL string var sid string + isV6 := version == "" || version == "6" - // Handle Pi-hole v6 authentication - if version == "" || version == "6" { + if isV6 { if appPassword == "" { return nil, errors.New("missing app password") } - // If SID env var is not set, get a new SID - if os.Getenv("SID") == "" { - sid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) - if err != nil { - return nil, err - } - os.Setenv("SID", sid) + sid = os.Getenv("SID") + // Only get a new SID if it's not set or is invalid + if sid == "" { + newSid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) + if err != nil { + return nil, fmt.Errorf("failed to get SID: %w", err) // Use %w for wrapping + } + sid = newSid + os.Setenv("SID", sid) + } else { + // Check existing SID validity. Only get a new one if the check fails. + err := checkPiholeSID(instanceURL, appPassword, sid, allowInsecure) + if err != nil { + newSid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) + if err != nil { + return nil, fmt.Errorf("failed to get SID after invalid SID check: %w", err) + } + sid = newSid + os.Setenv("SID", sid) + } } - sid := os.Getenv("SID") - requestURL = strings.TrimRight(instanceURL, "/") + "/api/stats/summary?sid=" + sid + + requestURL = instanceURL + "/api/stats/summary?sid=" + sid } else { if token == "" { return nil, errors.New("missing API token") } - requestURL = strings.TrimRight(instanceURL, "/") + "/admin/api.php?summaryRaw&topItems&overTimeData10mins&auth=" + token - + requestURL = instanceURL + "/admin/api.php?summaryRaw&topItems&overTimeData10mins&auth=" + token } request, err := http.NewRequest("GET", requestURL, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create HTTP request: %w", err) } var client requestDoer - if !allowInsecure { - client = defaultHTTPClient - } else { + client = defaultHTTPClient + if allowInsecure { client = defaultInsecureHTTPClient } var responseJson interface{} - - if version == "" || version == "6" { + if isV6 { responseJson, err = decodeJsonFromRequest[piholeStatsResponse](client, request) - } else { responseJson, err = decodeJsonFromRequest[legacyPiholeStatsResponse](client, request) } + if err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode JSON response: %w", err) } switch r := responseJson.(type) { case *piholeStatsResponse: - // Fetch top domains separately for v6 + // Fetch top domains separately for v6+. topDomains, err := fetchPiholeTopDomains(instanceURL, sid, allowInsecure) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to fetch top domains: %w", err) } return parsePiholeStats(r, topDomains), nil case *legacyPiholeStatsResponse: From 65a412eb5995fe612cc38e715d077f4e48e77737 Mon Sep 17 00:00:00 2001 From: Keith Carichner Jr Date: Wed, 26 Feb 2025 16:20:43 -0500 Subject: [PATCH 006/119] Pushing latest. Add ability to fetch series data from new api. --- internal/glance/widget-dns-stats.go | 54 +++++++++++++++++++++-------- internal/glance/widget-utils.go | 10 ++++++ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index cbb9b11..176c284 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -350,7 +350,7 @@ func piholeGetSID(instanceURL, appPassword string, allowInsecure bool) (string, } // checkPiholeSID checks if the SID is valid by checking HTTP response status code from /api/auth. -func checkPiholeSID(instanceURL string, appPassword, sid string, allowInsecure bool) error { +func checkPiholeSID(instanceURL string, sid string, allowInsecure bool) error { requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth?sid=" + sid request, err := http.NewRequest("GET", requestURL, nil) @@ -399,7 +399,7 @@ func fetchPiholeTopDomains(instanceURL string, sid string, allowInsecure bool) ( // fetchPiholeSeries fetches the series data for Pi-hole v6+ (QueriesSeries and BlockedSeries). func fetchPiholeSeries(instanceURL string, sid string, allowInsecure bool) (piholeQueriesSeries, map[int64]int, error) { - requestURL := strings.TrimRight(instanceURL, "/") + "/api/stats/over_time_data?sid=" + sid + requestURL := strings.TrimRight(instanceURL, "/") + "/api/history?sid=" + sid request, err := http.NewRequest("GET", requestURL, nil) if err != nil { @@ -413,17 +413,30 @@ func fetchPiholeSeries(instanceURL string, sid string, allowInsecure bool) (piho client = defaultInsecureHTTPClient } + // Define the correct struct to match the API response var responseJson struct { - QueriesSeries piholeQueriesSeries `json:"queries_over_time"` - BlockedSeries map[int64]int `json:"blocked_over_time"` + History []struct { + Timestamp int64 `json:"timestamp"` + Total int `json:"total"` + Blocked int `json:"blocked"` + } `json:"history"` } - err = decodeJsonFromRequest[&responseJson](client, request) + err = decodeJsonInto(client, request, &responseJson) if err != nil { return nil, nil, err } - return responseJson.QueriesSeries, responseJson.BlockedSeries, nil + queriesSeries := make(piholeQueriesSeries) + blockedSeries := make(map[int64]int) + + // Populate the series data from history array + for _, entry := range responseJson.History { + queriesSeries[entry.Timestamp] = entry.Total + blockedSeries[entry.Timestamp] = entry.Blocked + } + + return queriesSeries, blockedSeries, nil } // Helper functions to process the responses @@ -531,21 +544,19 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr } sid = os.Getenv("SID") - // Only get a new SID if it's not set or is invalid if sid == "" { newSid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) if err != nil { - return nil, fmt.Errorf("failed to get SID: %w", err) // Use %w for wrapping + return nil, fmt.Errorf("failed to get SID: %w", err) } sid = newSid os.Setenv("SID", sid) } else { - // Check existing SID validity. Only get a new one if the check fails. - err := checkPiholeSID(instanceURL, appPassword, sid, allowInsecure) + err := checkPiholeSID(instanceURL, sid, allowInsecure) if err != nil { newSid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) if err != nil { - return nil, fmt.Errorf("failed to get SID after invalid SID check: %w", err) + return nil, fmt.Errorf("failed to get SID after invalid check: %w", err) } sid = newSid os.Setenv("SID", sid) @@ -553,7 +564,6 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr } requestURL = instanceURL + "/api/stats/summary?sid=" + sid - } else { if token == "" { return nil, errors.New("missing API token") @@ -567,8 +577,9 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr } var client requestDoer - client = defaultHTTPClient - if allowInsecure { + if !allowInsecure { + client = defaultHTTPClient + } else { client = defaultInsecureHTTPClient } @@ -585,14 +596,27 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr switch r := responseJson.(type) { case *piholeStatsResponse: - // Fetch top domains separately for v6+. + // Fetch top domains separately for v6+ topDomains, err := fetchPiholeTopDomains(instanceURL, sid, allowInsecure) if err != nil { return nil, fmt.Errorf("failed to fetch top domains: %w", err) } + + // Fetch series data separately for v6+ + queriesSeries, blockedSeries, err := fetchPiholeSeries(instanceURL, sid, allowInsecure) + if err != nil { + return nil, fmt.Errorf("failed to fetch queries series: %w", err) + } + + // Merge series data + r.QueriesSeries = queriesSeries + r.BlockedSeries = blockedSeries + return parsePiholeStats(r, topDomains), nil + case *legacyPiholeStatsResponse: return parsePiholeStatsLegacy(r, noGraph), nil + default: return nil, errors.New("unexpected response type") } diff --git a/internal/glance/widget-utils.go b/internal/glance/widget-utils.go index 8fb76dd..fa141c1 100644 --- a/internal/glance/widget-utils.go +++ b/internal/glance/widget-utils.go @@ -82,6 +82,16 @@ func decodeJsonFromRequest[T any](client requestDoer, request *http.Request) (T, return result, nil } +func decodeJsonInto[T any](client requestDoer, request *http.Request, out *T) error { + result, err := decodeJsonFromRequest[T](client, request) + if err != nil { + return err + } + + *out = result + return nil +} + func decodeJsonFromRequestTask[T any](client requestDoer) func(*http.Request) (T, error) { return func(request *http.Request) (T, error) { return decodeJsonFromRequest[T](client, request) From 99660aeee8728ce90ca2b0c5a2ae89fb2501a5f7 Mon Sep 17 00:00:00 2001 From: Ralph Ocdol Date: Thu, 27 Feb 2025 23:02:10 +0800 Subject: [PATCH 007/119] Fix Graph and move SID to header --- internal/glance/widget-dns-stats.go | 92 +++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index 176c284..de25675 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -259,7 +259,17 @@ type piholeStatsResponse struct { BlockedSeries map[int64]int `json:"ads_over_time"` // Will always be empty. } -type piholeTopDomainsResponse map[string]int +type piholeTopDomainsResponse struct { + Domains []Domains `json:"domains"` + TotalQueries int `json:"total_queries"` + BlockedQueries int `json:"blocked_queries"` + Took float64 `json:"took"` +} + +type Domains struct { + Domain string `json:"domain"` + Count int `json:"count"` +} // If the user has query logging disabled it's possible for domains_over_time to be returned as an // empty array rather than a map which will prevent unmashalling the rest of the data so we use @@ -351,12 +361,13 @@ func piholeGetSID(instanceURL, appPassword string, allowInsecure bool) (string, // checkPiholeSID checks if the SID is valid by checking HTTP response status code from /api/auth. func checkPiholeSID(instanceURL string, sid string, allowInsecure bool) error { - requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth?sid=" + sid + requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth" request, err := http.NewRequest("GET", requestURL, nil) if err != nil { return err } + request.Header.Set("x-ftl-sid", sid) var client requestDoer if !allowInsecure { @@ -380,12 +391,13 @@ func checkPiholeSID(instanceURL string, sid string, allowInsecure bool) error { // fetchPiholeTopDomains fetches the top blocked domains for Pi-hole v6+. func fetchPiholeTopDomains(instanceURL string, sid string, allowInsecure bool) (piholeTopDomainsResponse, error) { - requestURL := strings.TrimRight(instanceURL, "/") + "/api/stats/top_domains?blocked=true&sid=" + sid + requestURL := strings.TrimRight(instanceURL, "/") + "/api/stats/top_domains?blocked=true" request, err := http.NewRequest("GET", requestURL, nil) if err != nil { - return nil, err + return piholeTopDomainsResponse{}, err } + request.Header.Set("x-ftl-sid", sid) var client requestDoer if !allowInsecure { @@ -399,12 +411,13 @@ func fetchPiholeTopDomains(instanceURL string, sid string, allowInsecure bool) ( // fetchPiholeSeries fetches the series data for Pi-hole v6+ (QueriesSeries and BlockedSeries). func fetchPiholeSeries(instanceURL string, sid string, allowInsecure bool) (piholeQueriesSeries, map[int64]int, error) { - requestURL := strings.TrimRight(instanceURL, "/") + "/api/history?sid=" + sid + requestURL := strings.TrimRight(instanceURL, "/") + "/api/history" request, err := http.NewRequest("GET", requestURL, nil) if err != nil { return nil, nil, err } + request.Header.Set("x-ftl-sid", sid) var client requestDoer if !allowInsecure { @@ -440,7 +453,7 @@ func fetchPiholeSeries(instanceURL string, sid string, allowInsecure bool) (piho } // Helper functions to process the responses -func parsePiholeStats(r *piholeStatsResponse, topDomains piholeTopDomainsResponse) *dnsStats { +func parsePiholeStats(r piholeStatsResponse, topDomains piholeTopDomainsResponse, noGraph bool) *dnsStats { stats := &dnsStats{ TotalQueries: r.Queries.Total, @@ -449,12 +462,12 @@ func parsePiholeStats(r *piholeStatsResponse, topDomains piholeTopDomainsRespons DomainsBlocked: r.Gravity.DomainsBlocked, } - if len(topDomains) > 0 { - domains := make([]dnsStatsBlockedDomain, 0, len(topDomains)) - for domain, count := range topDomains { + if len(topDomains.Domains) > 0 { + domains := make([]dnsStatsBlockedDomain, 0, len(topDomains.Domains)) + for _, d := range topDomains.Domains { domains = append(domains, dnsStatsBlockedDomain{ - Domain: domain, - PercentBlocked: int(float64(count) / float64(r.Queries.Blocked) * 100), // Calculate percentage here + Domain: d.Domain, + PercentBlocked: int(float64(d.Count) / float64(r.Queries.Blocked) * 100), }) } @@ -463,10 +476,49 @@ func parsePiholeStats(r *piholeStatsResponse, topDomains piholeTopDomainsRespons }) stats.TopBlockedDomains = domains[:min(len(domains), 5)] } + if noGraph { + return stats + } + // Pihole _should_ return data for the last 24 hours + if len(r.QueriesSeries) != 145 || len(r.BlockedSeries) != 145 { + return stats + } + + + var lowestTimestamp int64 = 0 + for timestamp := range r.QueriesSeries { + if lowestTimestamp == 0 || timestamp < lowestTimestamp { + lowestTimestamp = timestamp + } + } + maxQueriesInSeries := 0 + + for i := 0; i < 8; i++ { + queries := 0 + blocked := 0 + for j := 0; j < 18; j++ { + index := lowestTimestamp + int64(i*10800+j*600) + queries += r.QueriesSeries[index] + blocked += r.BlockedSeries[index] + } + if queries > maxQueriesInSeries { + maxQueriesInSeries = queries + } + stats.Series[i] = dnsStatsSeries{ + Queries: queries, + Blocked: blocked, + } + if queries > 0 { + stats.Series[i].PercentBlocked = int(float64(blocked) / float64(queries) * 100) + } + } + for i := 0; i < 8; i++ { + stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) + } return stats } -func parsePiholeStatsLegacy(r *legacyPiholeStatsResponse, noGraph bool) *dnsStats { +func parsePiholeStatsLegacy(r legacyPiholeStatsResponse, noGraph bool) *dnsStats { stats := &dnsStats{ TotalQueries: r.TotalQueries, @@ -563,7 +615,7 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr } } - requestURL = instanceURL + "/api/stats/summary?sid=" + sid + requestURL = instanceURL + "/api/stats/summary" } else { if token == "" { return nil, errors.New("missing API token") @@ -576,6 +628,10 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr return nil, fmt.Errorf("failed to create HTTP request: %w", err) } + if isV6 { + request.Header.Set("x-ftl-sid", sid) + } + var client requestDoer if !allowInsecure { client = defaultHTTPClient @@ -595,7 +651,7 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr } switch r := responseJson.(type) { - case *piholeStatsResponse: + case piholeStatsResponse: // Fetch top domains separately for v6+ topDomains, err := fetchPiholeTopDomains(instanceURL, sid, allowInsecure) if err != nil { @@ -611,13 +667,13 @@ func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGr // Merge series data r.QueriesSeries = queriesSeries r.BlockedSeries = blockedSeries + + return parsePiholeStats(r, topDomains, noGraph), nil - return parsePiholeStats(r, topDomains), nil - - case *legacyPiholeStatsResponse: + case legacyPiholeStatsResponse: return parsePiholeStatsLegacy(r, noGraph), nil default: return nil, errors.New("unexpected response type") } -} +} \ No newline at end of file From 3b79c8e09fc9d3056e978006d7989e0e1f70c6bc Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sat, 15 Mar 2025 10:27:39 +0000 Subject: [PATCH 008/119] Remove symbol-link-template --- docs/glance.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/glance.yml b/docs/glance.yml index 92b4de1..35dc7cb 100644 --- a/docs/glance.yml +++ b/docs/glance.yml @@ -66,9 +66,6 @@ pages: # hide-location: true - type: markets - # The link to go to when clicking on the symbol in the UI, - # {SYMBOL} will be substituded with the symbol for each market - symbol-link-template: https://www.tradingview.com/symbols/{SYMBOL}/news markets: - symbol: SPY name: S&P 500 From 4b2438c298c76aa88e7fed6d5f494fcc05378d05 Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sat, 15 Mar 2025 18:42:58 +0000 Subject: [PATCH 009/119] Refactor DNS stats widget --- docs/configuration.md | 20 +- internal/glance/static/main.css | 1 + internal/glance/templates/dns-stats.html | 4 +- internal/glance/utils.go | 4 + internal/glance/widget-dns-stats.go | 770 +++++++++++------------ internal/glance/widget-utils.go | 10 - 6 files changed, 394 insertions(+), 415 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a5404d7..140be04 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1765,35 +1765,31 @@ Preview: | allow-insecure | bool | no | false | | url | string | yes | | | username | string | when service is `adguard` | | -| password | string | when service is `adguard` | | +| password | string | when service is `adguard` or `pihole6` | | | token | string | when service is `pihole` | | | hide-graph | bool | no | false | | hide-top-domains | bool | no | false | | hour-format | string | no | 12h | ##### `service` -Either `adguard` or `pihole`. +Either `adguard`, or `pihole` (major version 5 and below) or `pihole6` (major version 6 and above). ##### `allow-insecure` Whether to allow invalid/self-signed certificates when making the request to the service. ##### `url` -The base URL of the service. Can be specified from an environment variable using the syntax `${VARIABLE_NAME}`. +The base URL of the service. ##### `username` -Only required when using AdGuard Home. The username used to log into the admin dashboard. Can be specified from an environment variable using the syntax `${VARIABLE_NAME}`. +Only required when using AdGuard Home. The username used to log into the admin dashboard. ##### `password` -Only required when using AdGuard Home. The password used to log into the admin dashboard. Can be specified from an environment variable using the syntax `${VARIABLE_NAME}`. +Required when using AdGuard Home, where the password is the one used to log into the admin dashboard. -##### `token` (Deprecated) -Only required when using Pi-hole major version 5 or earlier. The API token which can be found in `Settings -> API -> Show API token`. Can be specified from an environment variable using the syntax `${VARIABLE_NAME}`. +Also requried when using Pi-hole major version 6 and above, where the password is the one used to log into the admin dashboard or the application password, which can be found in `Settings -> Web Interface / API -> Configure app password`. -##### `app-password` -Only required when using Pi-hole. The App Password can be found in `Settings -> Web Interface / API -> Configure app password`. - -##### `pihole-version` -Only required if using an older version of Pi-hole (major version 5 or earlier). +##### `token` +Only required when using Pi-hole major version 5 or earlier. The API token which can be found in `Settings -> API -> Show API token`. ##### `hide-graph` Whether to hide the graph showing the number of queries over time. diff --git a/internal/glance/static/main.css b/internal/glance/static/main.css index a271d4a..33f1e78 100644 --- a/internal/glance/static/main.css +++ b/internal/glance/static/main.css @@ -1256,6 +1256,7 @@ details[open] .summary::after { .dns-stats-graph-bar > .blocked { background-color: var(--color-negative); + flex-basis: calc(var(--percent) - 1px); } .dns-stats-graph-column:nth-child(even) .dns-stats-graph-time { diff --git a/internal/glance/templates/dns-stats.html b/internal/glance/templates/dns-stats.html index 8128edf..f2c6358 100644 --- a/internal/glance/templates/dns-stats.html +++ b/internal/glance/templates/dns-stats.html @@ -59,8 +59,8 @@ {{ if ne $column.Queries $column.Blocked }}
{{ end }} - {{ if or (gt $column.Blocked 0) (and (lt $column.PercentTotal 15) (lt $column.PercentBlocked 10)) }} -
+ {{ if gt $column.PercentBlocked 0 }} +
{{ end }} {{ end }} diff --git a/internal/glance/utils.go b/internal/glance/utils.go index 8455bfe..72d5a28 100644 --- a/internal/glance/utils.go +++ b/internal/glance/utils.go @@ -186,3 +186,7 @@ func ternary[T any](condition bool, a, b T) T { return b } + +// Having compile time errors about unused variables is cool and all, but I don't want to +// have to constantly comment out my code while I'm working on it and testing things out +func ItsUsedTrustMeBro(...any) {} diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index de25675..b8d6447 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -8,20 +8,28 @@ import ( "fmt" "html/template" "io" + "log/slog" "net/http" - "os" "sort" "strings" + "sync" "time" ) var dnsStatsWidgetTemplate = mustParseTemplate("dns-stats.html", "widget-base.html") +const ( + dnsStatsBars = 8 + dnsStatsHoursSpan = 24 + dnsStatsHoursPerBar int = dnsStatsHoursSpan / dnsStatsBars +) + type dnsStatsWidget struct { widgetBase `yaml:",inline"` - TimeLabels [8]string `yaml:"-"` - Stats *dnsStats `yaml:"-"` + TimeLabels [8]string `yaml:"-"` + Stats *dnsStats `yaml:"-"` + piholeSessionID string `yaml:"-"` HourFormat string `yaml:"hour-format"` HideGraph bool `yaml:"hide-graph"` @@ -30,17 +38,15 @@ type dnsStatsWidget struct { AllowInsecure bool `yaml:"allow-insecure"` URL string `yaml:"url"` Token string `yaml:"token"` - AppPassword string `yaml:"app-password"` - PiHoleVersion string `yaml:"pihole-version"` Username string `yaml:"username"` Password string `yaml:"password"` } func makeDNSWidgetTimeLabels(format string) [8]string { now := time.Now() - var labels [8]string + var labels [dnsStatsBars]string - for h := 24; h > 0; h -= 3 { + for h := dnsStatsHoursSpan; h > 0; h -= dnsStatsHoursPerBar { labels[7-(h/3-1)] = strings.ToLower(now.Add(-time.Duration(h) * time.Hour).Format(format)) } @@ -53,8 +59,12 @@ func (widget *dnsStatsWidget) initialize() error { withTitleURL(string(widget.URL)). withCacheDuration(10 * time.Minute) - if widget.Service != "adguard" && widget.Service != "pihole" { - return errors.New("service must be either 'adguard' or 'pihole'") + switch widget.Service { + case "adguard": + case "pihole6": + case "pihole": + default: + return errors.New("service must be one of: adguard, pihole6, pihole") } return nil @@ -64,10 +74,24 @@ func (widget *dnsStatsWidget) update(ctx context.Context) { var stats *dnsStats var err error - if widget.Service == "adguard" { + switch widget.Service { + case "adguard": stats, err = fetchAdguardStats(widget.URL, widget.AllowInsecure, widget.Username, widget.Password, widget.HideGraph) - } else { - stats, err = fetchPiholeStats(widget.URL, widget.AllowInsecure, widget.Token, widget.HideGraph, widget.PiHoleVersion, widget.AppPassword) + case "pihole": + stats, err = fetchPihole5Stats(widget.URL, widget.AllowInsecure, widget.Token, widget.HideGraph) + case "pihole6": + var newSessionID string + stats, newSessionID, err = fetchPiholeStats( + widget.URL, + widget.AllowInsecure, + widget.Password, + widget.piholeSessionID, + !widget.HideGraph, + !widget.HideTopDomains, + ) + if err == nil { + widget.piholeSessionID = newSessionID + } } if !widget.canContinueUpdateAfterHandlingErr(err) { @@ -89,11 +113,11 @@ func (widget *dnsStatsWidget) Render() template.HTML { type dnsStats struct { TotalQueries int - BlockedQueries int + BlockedQueries int // we don't actually use this anywhere in templates, maybe remove it later? BlockedPercent int ResponseTime int DomainsBlocked int - Series [8]dnsStatsSeries + Series [dnsStatsBars]dnsStatsSeries TopBlockedDomains []dnsStatsBlockedDomain } @@ -128,13 +152,7 @@ func fetchAdguardStats(instanceURL string, allowInsecure bool, username, passwor request.SetBasicAuth(username, password) - var client requestDoer - if !allowInsecure { - client = defaultHTTPClient - } else { - client = defaultInsecureHTTPClient - } - + var client = ternary(allowInsecure, defaultInsecureHTTPClient, defaultHTTPClient) responseJson, err := decodeJsonFromRequest[adguardStatsResponse](client, request) if err != nil { return nil, err @@ -155,7 +173,7 @@ func fetchAdguardStats(instanceURL string, allowInsecure bool, username, passwor stats.BlockedPercent = int(float64(responseJson.BlockedQueries) / float64(responseJson.TotalQueries) * 100) - for i := 0; i < topBlockedDomainsCount; i++ { + for i := range topBlockedDomainsCount { domain := responseJson.TopBlockedDomains[i] var firstDomain string @@ -184,31 +202,27 @@ func fetchAdguardStats(instanceURL string, allowInsecure bool, username, passwor queriesSeries := responseJson.QueriesSeries blockedSeries := responseJson.BlockedSeries - const bars = 8 - const hoursSpan = 24 - const hoursPerBar int = hoursSpan / bars - - if len(queriesSeries) > hoursSpan { - queriesSeries = queriesSeries[len(queriesSeries)-hoursSpan:] - } else if len(queriesSeries) < hoursSpan { - queriesSeries = append(make([]int, hoursSpan-len(queriesSeries)), queriesSeries...) + if len(queriesSeries) > dnsStatsHoursSpan { + queriesSeries = queriesSeries[len(queriesSeries)-dnsStatsHoursSpan:] + } else if len(queriesSeries) < dnsStatsHoursSpan { + queriesSeries = append(make([]int, dnsStatsHoursSpan-len(queriesSeries)), queriesSeries...) } - if len(blockedSeries) > hoursSpan { - blockedSeries = blockedSeries[len(blockedSeries)-hoursSpan:] - } else if len(blockedSeries) < hoursSpan { - blockedSeries = append(make([]int, hoursSpan-len(blockedSeries)), blockedSeries...) + if len(blockedSeries) > dnsStatsHoursSpan { + blockedSeries = blockedSeries[len(blockedSeries)-dnsStatsHoursSpan:] + } else if len(blockedSeries) < dnsStatsHoursSpan { + blockedSeries = append(make([]int, dnsStatsHoursSpan-len(blockedSeries)), blockedSeries...) } maxQueriesInSeries := 0 - for i := 0; i < bars; i++ { + for i := range dnsStatsBars { queries := 0 blocked := 0 - for j := 0; j < hoursPerBar; j++ { - queries += queriesSeries[i*hoursPerBar+j] - blocked += blockedSeries[i*hoursPerBar+j] + for j := range dnsStatsHoursPerBar { + queries += queriesSeries[i*dnsStatsHoursPerBar+j] + blocked += blockedSeries[i*dnsStatsHoursPerBar+j] } stats.Series[i] = dnsStatsSeries{ @@ -225,7 +239,7 @@ func fetchAdguardStats(instanceURL string, allowInsecure bool, username, passwor } } - for i := 0; i < bars; i++ { + for i := range dnsStatsBars { stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) } @@ -233,56 +247,28 @@ func fetchAdguardStats(instanceURL string, allowInsecure bool, username, passwor } // Legacy Pi-hole stats response (before v6) -type legacyPiholeStatsResponse struct { - TotalQueries int `json:"dns_queries_today"` - QueriesSeries piholeQueriesSeries `json:"domains_over_time"` - BlockedQueries int `json:"ads_blocked_today"` - BlockedSeries map[int64]int `json:"ads_over_time"` - BlockedPercentage float64 `json:"ads_percentage_today"` - TopBlockedDomains piholeTopBlockedDomains `json:"top_ads"` - DomainsBlocked int `json:"domains_being_blocked"` -} - -// Pi-hole v6+ response format -type piholeStatsResponse struct { - Queries struct { - Total int `json:"total"` - Blocked int `json:"blocked"` - PercentBlocked float64 `json:"percent_blocked"` - } `json:"queries"` - Gravity struct { - DomainsBlocked int `json:"domains_being_blocked"` - } `json:"gravity"` - //Note we do not need the full structure. We extract the values needed - //Adding dummy fields to allow easier json parsing. - QueriesSeries piholeQueriesSeries `json:"domains_over_time"` // Will always be empty - BlockedSeries map[int64]int `json:"ads_over_time"` // Will always be empty. -} - -type piholeTopDomainsResponse struct { - Domains []Domains `json:"domains"` - TotalQueries int `json:"total_queries"` - BlockedQueries int `json:"blocked_queries"` - Took float64 `json:"took"` -} - -type Domains struct { - Domain string `json:"domain"` - Count int `json:"count"` +type pihole5StatsResponse struct { + TotalQueries int `json:"dns_queries_today"` + QueriesSeries pihole5QueriesSeries `json:"domains_over_time"` + BlockedQueries int `json:"ads_blocked_today"` + BlockedSeries map[int64]int `json:"ads_over_time"` + BlockedPercentage float64 `json:"ads_percentage_today"` + TopBlockedDomains pihole5TopBlockedDomains `json:"top_ads"` + DomainsBlocked int `json:"domains_being_blocked"` } // If the user has query logging disabled it's possible for domains_over_time to be returned as an // empty array rather than a map which will prevent unmashalling the rest of the data so we use // custom unmarshal behavior to fallback to an empty map. // See https://github.com/glanceapp/glance/issues/289 -type piholeQueriesSeries map[int64]int +type pihole5QueriesSeries map[int64]int -func (p *piholeQueriesSeries) UnmarshalJSON(data []byte) error { +func (p *pihole5QueriesSeries) UnmarshalJSON(data []byte) error { temp := make(map[int64]int) err := json.Unmarshal(data, &temp) if err != nil { - *p = make(piholeQueriesSeries) + *p = make(pihole5QueriesSeries) } else { *p = temp } @@ -292,16 +278,16 @@ func (p *piholeQueriesSeries) UnmarshalJSON(data []byte) error { // If user has some level of privacy enabled on Pihole, `json:"top_ads"` is an empty array // Use custom unmarshal behavior to avoid not getting the rest of the valid data when unmarshalling -type piholeTopBlockedDomains map[string]int +type pihole5TopBlockedDomains map[string]int -func (p *piholeTopBlockedDomains) UnmarshalJSON(data []byte) error { +func (p *pihole5TopBlockedDomains) UnmarshalJSON(data []byte) error { // NOTE: do not change to piholeTopBlockedDomains type here or it will cause a stack overflow // because of the UnmarshalJSON method getting called recursively temp := make(map[string]int) err := json.Unmarshal(data, &temp) if err != nil { - *p = make(piholeTopBlockedDomains) + *p = make(pihole5TopBlockedDomains) } else { *p = temp } @@ -309,125 +295,175 @@ func (p *piholeTopBlockedDomains) UnmarshalJSON(data []byte) error { return nil } -// piholeGetSID retrieves a new SID from Pi-hole using the app password. -func piholeGetSID(instanceURL, appPassword string, allowInsecure bool) (string, error) { - var client requestDoer - if !allowInsecure { - client = defaultHTTPClient - } else { - client = defaultInsecureHTTPClient +func fetchPihole5Stats(instanceURL string, allowInsecure bool, token string, noGraph bool) (*dnsStats, error) { + if token == "" { + return nil, errors.New("missing API token") } - requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth" - requestBody := []byte(`{"password":"` + appPassword + `"}`) - - request, err := http.NewRequest("POST", requestURL, bytes.NewBuffer(requestBody)) - if err != nil { - return "", errors.New("failed to create authentication request: " + err.Error()) - } - request.Header.Set("Content-Type", "application/json") - - response, err := client.Do(request) - if err != nil { - return "", errors.New("failed to send authentication request: " + err.Error()) - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return "", errors.New("authentication failed, received status: " + response.Status) - } - - body, err := io.ReadAll(response.Body) - if err != nil { - return "", errors.New("failed to read authentication response: " + err.Error()) - } - - var jsonResponse struct { - Session struct { - SID string `json:"sid"` - } `json:"session"` - } - - if err := json.Unmarshal(body, &jsonResponse); err != nil { - return "", errors.New("failed to parse authentication response: " + err.Error()) - } - - if jsonResponse.Session.SID == "" { - return "", errors.New("authentication response did not contain a valid SID") - } - - return jsonResponse.Session.SID, nil -} - -// checkPiholeSID checks if the SID is valid by checking HTTP response status code from /api/auth. -func checkPiholeSID(instanceURL string, sid string, allowInsecure bool) error { - requestURL := strings.TrimRight(instanceURL, "/") + "/api/auth" + requestURL := strings.TrimRight(instanceURL, "/") + + "/admin/api.php?summaryRaw&topItems&overTimeData10mins&auth=" + token request, err := http.NewRequest("GET", requestURL, nil) if err != nil { - return err - } - request.Header.Set("x-ftl-sid", sid) - - var client requestDoer - if !allowInsecure { - client = defaultHTTPClient - } else { - client = defaultInsecureHTTPClient + return nil, err } - response, err := client.Do(request) + var client = ternary(allowInsecure, defaultInsecureHTTPClient, defaultHTTPClient) + responseJson, err := decodeJsonFromRequest[pihole5StatsResponse](client, request) if err != nil { - return err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return errors.New("SID is invalid, received status: " + response.Status) + return nil, err } - return nil + stats := &dnsStats{ + TotalQueries: responseJson.TotalQueries, + BlockedQueries: responseJson.BlockedQueries, + BlockedPercent: int(responseJson.BlockedPercentage), + DomainsBlocked: responseJson.DomainsBlocked, + } + + if len(responseJson.TopBlockedDomains) > 0 { + domains := make([]dnsStatsBlockedDomain, 0, len(responseJson.TopBlockedDomains)) + + for domain, count := range responseJson.TopBlockedDomains { + domains = append(domains, dnsStatsBlockedDomain{ + Domain: domain, + PercentBlocked: int(float64(count) / float64(responseJson.BlockedQueries) * 100), + }) + } + + sort.Slice(domains, func(a, b int) bool { + return domains[a].PercentBlocked > domains[b].PercentBlocked + }) + + stats.TopBlockedDomains = domains[:min(len(domains), 5)] + } + + if noGraph { + return stats, nil + } + + // Pihole _should_ return data for the last 24 hours in a 10 minute interval, 6*24 = 144 + if len(responseJson.QueriesSeries) != 144 || len(responseJson.BlockedSeries) != 144 { + slog.Warn( + "DNS stats for pihole: did not get expected 144 data points", + "len(queries)", len(responseJson.QueriesSeries), + "len(blocked)", len(responseJson.BlockedSeries), + ) + return stats, nil + } + + var lowestTimestamp int64 = 0 + for timestamp := range responseJson.QueriesSeries { + if lowestTimestamp == 0 || timestamp < lowestTimestamp { + lowestTimestamp = timestamp + } + } + + maxQueriesInSeries := 0 + + for i := range dnsStatsBars { + queries := 0 + blocked := 0 + + for j := range 18 { + index := lowestTimestamp + int64(i*10800+j*600) + + queries += responseJson.QueriesSeries[index] + blocked += responseJson.BlockedSeries[index] + } + + if queries > maxQueriesInSeries { + maxQueriesInSeries = queries + } + + stats.Series[i] = dnsStatsSeries{ + Queries: queries, + Blocked: blocked, + } + + if queries > 0 { + stats.Series[i].PercentBlocked = int(float64(blocked) / float64(queries) * 100) + } + } + + for i := range dnsStatsBars { + stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) + } + + return stats, nil } -// fetchPiholeTopDomains fetches the top blocked domains for Pi-hole v6+. -func fetchPiholeTopDomains(instanceURL string, sid string, allowInsecure bool) (piholeTopDomainsResponse, error) { - requestURL := strings.TrimRight(instanceURL, "/") + "/api/stats/top_domains?blocked=true" +func fetchPiholeStats( + instanceURL string, + allowInsecure bool, + password string, + sessionID string, + includeGraph bool, + includeTopDomains bool, +) (*dnsStats, string, error) { + instanceURL = strings.TrimRight(instanceURL, "/") + var client = ternary(allowInsecure, defaultInsecureHTTPClient, defaultHTTPClient) - request, err := http.NewRequest("GET", requestURL, nil) - if err != nil { - return piholeTopDomainsResponse{}, err + fetchNewSessionID := func() error { + newSessionID, err := fetchPiholeSessionID(instanceURL, client, password) + if err != nil { + return err + } + sessionID = newSessionID + return nil } - request.Header.Set("x-ftl-sid", sid) - var client requestDoer - if !allowInsecure { - client = defaultHTTPClient + if sessionID == "" { + if err := fetchNewSessionID(); err != nil { + slog.Error("Failed to fetch Pihole v6 session ID", "error", err) + return nil, "", fmt.Errorf("fetching session ID: %v", err) + } } else { - client = defaultInsecureHTTPClient + isValid, err := checkPiholeSessionIDIsValid(instanceURL, client, sessionID) + if err != nil { + slog.Error("Failed to check Pihole v6 session ID validity", "error", err) + return nil, "", fmt.Errorf("checking session ID: %v", err) + } + + if !isValid { + if err := fetchNewSessionID(); err != nil { + slog.Error("Failed to renew Pihole v6 session ID", "error", err) + return nil, "", fmt.Errorf("renewing session ID: %v", err) + } + } } - return decodeJsonFromRequest[piholeTopDomainsResponse](client, request) -} + var wg sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() -// fetchPiholeSeries fetches the series data for Pi-hole v6+ (QueriesSeries and BlockedSeries). -func fetchPiholeSeries(instanceURL string, sid string, allowInsecure bool) (piholeQueriesSeries, map[int64]int, error) { - requestURL := strings.TrimRight(instanceURL, "/") + "/api/history" - - request, err := http.NewRequest("GET", requestURL, nil) - if err != nil { - return nil, nil, err - } - request.Header.Set("x-ftl-sid", sid) - - var client requestDoer - if !allowInsecure { - client = defaultHTTPClient - } else { - client = defaultInsecureHTTPClient + type statsResponseJson struct { + Queries struct { + Total int `json:"total"` + Blocked int `json:"blocked"` + PercentBlocked float64 `json:"percent_blocked"` + } `json:"queries"` + Gravity struct { + DomainsBlocked int `json:"domains_being_blocked"` + } `json:"gravity"` } - // Define the correct struct to match the API response - var responseJson struct { + statsRequest, _ := http.NewRequestWithContext(ctx, "GET", instanceURL+"/api/stats/summary", nil) + statsRequest.Header.Set("x-ftl-sid", sessionID) + + var statsResponse statsResponseJson + var statsErr error + + wg.Add(1) + go func() { + defer wg.Done() + statsResponse, statsErr = decodeJsonFromRequest[statsResponseJson](client, statsRequest) + if statsErr != nil { + cancel() + } + }() + + type seriesResponseJson struct { History []struct { Timestamp int64 `json:"timestamp"` Total int `json:"total"` @@ -435,39 +471,124 @@ func fetchPiholeSeries(instanceURL string, sid string, allowInsecure bool) (piho } `json:"history"` } - err = decodeJsonInto(client, request, &responseJson) - if err != nil { - return nil, nil, err + var seriesResponse seriesResponseJson + var seriesErr error + + if includeGraph { + seriesRequest, _ := http.NewRequestWithContext(ctx, "GET", instanceURL+"/api/history", nil) + seriesRequest.Header.Set("x-ftl-sid", sessionID) + + wg.Add(1) + go func() { + defer wg.Done() + seriesResponse, seriesErr = decodeJsonFromRequest[seriesResponseJson](client, seriesRequest) + }() } - queriesSeries := make(piholeQueriesSeries) - blockedSeries := make(map[int64]int) - - // Populate the series data from history array - for _, entry := range responseJson.History { - queriesSeries[entry.Timestamp] = entry.Total - blockedSeries[entry.Timestamp] = entry.Blocked + type topDomainsResponseJson struct { + Domains []struct { + Domain string `json:"domain"` + Count int `json:"count"` + } `json:"domains"` + TotalQueries int `json:"total_queries"` + BlockedQueries int `json:"blocked_queries"` + Took float64 `json:"took"` } - return queriesSeries, blockedSeries, nil -} + var topDomainsResponse topDomainsResponseJson + var topDomainsErr error -// Helper functions to process the responses -func parsePiholeStats(r piholeStatsResponse, topDomains piholeTopDomainsResponse, noGraph bool) *dnsStats { + if includeTopDomains { + topDomainsRequest, _ := http.NewRequestWithContext(ctx, "GET", instanceURL+"/api/stats/top_domains?blocked=true", nil) + topDomainsRequest.Header.Set("x-ftl-sid", sessionID) + + wg.Add(1) + go func() { + defer wg.Done() + topDomainsResponse, topDomainsErr = decodeJsonFromRequest[topDomainsResponseJson](client, topDomainsRequest) + }() + } + + wg.Wait() + partialContent := false + + if statsErr != nil { + return nil, "", fmt.Errorf("fetching stats: %v", statsErr) + } + + if includeGraph && seriesErr != nil { + slog.Error("Failed to fetch Pihole v6 graph data", "error", seriesErr) + partialContent = true + } + + if includeTopDomains && topDomainsErr != nil { + slog.Error("Failed to fetch Pihole v6 top domains", "error", topDomainsErr) + partialContent = true + } stats := &dnsStats{ - TotalQueries: r.Queries.Total, - BlockedQueries: r.Queries.Blocked, - BlockedPercent: int(r.Queries.PercentBlocked), - DomainsBlocked: r.Gravity.DomainsBlocked, + TotalQueries: statsResponse.Queries.Total, + BlockedQueries: statsResponse.Queries.Blocked, + BlockedPercent: int(statsResponse.Queries.PercentBlocked), + DomainsBlocked: statsResponse.Gravity.DomainsBlocked, } - if len(topDomains.Domains) > 0 { - domains := make([]dnsStatsBlockedDomain, 0, len(topDomains.Domains)) - for _, d := range topDomains.Domains { + ItsUsedTrustMeBro(seriesResponse, topDomainsResponse) + + if includeGraph && seriesErr == nil { + if len(seriesResponse.History) != 145 { + slog.Error( + "Pihole v6 graph data has unexpected length", + "length", len(seriesResponse.History), + "expected", 145, + ) + partialContent = true + } else { + // The API from v5 used to return 144 data points, but v6 returns 145. + // We only show data from the last 24 hours hours, Pihole returns data + // points in a 10 minute interval, 24*(60/10) = 144. Why is there an extra + // data point? I don't know, but we'll just ignore the first one since it's + // the oldest data point. + history := seriesResponse.History[1:] + + const interval = 10 + const dataPointsPerBar = dnsStatsHoursPerBar * (60 / interval) + + maxQueriesInSeries := 0 + + for i := range dnsStatsBars { + queries := 0 + blocked := 0 + for j := range dataPointsPerBar { + index := i*dataPointsPerBar + j + queries += history[index].Total + blocked += history[index].Blocked + } + if queries > maxQueriesInSeries { + maxQueriesInSeries = queries + } + stats.Series[i] = dnsStatsSeries{ + Queries: queries, + Blocked: blocked, + } + if queries > 0 { + stats.Series[i].PercentBlocked = int(float64(blocked) / float64(queries) * 100) + } + } + + for i := range dnsStatsBars { + stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) + } + } + } + + if includeTopDomains && topDomainsErr == nil && len(topDomainsResponse.Domains) > 0 { + domains := make([]dnsStatsBlockedDomain, 0, len(topDomainsResponse.Domains)) + for i := range topDomainsResponse.Domains { + d := &topDomainsResponse.Domains[i] domains = append(domains, dnsStatsBlockedDomain{ Domain: d.Domain, - PercentBlocked: int(float64(d.Count) / float64(r.Queries.Blocked) * 100), + PercentBlocked: int(float64(d.Count) / float64(statsResponse.Queries.Blocked) * 100), }) } @@ -476,204 +597,71 @@ func parsePiholeStats(r piholeStatsResponse, topDomains piholeTopDomainsResponse }) stats.TopBlockedDomains = domains[:min(len(domains), 5)] } - if noGraph { - return stats - } - // Pihole _should_ return data for the last 24 hours - if len(r.QueriesSeries) != 145 || len(r.BlockedSeries) != 145 { - return stats - } - - - var lowestTimestamp int64 = 0 - for timestamp := range r.QueriesSeries { - if lowestTimestamp == 0 || timestamp < lowestTimestamp { - lowestTimestamp = timestamp - } - } - maxQueriesInSeries := 0 - - for i := 0; i < 8; i++ { - queries := 0 - blocked := 0 - for j := 0; j < 18; j++ { - index := lowestTimestamp + int64(i*10800+j*600) - queries += r.QueriesSeries[index] - blocked += r.BlockedSeries[index] - } - if queries > maxQueriesInSeries { - maxQueriesInSeries = queries - } - stats.Series[i] = dnsStatsSeries{ - Queries: queries, - Blocked: blocked, - } - if queries > 0 { - stats.Series[i].PercentBlocked = int(float64(blocked) / float64(queries) * 100) - } - } - for i := 0; i < 8; i++ { - stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) - } - return stats -} -func parsePiholeStatsLegacy(r legacyPiholeStatsResponse, noGraph bool) *dnsStats { - - stats := &dnsStats{ - TotalQueries: r.TotalQueries, - BlockedQueries: r.BlockedQueries, - BlockedPercent: int(r.BlockedPercentage), - DomainsBlocked: r.DomainsBlocked, - } - if len(r.TopBlockedDomains) > 0 { - domains := make([]dnsStatsBlockedDomain, 0, len(r.TopBlockedDomains)) - - for domain, count := range r.TopBlockedDomains { - domains = append(domains, dnsStatsBlockedDomain{ - Domain: domain, - PercentBlocked: int(float64(count) / float64(r.BlockedQueries) * 100), - }) - } - - sort.Slice(domains, func(a, b int) bool { - return domains[a].PercentBlocked > domains[b].PercentBlocked - }) - - stats.TopBlockedDomains = domains[:min(len(domains), 5)] - } - if noGraph { - return stats - } - - // Pihole _should_ return data for the last 24 hours in a 10 minute interval, 6*24 = 144 - if len(r.QueriesSeries) != 144 || len(r.BlockedSeries) != 144 { - return stats - } - - var lowestTimestamp int64 = 0 - for timestamp := range r.QueriesSeries { - if lowestTimestamp == 0 || timestamp < lowestTimestamp { - lowestTimestamp = timestamp - } - } - maxQueriesInSeries := 0 - - for i := 0; i < 8; i++ { - queries := 0 - blocked := 0 - for j := 0; j < 18; j++ { - index := lowestTimestamp + int64(i*10800+j*600) - queries += r.QueriesSeries[index] - blocked += r.BlockedSeries[index] - } - if queries > maxQueriesInSeries { - maxQueriesInSeries = queries - } - stats.Series[i] = dnsStatsSeries{ - Queries: queries, - Blocked: blocked, - } - if queries > 0 { - stats.Series[i].PercentBlocked = int(float64(blocked) / float64(queries) * 100) - } - } - for i := 0; i < 8; i++ { - stats.Series[i].PercentTotal = int(float64(stats.Series[i].Queries) / float64(maxQueriesInSeries) * 100) - } - return stats + return stats, sessionID, ternary(partialContent, errPartialContent, nil) } -func fetchPiholeStats(instanceURL string, allowInsecure bool, token string, noGraph bool, version, appPassword string) (*dnsStats, error) { - instanceURL = strings.TrimRight(instanceURL, "/") - var requestURL string - var sid string - isV6 := version == "" || version == "6" +func fetchPiholeSessionID(instanceURL string, client *http.Client, password string) (string, error) { + requestBody := []byte(`{"password":"` + password + `"}`) - if isV6 { - if appPassword == "" { - return nil, errors.New("missing app password") - } - - sid = os.Getenv("SID") - if sid == "" { - newSid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) - if err != nil { - return nil, fmt.Errorf("failed to get SID: %w", err) - } - sid = newSid - os.Setenv("SID", sid) - } else { - err := checkPiholeSID(instanceURL, sid, allowInsecure) - if err != nil { - newSid, err := piholeGetSID(instanceURL, appPassword, allowInsecure) - if err != nil { - return nil, fmt.Errorf("failed to get SID after invalid check: %w", err) - } - sid = newSid - os.Setenv("SID", sid) - } - } - - requestURL = instanceURL + "/api/stats/summary" - } else { - if token == "" { - return nil, errors.New("missing API token") - } - requestURL = instanceURL + "/admin/api.php?summaryRaw&topItems&overTimeData10mins&auth=" + token - } - - request, err := http.NewRequest("GET", requestURL, nil) + request, err := http.NewRequest("POST", instanceURL+"/api/auth", bytes.NewBuffer(requestBody)) if err != nil { - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - - if isV6 { - request.Header.Set("x-ftl-sid", sid) - } - - var client requestDoer - if !allowInsecure { - client = defaultHTTPClient - } else { - client = defaultInsecureHTTPClient - } - - var responseJson interface{} - if isV6 { - responseJson, err = decodeJsonFromRequest[piholeStatsResponse](client, request) - } else { - responseJson, err = decodeJsonFromRequest[legacyPiholeStatsResponse](client, request) + return "", fmt.Errorf("creating authentication request: %v", err) } + request.Header.Set("Content-Type", "application/json") + response, err := client.Do(request) if err != nil { - return nil, fmt.Errorf("failed to decode JSON response: %w", err) + return "", fmt.Errorf("sending authentication request: %v", err) + } + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + return "", fmt.Errorf("reading authentication response: %v", err) } - switch r := responseJson.(type) { - case piholeStatsResponse: - // Fetch top domains separately for v6+ - topDomains, err := fetchPiholeTopDomains(instanceURL, sid, allowInsecure) - if err != nil { - return nil, fmt.Errorf("failed to fetch top domains: %w", err) - } - - // Fetch series data separately for v6+ - queriesSeries, blockedSeries, err := fetchPiholeSeries(instanceURL, sid, allowInsecure) - if err != nil { - return nil, fmt.Errorf("failed to fetch queries series: %w", err) - } - - // Merge series data - r.QueriesSeries = queriesSeries - r.BlockedSeries = blockedSeries - - return parsePiholeStats(r, topDomains, noGraph), nil - - case legacyPiholeStatsResponse: - return parsePiholeStatsLegacy(r, noGraph), nil - - default: - return nil, errors.New("unexpected response type") + var jsonResponse struct { + Session struct { + SID string `json:"sid"` + Message string `json:"message"` + } `json:"session"` } -} \ No newline at end of file + + if err := json.Unmarshal(body, &jsonResponse); err != nil { + return "", fmt.Errorf("parsing authentication response: %v", err) + } + + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf( + "authentication request returned status %s with message '%s'", + response.Status, jsonResponse.Session.Message, + ) + } + + if jsonResponse.Session.SID == "" { + return "", errors.New("authentication response returned empty session ID") + } + + return jsonResponse.Session.SID, nil +} + +func checkPiholeSessionIDIsValid(instanceURL string, client *http.Client, sessionID string) (bool, error) { + request, err := http.NewRequest("GET", instanceURL+"/api/auth", nil) + if err != nil { + return false, fmt.Errorf("creating session ID check request: %v", err) + } + request.Header.Set("x-ftl-sid", sessionID) + + response, err := client.Do(request) + if err != nil { + return false, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusUnauthorized { + return false, fmt.Errorf("session ID check request returned status %s", response.Status) + } + + return response.StatusCode == http.StatusOK, nil +} diff --git a/internal/glance/widget-utils.go b/internal/glance/widget-utils.go index fa141c1..8fb76dd 100644 --- a/internal/glance/widget-utils.go +++ b/internal/glance/widget-utils.go @@ -82,16 +82,6 @@ func decodeJsonFromRequest[T any](client requestDoer, request *http.Request) (T, return result, nil } -func decodeJsonInto[T any](client requestDoer, request *http.Request, out *T) error { - result, err := decodeJsonFromRequest[T](client, request) - if err != nil { - return err - } - - *out = result - return nil -} - func decodeJsonFromRequestTask[T any](client requestDoer) func(*http.Request) (T, error) { return func(request *http.Request) (T, error) { return decodeJsonFromRequest[T](client, request) From 1b3f022b2dcc8abb2a18626c4e050f3683902ec2 Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sat, 15 Mar 2025 19:03:48 +0000 Subject: [PATCH 010/119] Rename service to reduce ambiguity --- docs/configuration.md | 4 ++-- internal/glance/widget-dns-stats.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a000b94..231c04c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1765,14 +1765,14 @@ Preview: | allow-insecure | bool | no | false | | url | string | yes | | | username | string | when service is `adguard` | | -| password | string | when service is `adguard` or `pihole6` | | +| password | string | when service is `adguard` or `pihole-v6` | | | token | string | when service is `pihole` | | | hide-graph | bool | no | false | | hide-top-domains | bool | no | false | | hour-format | string | no | 12h | ##### `service` -Either `adguard`, or `pihole` (major version 5 and below) or `pihole6` (major version 6 and above). +Either `adguard`, or `pihole` (major version 5 and below) or `pihole-v6` (major version 6 and above). ##### `allow-insecure` Whether to allow invalid/self-signed certificates when making the request to the service. diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index b8d6447..9c04fbe 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -61,10 +61,10 @@ func (widget *dnsStatsWidget) initialize() error { switch widget.Service { case "adguard": - case "pihole6": + case "pihole-v6": case "pihole": default: - return errors.New("service must be one of: adguard, pihole6, pihole") + return errors.New("service must be one of: adguard, pihole-v6, pihole") } return nil From 14bdcf71feeb4d999540cdf456ea7aff270e6fd8 Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sat, 15 Mar 2025 19:10:21 +0000 Subject: [PATCH 011/119] Remove debug code --- internal/glance/widget-dns-stats.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index 9c04fbe..12b471d 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -533,8 +533,6 @@ func fetchPiholeStats( DomainsBlocked: statsResponse.Gravity.DomainsBlocked, } - ItsUsedTrustMeBro(seriesResponse, topDomainsResponse) - if includeGraph && seriesErr == nil { if len(seriesResponse.History) != 145 { slog.Error( From 4c1165533c082039be9a64121931daf1b44ef877 Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sat, 15 Mar 2025 19:41:08 +0000 Subject: [PATCH 012/119] Define service strings as consts to avoid forgetting to change them in all places --- internal/glance/widget-dns-stats.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index 12b471d..b4e32c1 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -42,6 +42,12 @@ type dnsStatsWidget struct { Password string `yaml:"password"` } +const ( + dnsServiceAdguard = "adguard" + dnsServicePihole = "pihole" + dnsServicePiholeV6 = "pihole-v6" +) + func makeDNSWidgetTimeLabels(format string) [8]string { now := time.Now() var labels [dnsStatsBars]string @@ -60,11 +66,11 @@ func (widget *dnsStatsWidget) initialize() error { withCacheDuration(10 * time.Minute) switch widget.Service { - case "adguard": - case "pihole-v6": - case "pihole": + case dnsServiceAdguard: + case dnsServicePiholeV6: + case dnsServicePihole: default: - return errors.New("service must be one of: adguard, pihole-v6, pihole") + return fmt.Errorf("service must be one of: %s, %s, %s", dnsServiceAdguard, dnsServicePihole, dnsServicePiholeV6) } return nil @@ -75,11 +81,11 @@ func (widget *dnsStatsWidget) update(ctx context.Context) { var err error switch widget.Service { - case "adguard": + case dnsServiceAdguard: stats, err = fetchAdguardStats(widget.URL, widget.AllowInsecure, widget.Username, widget.Password, widget.HideGraph) - case "pihole": + case dnsServicePihole: stats, err = fetchPihole5Stats(widget.URL, widget.AllowInsecure, widget.Token, widget.HideGraph) - case "pihole6": + case dnsServicePiholeV6: var newSessionID string stats, newSessionID, err = fetchPiholeStats( widget.URL, From 047d13afd1edd16802c8cf6eab7047dc1ac54e04 Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sun, 16 Mar 2025 01:24:56 +0000 Subject: [PATCH 013/119] Fix summary triangle showing on Safari --- internal/glance/static/main.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/glance/static/main.css b/internal/glance/static/main.css index f686c59..672fc24 100644 --- a/internal/glance/static/main.css +++ b/internal/glance/static/main.css @@ -552,6 +552,10 @@ kbd:active { z-index: 1; } +.summary::-webkit-details-marker { + display: none; +} + .details[open] .summary { margin-bottom: .8rem; } From fbcea127864a7b9e21e840540c1c3c9fa26967ee Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sun, 16 Mar 2025 01:25:13 +0000 Subject: [PATCH 014/119] Add more info to logged message --- internal/glance/widget-dns-stats.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/glance/widget-dns-stats.go b/internal/glance/widget-dns-stats.go index b4e32c1..eb42b5a 100644 --- a/internal/glance/widget-dns-stats.go +++ b/internal/glance/widget-dns-stats.go @@ -644,7 +644,10 @@ func fetchPiholeSessionID(instanceURL string, client *http.Client, password stri } if jsonResponse.Session.SID == "" { - return "", errors.New("authentication response returned empty session ID") + return "", fmt.Errorf( + "authentication response returned empty session ID, status code %d, message '%s'", + response.StatusCode, jsonResponse.Session.Message, + ) } return jsonResponse.Session.SID, nil From 1615c20e66b5097d2cd1a40bd9802b4e46f1c25a Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sun, 16 Mar 2025 01:32:35 +0000 Subject: [PATCH 015/119] Change custom api int64 types to int Some of Go's native template functions return int and having to juggle between int and int64 might get messy so we'll try to stick to having a single int type --- internal/glance/widget-custom-api.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/glance/widget-custom-api.go b/internal/glance/widget-custom-api.go index c8e3773..197ba68 100644 --- a/internal/glance/widget-custom-api.go +++ b/internal/glance/widget-custom-api.go @@ -153,12 +153,12 @@ func (r *decoratedGJSONResult) String(key string) string { return r.Get(key).String() } -func (r *decoratedGJSONResult) Int(key string) int64 { +func (r *decoratedGJSONResult) Int(key string) int { if key == "" { - return r.Result.Int() + return int(r.Result.Int()) } - return r.Get(key).Int() + return int(r.Get(key).Int()) } func (r *decoratedGJSONResult) Float(key string) float64 { @@ -179,11 +179,11 @@ func (r *decoratedGJSONResult) Bool(key string) bool { var customAPITemplateFuncs = func() template.FuncMap { funcs := template.FuncMap{ - "toFloat": func(a int64) float64 { + "toFloat": func(a int) float64 { return float64(a) }, - "toInt": func(a float64) int64 { - return int64(a) + "toInt": func(a float64) int { + return int(a) }, "add": func(a, b float64) float64 { return a + b From 71112173b9df1f96fe159964d5d2238f9ecd37d3 Mon Sep 17 00:00:00 2001 From: Svilen Markov <7613769+svilenmarkov@users.noreply.github.com> Date: Sun, 16 Mar 2025 23:15:19 +0000 Subject: [PATCH 016/119] Fix title link not opening in new tab --- internal/glance/templates/videos-vertical-list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/glance/templates/videos-vertical-list.html b/internal/glance/templates/videos-vertical-list.html index b7ea6b2..a735a74 100644 --- a/internal/glance/templates/videos-vertical-list.html +++ b/internal/glance/templates/videos-vertical-list.html @@ -6,7 +6,7 @@
  • - {{ .Title }} + {{ .Title }}