feat: Use conditional requests for RSS feeds

This commit is contained in:
Charles Harries 2025-03-17 19:06:32 +00:00
parent 0b2cfc45c1
commit f36527995e
2 changed files with 102 additions and 32 deletions

View File

@ -2,6 +2,7 @@ package glance
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"html" "html"
"html/template" "html/template"
@ -25,18 +26,25 @@ var (
rssWidgetHorizontalCards2Template = mustParseTemplate("rss-horizontal-cards-2.html", "widget-base.html") rssWidgetHorizontalCards2Template = mustParseTemplate("rss-horizontal-cards-2.html", "widget-base.html")
) )
type cachedFeed struct {
LastModified time.Time
Etag string
Items rssFeedItemList
}
type rssWidget struct { type rssWidget struct {
widgetBase `yaml:",inline"` widgetBase `yaml:",inline"`
FeedRequests []rssFeedRequest `yaml:"feeds"` FeedRequests []rssFeedRequest `yaml:"feeds"`
Style string `yaml:"style"` Style string `yaml:"style"`
ThumbnailHeight float64 `yaml:"thumbnail-height"` ThumbnailHeight float64 `yaml:"thumbnail-height"`
CardHeight float64 `yaml:"card-height"` CardHeight float64 `yaml:"card-height"`
Items rssFeedItemList `yaml:"-"` Items rssFeedItemList `yaml:"-"`
Limit int `yaml:"limit"` Limit int `yaml:"limit"`
CollapseAfter int `yaml:"collapse-after"` CollapseAfter int `yaml:"collapse-after"`
SingleLineTitles bool `yaml:"single-line-titles"` SingleLineTitles bool `yaml:"single-line-titles"`
PreserveOrder bool `yaml:"preserve-order"` PreserveOrder bool `yaml:"preserve-order"`
NoItemsMessage string `yaml:"-"` NoItemsMessage string `yaml:"-"`
CachedFeeds map[string]cachedFeed `yaml:"-"`
} }
func (widget *rssWidget) initialize() error { func (widget *rssWidget) initialize() error {
@ -70,21 +78,41 @@ func (widget *rssWidget) initialize() error {
} }
func (widget *rssWidget) update(ctx context.Context) { func (widget *rssWidget) update(ctx context.Context) {
items, err := fetchItemsFromRSSFeeds(widget.FeedRequests) // Populate If-Modified-Since header and Etag
for i, req := range widget.FeedRequests {
if cachedFeed, ok := widget.CachedFeeds[req.URL]; ok {
widget.FeedRequests[i].IfModifiedSince = cachedFeed.LastModified
widget.FeedRequests[i].Etag = cachedFeed.Etag
}
}
allItems, feeds, err := fetchItemsFromRSSFeeds(widget.FeedRequests, widget.CachedFeeds)
if !widget.canContinueUpdateAfterHandlingErr(err) { if !widget.canContinueUpdateAfterHandlingErr(err) {
return return
} }
if !widget.PreserveOrder { if !widget.PreserveOrder {
items.sortByNewest() allItems.sortByNewest()
} }
if len(items) > widget.Limit { if len(allItems) > widget.Limit {
items = items[:widget.Limit] allItems = allItems[:widget.Limit]
} }
widget.Items = items widget.Items = allItems
cachedFeeds := make(map[string]cachedFeed)
for _, feed := range feeds {
if !feed.LastModified.IsZero() || feed.Etag != "" {
cachedFeeds[feed.URL] = cachedFeed{
LastModified: feed.LastModified,
Etag: feed.Etag,
Items: feed.Items,
}
}
}
widget.CachedFeeds = cachedFeeds
} }
func (widget *rssWidget) Render() template.HTML { func (widget *rssWidget) Render() template.HTML {
@ -152,10 +180,19 @@ type rssFeedRequest struct {
ItemLinkPrefix string `yaml:"item-link-prefix"` ItemLinkPrefix string `yaml:"item-link-prefix"`
Headers map[string]string `yaml:"headers"` Headers map[string]string `yaml:"headers"`
IsDetailed bool `yaml:"-"` IsDetailed bool `yaml:"-"`
IfModifiedSince time.Time `yaml:"-"`
Etag string `yaml:"-"`
} }
type rssFeedItemList []rssFeedItem type rssFeedItemList []rssFeedItem
type rssFeedResponse struct {
URL string
Items rssFeedItemList
LastModified time.Time
Etag string
}
func (f rssFeedItemList) sortByNewest() rssFeedItemList { func (f rssFeedItemList) sortByNewest() rssFeedItemList {
sort.Slice(f, func(i, j int) bool { sort.Slice(f, func(i, j int) bool {
return f[i].PublishedAt.After(f[j].PublishedAt) return f[i].PublishedAt.After(f[j].PublishedAt)
@ -166,41 +203,67 @@ func (f rssFeedItemList) sortByNewest() rssFeedItemList {
var feedParser = gofeed.NewParser() var feedParser = gofeed.NewParser()
func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) { func fetchItemsFromRSSFeedTask(request rssFeedRequest) (rssFeedResponse, error) {
feedResponse := rssFeedResponse{URL: request.URL}
req, err := http.NewRequest("GET", request.URL, nil) req, err := http.NewRequest("GET", request.URL, nil)
if err != nil { if err != nil {
return nil, err return feedResponse, err
} }
req.Header.Add("User-Agent", fmt.Sprintf("Glance v%s", buildVersion))
for key, value := range request.Headers { for key, value := range request.Headers {
req.Header.Add(key, value) req.Header.Add(key, value)
} }
if !request.IfModifiedSince.IsZero() {
req.Header.Add("If-Modified-Since", request.IfModifiedSince.Format(http.TimeFormat))
}
if request.Etag != "" {
req.Header.Add("If-None-Match", request.Etag)
}
resp, err := defaultHTTPClient.Do(req) resp, err := defaultHTTPClient.Do(req)
if err != nil { if err != nil {
return nil, err return feedResponse, err
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusNotModified {
return feedResponse, errNotModified
}
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL) return feedResponse, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL)
} }
body, err := io.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return nil, err return feedResponse, err
} }
feed, err := feedParser.ParseString(string(body)) feed, err := feedParser.ParseString(string(body))
if err != nil { if err != nil {
return nil, err return feedResponse, err
} }
if request.Limit > 0 && len(feed.Items) > request.Limit { if request.Limit > 0 && len(feed.Items) > request.Limit {
feed.Items = feed.Items[:request.Limit] feed.Items = feed.Items[:request.Limit]
} }
items := make(rssFeedItemList, 0, len(feed.Items)) items := make([]rssFeedItem, 0, len(feed.Items))
if lastModified := resp.Header.Get("Last-Modified"); lastModified != "" {
if t, err := time.Parse(http.TimeFormat, lastModified); err == nil {
feedResponse.LastModified = t
}
}
if etag := resp.Header.Get("Etag"); etag != "" {
feedResponse.Etag = etag
}
for i := range feed.Items { for i := range feed.Items {
item := feed.Items[i] item := feed.Items[i]
@ -289,7 +352,8 @@ func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) {
items = append(items, rssItem) items = append(items, rssItem)
} }
return items, nil feedResponse.Items = items
return feedResponse, nil
} }
func recursiveFindThumbnailInExtensions(extensions map[string][]gofeedext.Extension) string { func recursiveFindThumbnailInExtensions(extensions map[string][]gofeedext.Extension) string {
@ -322,33 +386,38 @@ func findThumbnailInItemExtensions(item *gofeed.Item) string {
return recursiveFindThumbnailInExtensions(media) return recursiveFindThumbnailInExtensions(media)
} }
func fetchItemsFromRSSFeeds(requests []rssFeedRequest) (rssFeedItemList, error) { func fetchItemsFromRSSFeeds(requests []rssFeedRequest, cachedFeeds map[string]cachedFeed) (rssFeedItemList, []rssFeedResponse, error) {
job := newJob(fetchItemsFromRSSFeedTask, requests).withWorkers(30) job := newJob(fetchItemsFromRSSFeedTask, requests).withWorkers(30)
feeds, errs, err := workerPoolDo(job) feeds, errs, err := workerPoolDo(job)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %v", errNoContent, err) return nil, nil, fmt.Errorf("%w: %v", errNoContent, err)
} }
failed := 0 failed := 0
notModified := 0
entries := make(rssFeedItemList, 0, len(feeds)*10) entries := make(rssFeedItemList, 0, len(feeds)*10)
for i := range feeds { for i := range feeds {
if errs[i] != nil { if errs[i] == nil {
entries = append(entries, feeds[i].Items...)
} else if errors.Is(errs[i], errNotModified) {
notModified++
entries = append(entries, cachedFeeds[feeds[i].URL].Items...)
slog.Debug("Feed not modified", "url", requests[i].URL, "debug", errs[i])
} else {
failed++ failed++
slog.Error("Failed to get RSS feed", "url", requests[i].URL, "error", errs[i]) slog.Error("Failed to get RSS feed", "url", requests[i].URL, "error", errs[i])
continue
} }
entries = append(entries, feeds[i]...)
} }
if failed == len(requests) { if failed == len(requests) {
return nil, errNoContent return nil, nil, errNoContent
} }
if failed > 0 { if failed > 0 {
return entries, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed) return entries, feeds, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed)
} }
return entries, nil return entries, feeds, nil
} }

View File

@ -19,6 +19,7 @@ import (
var ( var (
errNoContent = errors.New("failed to retrieve any content") errNoContent = errors.New("failed to retrieve any content")
errPartialContent = errors.New("failed to retrieve some of the content") errPartialContent = errors.New("failed to retrieve some of the content")
errNotModified = errors.New("content not modified")
) )
const defaultClientTimeout = 5 * time.Second const defaultClientTimeout = 5 * time.Second