extern/glance
1
0
Fork 0
mirror of https://github.com/glanceapp/glance.git synced 2025-06-21 10:27:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: Use conditional requests for RSS feeds

Browse Source
This commit is contained in:
Charles Harries 2025-03-17 19:06:32 +00:00
parent 0b2cfc45c1
commit f36527995e
2 changed files with 102 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
internal/glance/widget-rss.go
View File

@ -2,6 +2,7 @@ package glance
import (
"context"
"errors"
"fmt"
"html"
"html/template"
@ -25,18 +26,25 @@ var (
rssWidgetHorizontalCards2Template = mustParseTemplate("rss-horizontal-cards-2.html", "widget-base.html")
)
type cachedFeed struct {
LastModified time.Time
Etag string
Items rssFeedItemList
}
type rssWidget struct {
widgetBase `yaml:",inline"`
FeedRequests []rssFeedRequest `yaml:"feeds"`
Style string `yaml:"style"`
ThumbnailHeight float64 `yaml:"thumbnail-height"`
CardHeight float64 `yaml:"card-height"`
Items rssFeedItemList `yaml:"-"`
Limit int `yaml:"limit"`
CollapseAfter int `yaml:"collapse-after"`
SingleLineTitles bool `yaml:"single-line-titles"`
PreserveOrder bool `yaml:"preserve-order"`
NoItemsMessage string `yaml:"-"`
FeedRequests []rssFeedRequest `yaml:"feeds"`
Style string `yaml:"style"`
ThumbnailHeight float64 `yaml:"thumbnail-height"`
CardHeight float64 `yaml:"card-height"`
Items rssFeedItemList `yaml:"-"`
Limit int `yaml:"limit"`
CollapseAfter int `yaml:"collapse-after"`
SingleLineTitles bool `yaml:"single-line-titles"`
PreserveOrder bool `yaml:"preserve-order"`
NoItemsMessage string `yaml:"-"`
CachedFeeds map[string]cachedFeed `yaml:"-"`
}
func (widget *rssWidget) initialize() error {
@ -70,21 +78,41 @@ func (widget *rssWidget) initialize() error {
}
func (widget *rssWidget) update(ctx context.Context) {
items, err := fetchItemsFromRSSFeeds(widget.FeedRequests)
// Populate If-Modified-Since header and Etag
for i, req := range widget.FeedRequests {
if cachedFeed, ok := widget.CachedFeeds[req.URL]; ok {
widget.FeedRequests[i].IfModifiedSince = cachedFeed.LastModified
widget.FeedRequests[i].Etag = cachedFeed.Etag
}
}
allItems, feeds, err := fetchItemsFromRSSFeeds(widget.FeedRequests, widget.CachedFeeds)
if !widget.canContinueUpdateAfterHandlingErr(err) {
return
}
if !widget.PreserveOrder {
items.sortByNewest()
allItems.sortByNewest()
}
if len(items) > widget.Limit {
items = items[:widget.Limit]
if len(allItems) > widget.Limit {
allItems = allItems[:widget.Limit]
}
widget.Items = items
widget.Items = allItems
cachedFeeds := make(map[string]cachedFeed)
for _, feed := range feeds {
if !feed.LastModified.IsZero() || feed.Etag != "" {
cachedFeeds[feed.URL] = cachedFeed{
LastModified: feed.LastModified,
Etag: feed.Etag,
Items: feed.Items,
}
}
}
widget.CachedFeeds = cachedFeeds
}
func (widget *rssWidget) Render() template.HTML {
@ -152,10 +180,19 @@ type rssFeedRequest struct {
ItemLinkPrefix string `yaml:"item-link-prefix"`
Headers map[string]string `yaml:"headers"`
IsDetailed bool `yaml:"-"`
IfModifiedSince time.Time `yaml:"-"`
Etag string `yaml:"-"`
}
type rssFeedItemList []rssFeedItem
type rssFeedResponse struct {
URL string
Items rssFeedItemList
LastModified time.Time
Etag string
}
func (f rssFeedItemList) sortByNewest() rssFeedItemList {
sort.Slice(f, func(i, j int) bool {
return f[i].PublishedAt.After(f[j].PublishedAt)
@ -166,41 +203,67 @@ func (f rssFeedItemList) sortByNewest() rssFeedItemList {
var feedParser = gofeed.NewParser()
func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) {
func fetchItemsFromRSSFeedTask(request rssFeedRequest) (rssFeedResponse, error) {
feedResponse := rssFeedResponse{URL: request.URL}
req, err := http.NewRequest("GET", request.URL, nil)
if err != nil {
return nil, err
return feedResponse, err
}
req.Header.Add("User-Agent", fmt.Sprintf("Glance v%s", buildVersion))
for key, value := range request.Headers {
req.Header.Add(key, value)
}
if !request.IfModifiedSince.IsZero() {
req.Header.Add("If-Modified-Since", request.IfModifiedSince.Format(http.TimeFormat))
}
if request.Etag != "" {
req.Header.Add("If-None-Match", request.Etag)
}
resp, err := defaultHTTPClient.Do(req)
if err != nil {
return nil, err
return feedResponse, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotModified {
return feedResponse, errNotModified
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL)
return feedResponse, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
return feedResponse, err
}
feed, err := feedParser.ParseString(string(body))
if err != nil {
return nil, err
return feedResponse, err
}
if request.Limit > 0 && len(feed.Items) > request.Limit {
feed.Items = feed.Items[:request.Limit]
}
items := make(rssFeedItemList, 0, len(feed.Items))
items := make([]rssFeedItem, 0, len(feed.Items))
if lastModified := resp.Header.Get("Last-Modified"); lastModified != "" {
if t, err := time.Parse(http.TimeFormat, lastModified); err == nil {
feedResponse.LastModified = t
}
}
if etag := resp.Header.Get("Etag"); etag != "" {
feedResponse.Etag = etag
}
for i := range feed.Items {
item := feed.Items[i]
@ -289,7 +352,8 @@ func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) {
items = append(items, rssItem)
}
return items, nil
feedResponse.Items = items
return feedResponse, nil
}
func recursiveFindThumbnailInExtensions(extensions map[string][]gofeedext.Extension) string {
@ -322,33 +386,38 @@ func findThumbnailInItemExtensions(item *gofeed.Item) string {
return recursiveFindThumbnailInExtensions(media)
}
func fetchItemsFromRSSFeeds(requests []rssFeedRequest) (rssFeedItemList, error) {
func fetchItemsFromRSSFeeds(requests []rssFeedRequest, cachedFeeds map[string]cachedFeed) (rssFeedItemList, []rssFeedResponse, error) {
job := newJob(fetchItemsFromRSSFeedTask, requests).withWorkers(30)
feeds, errs, err := workerPoolDo(job)
if err != nil {
return nil, fmt.Errorf("%w: %v", errNoContent, err)
return nil, nil, fmt.Errorf("%w: %v", errNoContent, err)
}
failed := 0
notModified := 0
entries := make(rssFeedItemList, 0, len(feeds)*10)
for i := range feeds {
if errs[i] != nil {
if errs[i] == nil {
entries = append(entries, feeds[i].Items...)
} else if errors.Is(errs[i], errNotModified) {
notModified++
entries = append(entries, cachedFeeds[feeds[i].URL].Items...)
slog.Debug("Feed not modified", "url", requests[i].URL, "debug", errs[i])
} else {
failed++
slog.Error("Failed to get RSS feed", "url", requests[i].URL, "error", errs[i])
continue
}
entries = append(entries, feeds[i]...)
}
if failed == len(requests) {
return nil, errNoContent
return nil, nil, errNoContent
}
if failed > 0 {
return entries, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed)
return entries, feeds, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed)
}
return entries, nil
return entries, feeds, nil
}

1
internal/glance/widget-utils.go
View File

@ -19,6 +19,7 @@ import (
var (
errNoContent = errors.New("failed to retrieve any content")
errPartialContent = errors.New("failed to retrieve some of the content")
errNotModified = errors.New("content not modified")
)
const defaultClientTimeout = 5 * time.Second