mirror of
https://github.com/glanceapp/glance.git
synced 2025-06-21 10:27:45 +02:00
feat: Use conditional requests for RSS feeds
This commit is contained in:
parent
0b2cfc45c1
commit
f36527995e
@ -2,6 +2,7 @@ package glance
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"html"
|
"html"
|
||||||
"html/template"
|
"html/template"
|
||||||
@ -25,18 +26,25 @@ var (
|
|||||||
rssWidgetHorizontalCards2Template = mustParseTemplate("rss-horizontal-cards-2.html", "widget-base.html")
|
rssWidgetHorizontalCards2Template = mustParseTemplate("rss-horizontal-cards-2.html", "widget-base.html")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type cachedFeed struct {
|
||||||
|
LastModified time.Time
|
||||||
|
Etag string
|
||||||
|
Items rssFeedItemList
|
||||||
|
}
|
||||||
|
|
||||||
type rssWidget struct {
|
type rssWidget struct {
|
||||||
widgetBase `yaml:",inline"`
|
widgetBase `yaml:",inline"`
|
||||||
FeedRequests []rssFeedRequest `yaml:"feeds"`
|
FeedRequests []rssFeedRequest `yaml:"feeds"`
|
||||||
Style string `yaml:"style"`
|
Style string `yaml:"style"`
|
||||||
ThumbnailHeight float64 `yaml:"thumbnail-height"`
|
ThumbnailHeight float64 `yaml:"thumbnail-height"`
|
||||||
CardHeight float64 `yaml:"card-height"`
|
CardHeight float64 `yaml:"card-height"`
|
||||||
Items rssFeedItemList `yaml:"-"`
|
Items rssFeedItemList `yaml:"-"`
|
||||||
Limit int `yaml:"limit"`
|
Limit int `yaml:"limit"`
|
||||||
CollapseAfter int `yaml:"collapse-after"`
|
CollapseAfter int `yaml:"collapse-after"`
|
||||||
SingleLineTitles bool `yaml:"single-line-titles"`
|
SingleLineTitles bool `yaml:"single-line-titles"`
|
||||||
PreserveOrder bool `yaml:"preserve-order"`
|
PreserveOrder bool `yaml:"preserve-order"`
|
||||||
NoItemsMessage string `yaml:"-"`
|
NoItemsMessage string `yaml:"-"`
|
||||||
|
CachedFeeds map[string]cachedFeed `yaml:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (widget *rssWidget) initialize() error {
|
func (widget *rssWidget) initialize() error {
|
||||||
@ -70,21 +78,41 @@ func (widget *rssWidget) initialize() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (widget *rssWidget) update(ctx context.Context) {
|
func (widget *rssWidget) update(ctx context.Context) {
|
||||||
items, err := fetchItemsFromRSSFeeds(widget.FeedRequests)
|
// Populate If-Modified-Since header and Etag
|
||||||
|
for i, req := range widget.FeedRequests {
|
||||||
|
if cachedFeed, ok := widget.CachedFeeds[req.URL]; ok {
|
||||||
|
widget.FeedRequests[i].IfModifiedSince = cachedFeed.LastModified
|
||||||
|
widget.FeedRequests[i].Etag = cachedFeed.Etag
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
allItems, feeds, err := fetchItemsFromRSSFeeds(widget.FeedRequests, widget.CachedFeeds)
|
||||||
|
|
||||||
if !widget.canContinueUpdateAfterHandlingErr(err) {
|
if !widget.canContinueUpdateAfterHandlingErr(err) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !widget.PreserveOrder {
|
if !widget.PreserveOrder {
|
||||||
items.sortByNewest()
|
allItems.sortByNewest()
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(items) > widget.Limit {
|
if len(allItems) > widget.Limit {
|
||||||
items = items[:widget.Limit]
|
allItems = allItems[:widget.Limit]
|
||||||
}
|
}
|
||||||
|
|
||||||
widget.Items = items
|
widget.Items = allItems
|
||||||
|
|
||||||
|
cachedFeeds := make(map[string]cachedFeed)
|
||||||
|
for _, feed := range feeds {
|
||||||
|
if !feed.LastModified.IsZero() || feed.Etag != "" {
|
||||||
|
cachedFeeds[feed.URL] = cachedFeed{
|
||||||
|
LastModified: feed.LastModified,
|
||||||
|
Etag: feed.Etag,
|
||||||
|
Items: feed.Items,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
widget.CachedFeeds = cachedFeeds
|
||||||
}
|
}
|
||||||
|
|
||||||
func (widget *rssWidget) Render() template.HTML {
|
func (widget *rssWidget) Render() template.HTML {
|
||||||
@ -152,10 +180,19 @@ type rssFeedRequest struct {
|
|||||||
ItemLinkPrefix string `yaml:"item-link-prefix"`
|
ItemLinkPrefix string `yaml:"item-link-prefix"`
|
||||||
Headers map[string]string `yaml:"headers"`
|
Headers map[string]string `yaml:"headers"`
|
||||||
IsDetailed bool `yaml:"-"`
|
IsDetailed bool `yaml:"-"`
|
||||||
|
IfModifiedSince time.Time `yaml:"-"`
|
||||||
|
Etag string `yaml:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type rssFeedItemList []rssFeedItem
|
type rssFeedItemList []rssFeedItem
|
||||||
|
|
||||||
|
type rssFeedResponse struct {
|
||||||
|
URL string
|
||||||
|
Items rssFeedItemList
|
||||||
|
LastModified time.Time
|
||||||
|
Etag string
|
||||||
|
}
|
||||||
|
|
||||||
func (f rssFeedItemList) sortByNewest() rssFeedItemList {
|
func (f rssFeedItemList) sortByNewest() rssFeedItemList {
|
||||||
sort.Slice(f, func(i, j int) bool {
|
sort.Slice(f, func(i, j int) bool {
|
||||||
return f[i].PublishedAt.After(f[j].PublishedAt)
|
return f[i].PublishedAt.After(f[j].PublishedAt)
|
||||||
@ -166,41 +203,67 @@ func (f rssFeedItemList) sortByNewest() rssFeedItemList {
|
|||||||
|
|
||||||
var feedParser = gofeed.NewParser()
|
var feedParser = gofeed.NewParser()
|
||||||
|
|
||||||
func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) {
|
func fetchItemsFromRSSFeedTask(request rssFeedRequest) (rssFeedResponse, error) {
|
||||||
|
feedResponse := rssFeedResponse{URL: request.URL}
|
||||||
|
|
||||||
req, err := http.NewRequest("GET", request.URL, nil)
|
req, err := http.NewRequest("GET", request.URL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return feedResponse, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
req.Header.Add("User-Agent", fmt.Sprintf("Glance v%s", buildVersion))
|
||||||
|
|
||||||
for key, value := range request.Headers {
|
for key, value := range request.Headers {
|
||||||
req.Header.Add(key, value)
|
req.Header.Add(key, value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !request.IfModifiedSince.IsZero() {
|
||||||
|
req.Header.Add("If-Modified-Since", request.IfModifiedSince.Format(http.TimeFormat))
|
||||||
|
}
|
||||||
|
|
||||||
|
if request.Etag != "" {
|
||||||
|
req.Header.Add("If-None-Match", request.Etag)
|
||||||
|
}
|
||||||
|
|
||||||
resp, err := defaultHTTPClient.Do(req)
|
resp, err := defaultHTTPClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return feedResponse, err
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode == http.StatusNotModified {
|
||||||
|
return feedResponse, errNotModified
|
||||||
|
}
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL)
|
return feedResponse, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL)
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
body, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return feedResponse, err
|
||||||
}
|
}
|
||||||
|
|
||||||
feed, err := feedParser.ParseString(string(body))
|
feed, err := feedParser.ParseString(string(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return feedResponse, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if request.Limit > 0 && len(feed.Items) > request.Limit {
|
if request.Limit > 0 && len(feed.Items) > request.Limit {
|
||||||
feed.Items = feed.Items[:request.Limit]
|
feed.Items = feed.Items[:request.Limit]
|
||||||
}
|
}
|
||||||
|
|
||||||
items := make(rssFeedItemList, 0, len(feed.Items))
|
items := make([]rssFeedItem, 0, len(feed.Items))
|
||||||
|
|
||||||
|
if lastModified := resp.Header.Get("Last-Modified"); lastModified != "" {
|
||||||
|
if t, err := time.Parse(http.TimeFormat, lastModified); err == nil {
|
||||||
|
feedResponse.LastModified = t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if etag := resp.Header.Get("Etag"); etag != "" {
|
||||||
|
feedResponse.Etag = etag
|
||||||
|
}
|
||||||
|
|
||||||
for i := range feed.Items {
|
for i := range feed.Items {
|
||||||
item := feed.Items[i]
|
item := feed.Items[i]
|
||||||
@ -289,7 +352,8 @@ func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) {
|
|||||||
items = append(items, rssItem)
|
items = append(items, rssItem)
|
||||||
}
|
}
|
||||||
|
|
||||||
return items, nil
|
feedResponse.Items = items
|
||||||
|
return feedResponse, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func recursiveFindThumbnailInExtensions(extensions map[string][]gofeedext.Extension) string {
|
func recursiveFindThumbnailInExtensions(extensions map[string][]gofeedext.Extension) string {
|
||||||
@ -322,33 +386,38 @@ func findThumbnailInItemExtensions(item *gofeed.Item) string {
|
|||||||
return recursiveFindThumbnailInExtensions(media)
|
return recursiveFindThumbnailInExtensions(media)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchItemsFromRSSFeeds(requests []rssFeedRequest) (rssFeedItemList, error) {
|
func fetchItemsFromRSSFeeds(requests []rssFeedRequest, cachedFeeds map[string]cachedFeed) (rssFeedItemList, []rssFeedResponse, error) {
|
||||||
job := newJob(fetchItemsFromRSSFeedTask, requests).withWorkers(30)
|
job := newJob(fetchItemsFromRSSFeedTask, requests).withWorkers(30)
|
||||||
feeds, errs, err := workerPoolDo(job)
|
feeds, errs, err := workerPoolDo(job)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %v", errNoContent, err)
|
return nil, nil, fmt.Errorf("%w: %v", errNoContent, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
failed := 0
|
failed := 0
|
||||||
|
notModified := 0
|
||||||
|
|
||||||
entries := make(rssFeedItemList, 0, len(feeds)*10)
|
entries := make(rssFeedItemList, 0, len(feeds)*10)
|
||||||
|
|
||||||
for i := range feeds {
|
for i := range feeds {
|
||||||
if errs[i] != nil {
|
if errs[i] == nil {
|
||||||
|
entries = append(entries, feeds[i].Items...)
|
||||||
|
} else if errors.Is(errs[i], errNotModified) {
|
||||||
|
notModified++
|
||||||
|
entries = append(entries, cachedFeeds[feeds[i].URL].Items...)
|
||||||
|
slog.Debug("Feed not modified", "url", requests[i].URL, "debug", errs[i])
|
||||||
|
} else {
|
||||||
failed++
|
failed++
|
||||||
slog.Error("Failed to get RSS feed", "url", requests[i].URL, "error", errs[i])
|
slog.Error("Failed to get RSS feed", "url", requests[i].URL, "error", errs[i])
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
entries = append(entries, feeds[i]...)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if failed == len(requests) {
|
if failed == len(requests) {
|
||||||
return nil, errNoContent
|
return nil, nil, errNoContent
|
||||||
}
|
}
|
||||||
|
|
||||||
if failed > 0 {
|
if failed > 0 {
|
||||||
return entries, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed)
|
return entries, feeds, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed)
|
||||||
}
|
}
|
||||||
|
|
||||||
return entries, nil
|
return entries, feeds, nil
|
||||||
}
|
}
|
||||||
|
@ -19,6 +19,7 @@ import (
|
|||||||
var (
|
var (
|
||||||
errNoContent = errors.New("failed to retrieve any content")
|
errNoContent = errors.New("failed to retrieve any content")
|
||||||
errPartialContent = errors.New("failed to retrieve some of the content")
|
errPartialContent = errors.New("failed to retrieve some of the content")
|
||||||
|
errNotModified = errors.New("content not modified")
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultClientTimeout = 5 * time.Second
|
const defaultClientTimeout = 5 * time.Second
|
||||||
|
Loading…
x
Reference in New Issue
Block a user