Add filter on already seen links for RSS feeds

This commit is contained in:
Lucas L. 2025-03-22 19:59:00 +01:00
parent 83f0d16904
commit 18241136e3

View File

@ -331,6 +331,7 @@ func fetchItemsFromRSSFeeds(requests []rssFeedRequest) (rssFeedItemList, error)
failed := 0 failed := 0
entries := make(rssFeedItemList, 0, len(feeds)*10) entries := make(rssFeedItemList, 0, len(feeds)*10)
seen := make(map[string]struct{})
for i := range feeds { for i := range feeds {
if errs[i] != nil { if errs[i] != nil {
@ -339,7 +340,13 @@ func fetchItemsFromRSSFeeds(requests []rssFeedRequest) (rssFeedItemList, error)
continue continue
} }
entries = append(entries, feeds[i]...) for _, item := range feeds[i] {
if _, exists := seen[item.Link]; exists {
continue
}
entries = append(entries, item)
seen[item.Link] = struct{}{}
}
} }
if failed == len(requests) { if failed == len(requests) {