Fetch posts via JSON endpoint

This commit is contained in:
Jonas Knobloch 2024-05-12 17:14:04 +02:00
parent 344f518991
commit c05607934b
2 changed files with 25 additions and 85 deletions

View File

@ -1,10 +1,6 @@
package feed package feed
import ( import (
"context"
"fmt"
"github.com/mmcdole/gofeed"
"log/slog"
"net/http" "net/http"
"strings" "strings"
"time" "time"
@ -25,89 +21,41 @@ type lobstersPostResponseJson struct {
SubmitterUser string `json:"submitter_user"` SubmitterUser string `json:"submitter_user"`
UserIsAuthor bool `json:"user_is_author"` UserIsAuthor bool `json:"user_is_author"`
Tags []string `json:"tags"` Tags []string `json:"tags"`
Comments []struct {
ShortID string `json:"short_id"`
ShortIDURL string `json:"short_id_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
IsDeleted bool `json:"is_deleted"`
IsModerated bool `json:"is_moderated"`
Score int `json:"score"`
Flags int `json:"flags"`
ParentComment any `json:"parent_comment"`
Comment string `json:"comment"`
CommentPlain string `json:"comment_plain"`
URL string `json:"url"`
Depth int `json:"depth"`
CommentingUser string `json:"commenting_user"`
} `json:"comments"`
} }
var lobstersParser = gofeed.NewParser() type lobstersFeedResponseJson []lobstersPostResponseJson
func getLobstersTopPostIds(feed string) ([]string, error) { func getLobstersPostsFromFeed(feedUrl string) (ForumPosts, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) request, err := http.NewRequest("GET", feedUrl, nil)
defer cancel()
request := &RSSFeedRequest{
Url: feed,
Title: "Lobsters",
}
f, err := lobstersParser.ParseURLWithContext(request.Url, ctx)
if err != nil {
return nil, fmt.Errorf("%w: could not fetch posts from %s", ErrNoContent, feed)
}
postIds := make([]string, 0, len(f.Items))
for i := range f.Items {
postIds = append(postIds, f.Items[i].GUID)
}
return postIds, nil
}
func getLobstersPostsFromIds(postIds []string) (ForumPosts, error) {
requests := make([]*http.Request, len(postIds))
for i, id := range postIds {
request, _ := http.NewRequest("GET", id+".json", nil)
requests[i] = request
}
task := decodeJsonFromRequestTask[lobstersPostResponseJson](defaultClient)
job := newJob(task, requests).withWorkers(30)
results, errs, err := workerPoolDo(job)
if err != nil { if err != nil {
return nil, err return nil, err
} }
posts := make(ForumPosts, 0, len(postIds)) feed, err := decodeJsonFromRequest[lobstersFeedResponseJson](defaultClient, request)
for i := range results { if err != nil {
if errs[i] != nil { return nil, err
slog.Error("Failed to fetch or parse lobsters post", "error", errs[i], "url", requests[i].URL)
continue
} }
tags := strings.Join(results[i].Tags, ",") posts := make(ForumPosts, 0, len(feed))
for i := range feed {
tags := strings.Join(feed[i].Tags, ",")
if tags != "" { if tags != "" {
tags = " [" + tags + "]" tags = " [" + tags + "]"
} }
createdAt, _ := time.Parse(time.RFC3339, results[i].CreatedAt) createdAt, _ := time.Parse(time.RFC3339, feed[i].CreatedAt)
posts = append(posts, ForumPost{ posts = append(posts, ForumPost{
Title: results[i].Title + tags, Title: feed[i].Title + tags,
DiscussionUrl: results[i].CommentsURL, DiscussionUrl: feed[i].CommentsURL,
TargetUrl: results[i].URL, TargetUrl: feed[i].URL,
TargetUrlDomain: extractDomainFromUrl(results[i].URL), TargetUrlDomain: extractDomainFromUrl(feed[i].URL),
CommentCount: results[i].CommentCount, CommentCount: feed[i].CommentCount,
Score: results[i].Score, Score: feed[i].Score,
TimePosted: createdAt, TimePosted: createdAt,
}) })
} }
@ -116,23 +64,15 @@ func getLobstersPostsFromIds(postIds []string) (ForumPosts, error) {
return nil, ErrNoContent return nil, ErrNoContent
} }
if len(posts) != len(postIds) {
return posts, fmt.Errorf("%w could not fetch some lobsters posts", ErrPartialContent)
}
return posts, nil return posts, nil
} }
func FetchLobstersTopPosts(feed string, limit int) (ForumPosts, error) { func FetchLobstersTopPosts(feedUrl string) (ForumPosts, error) {
postIds, err := getLobstersTopPostIds(feed) posts, err := getLobstersPostsFromFeed(feedUrl)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(postIds) > limit { return posts, nil
postIds = postIds[:limit]
}
return getLobstersPostsFromIds(postIds)
} }

View File

@ -11,7 +11,7 @@ import (
type Lobsters struct { type Lobsters struct {
widgetBase `yaml:",inline"` widgetBase `yaml:",inline"`
Feed string `yaml:"feed"` FeedUrl string `yaml:"feed"`
Posts feed.ForumPosts `yaml:"-"` Posts feed.ForumPosts `yaml:"-"`
Limit int `yaml:"limit"` Limit int `yaml:"limit"`
CollapseAfter int `yaml:"collapse-after"` CollapseAfter int `yaml:"collapse-after"`
@ -21,8 +21,8 @@ type Lobsters struct {
func (widget *Lobsters) Initialize() error { func (widget *Lobsters) Initialize() error {
widget.withTitle("Lobsters").withCacheDuration(30 * time.Minute) widget.withTitle("Lobsters").withCacheDuration(30 * time.Minute)
if widget.Feed == "" { if widget.FeedUrl == "" {
widget.Feed = "https://lobste.rs/rss" widget.FeedUrl = "https://lobste.rs/active.json"
} }
if widget.Limit <= 0 { if widget.Limit <= 0 {
@ -37,7 +37,7 @@ func (widget *Lobsters) Initialize() error {
} }
func (widget *Lobsters) Update(ctx context.Context) { func (widget *Lobsters) Update(ctx context.Context) {
posts, err := feed.FetchLobstersTopPosts(widget.Feed, widget.Limit) posts, err := feed.FetchLobstersTopPosts(widget.FeedUrl)
if !widget.canContinueUpdateAfterHandlingErr(err) { if !widget.canContinueUpdateAfterHandlingErr(err) {
return return