diff --git a/internal/feed/lobsters.go b/internal/feed/lobsters.go index b503492..d18b607 100644 --- a/internal/feed/lobsters.go +++ b/internal/feed/lobsters.go @@ -1,10 +1,6 @@ package feed import ( - "context" - "fmt" - "github.com/mmcdole/gofeed" - "log/slog" "net/http" "strings" "time" @@ -25,89 +21,41 @@ type lobstersPostResponseJson struct { SubmitterUser string `json:"submitter_user"` UserIsAuthor bool `json:"user_is_author"` Tags []string `json:"tags"` - Comments []struct { - ShortID string `json:"short_id"` - ShortIDURL string `json:"short_id_url"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - IsDeleted bool `json:"is_deleted"` - IsModerated bool `json:"is_moderated"` - Score int `json:"score"` - Flags int `json:"flags"` - ParentComment any `json:"parent_comment"` - Comment string `json:"comment"` - CommentPlain string `json:"comment_plain"` - URL string `json:"url"` - Depth int `json:"depth"` - CommentingUser string `json:"commenting_user"` - } `json:"comments"` } -var lobstersParser = gofeed.NewParser() +type lobstersFeedResponseJson []lobstersPostResponseJson -func getLobstersTopPostIds(feed string) ([]string, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - - request := &RSSFeedRequest{ - Url: feed, - Title: "Lobsters", - } - - f, err := lobstersParser.ParseURLWithContext(request.Url, ctx) - - if err != nil { - return nil, fmt.Errorf("%w: could not fetch posts from %s", ErrNoContent, feed) - } - - postIds := make([]string, 0, len(f.Items)) - - for i := range f.Items { - postIds = append(postIds, f.Items[i].GUID) - } - - return postIds, nil -} - -func getLobstersPostsFromIds(postIds []string) (ForumPosts, error) { - requests := make([]*http.Request, len(postIds)) - - for i, id := range postIds { - request, _ := http.NewRequest("GET", id+".json", nil) - requests[i] = request - } - - task := decodeJsonFromRequestTask[lobstersPostResponseJson](defaultClient) - job := newJob(task, requests).withWorkers(30) - results, errs, err := workerPoolDo(job) +func getLobstersPostsFromFeed(feedUrl string) (ForumPosts, error) { + request, err := http.NewRequest("GET", feedUrl, nil) if err != nil { return nil, err } - posts := make(ForumPosts, 0, len(postIds)) + feed, err := decodeJsonFromRequest[lobstersFeedResponseJson](defaultClient, request) - for i := range results { - if errs[i] != nil { - slog.Error("Failed to fetch or parse lobsters post", "error", errs[i], "url", requests[i].URL) - continue - } + if err != nil { + return nil, err + } - tags := strings.Join(results[i].Tags, ",") + posts := make(ForumPosts, 0, len(feed)) + + for i := range feed { + tags := strings.Join(feed[i].Tags, ",") if tags != "" { tags = " [" + tags + "]" } - createdAt, _ := time.Parse(time.RFC3339, results[i].CreatedAt) + createdAt, _ := time.Parse(time.RFC3339, feed[i].CreatedAt) posts = append(posts, ForumPost{ - Title: results[i].Title + tags, - DiscussionUrl: results[i].CommentsURL, - TargetUrl: results[i].URL, - TargetUrlDomain: extractDomainFromUrl(results[i].URL), - CommentCount: results[i].CommentCount, - Score: results[i].Score, + Title: feed[i].Title + tags, + DiscussionUrl: feed[i].CommentsURL, + TargetUrl: feed[i].URL, + TargetUrlDomain: extractDomainFromUrl(feed[i].URL), + CommentCount: feed[i].CommentCount, + Score: feed[i].Score, TimePosted: createdAt, }) } @@ -116,23 +64,15 @@ func getLobstersPostsFromIds(postIds []string) (ForumPosts, error) { return nil, ErrNoContent } - if len(posts) != len(postIds) { - return posts, fmt.Errorf("%w could not fetch some lobsters posts", ErrPartialContent) - } - return posts, nil } -func FetchLobstersTopPosts(feed string, limit int) (ForumPosts, error) { - postIds, err := getLobstersTopPostIds(feed) +func FetchLobstersTopPosts(feedUrl string) (ForumPosts, error) { + posts, err := getLobstersPostsFromFeed(feedUrl) if err != nil { return nil, err } - if len(postIds) > limit { - postIds = postIds[:limit] - } - - return getLobstersPostsFromIds(postIds) + return posts, nil } diff --git a/internal/widget/lobsters.go b/internal/widget/lobsters.go index 1e91613..1b508d7 100644 --- a/internal/widget/lobsters.go +++ b/internal/widget/lobsters.go @@ -11,7 +11,7 @@ import ( type Lobsters struct { widgetBase `yaml:",inline"` - Feed string `yaml:"feed"` + FeedUrl string `yaml:"feed"` Posts feed.ForumPosts `yaml:"-"` Limit int `yaml:"limit"` CollapseAfter int `yaml:"collapse-after"` @@ -21,8 +21,8 @@ type Lobsters struct { func (widget *Lobsters) Initialize() error { widget.withTitle("Lobsters").withCacheDuration(30 * time.Minute) - if widget.Feed == "" { - widget.Feed = "https://lobste.rs/rss" + if widget.FeedUrl == "" { + widget.FeedUrl = "https://lobste.rs/active.json" } if widget.Limit <= 0 { @@ -37,7 +37,7 @@ func (widget *Lobsters) Initialize() error { } func (widget *Lobsters) Update(ctx context.Context) { - posts, err := feed.FetchLobstersTopPosts(widget.Feed, widget.Limit) + posts, err := feed.FetchLobstersTopPosts(widget.FeedUrl) if !widget.canContinueUpdateAfterHandlingErr(err) { return