Fetch posts via JSON endpoint

internal/feed/lobsters.go

@@ -1,10 +1,6 @@
 package feed
 
 import (
-	"context"
-	"fmt"
-	"github.com/mmcdole/gofeed"
-	"log/slog"
 	"net/http"
 	"strings"
 	"time"
@@ -25,89 +21,41 @@ type lobstersPostResponseJson struct {
 	SubmitterUser    string   `json:"submitter_user"`
 	UserIsAuthor     bool     `json:"user_is_author"`
 	Tags             []string `json:"tags"`
-	Comments         []struct {
-		ShortID        string `json:"short_id"`
-		ShortIDURL     string `json:"short_id_url"`
-		CreatedAt      string `json:"created_at"`
-		UpdatedAt      string `json:"updated_at"`
-		IsDeleted      bool   `json:"is_deleted"`
-		IsModerated    bool   `json:"is_moderated"`
-		Score          int    `json:"score"`
-		Flags          int    `json:"flags"`
-		ParentComment  any    `json:"parent_comment"`
-		Comment        string `json:"comment"`
-		CommentPlain   string `json:"comment_plain"`
-		URL            string `json:"url"`
-		Depth          int    `json:"depth"`
-		CommentingUser string `json:"commenting_user"`
-	} `json:"comments"`
 }
 
-var lobstersParser = gofeed.NewParser()
+type lobstersFeedResponseJson []lobstersPostResponseJson
 
-func getLobstersTopPostIds(feed string) ([]string, error) {
+func getLobstersPostsFromFeed(feedUrl string) (ForumPosts, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	request, err := http.NewRequest("GET", feedUrl, nil)
-	defer cancel()
-
-	request := &RSSFeedRequest{
-		Url:   feed,
-		Title: "Lobsters",
-	}
-
-	f, err := lobstersParser.ParseURLWithContext(request.Url, ctx)
-
-	if err != nil {
-		return nil, fmt.Errorf("%w: could not fetch posts from %s", ErrNoContent, feed)
-	}
-
-	postIds := make([]string, 0, len(f.Items))
-
-	for i := range f.Items {
-		postIds = append(postIds, f.Items[i].GUID)
-	}
-
-	return postIds, nil
-}
-
-func getLobstersPostsFromIds(postIds []string) (ForumPosts, error) {
-	requests := make([]*http.Request, len(postIds))
-
-	for i, id := range postIds {
-		request, _ := http.NewRequest("GET", id+".json", nil)
-		requests[i] = request
-	}
-
-	task := decodeJsonFromRequestTask[lobstersPostResponseJson](defaultClient)
-	job := newJob(task, requests).withWorkers(30)
-	results, errs, err := workerPoolDo(job)
 
 	if err != nil {
 		return nil, err
 	}
 
-	posts := make(ForumPosts, 0, len(postIds))
+	feed, err := decodeJsonFromRequest[lobstersFeedResponseJson](defaultClient, request)
 
-	for i := range results {
+	if err != nil {
-		if errs[i] != nil {
+		return nil, err
-			slog.Error("Failed to fetch or parse lobsters post", "error", errs[i], "url", requests[i].URL)
-			continue
 	}
 
-		tags := strings.Join(results[i].Tags, ",")
+	posts := make(ForumPosts, 0, len(feed))
+
+	for i := range feed {
+		tags := strings.Join(feed[i].Tags, ",")
 
 		if tags != "" {
 			tags = " [" + tags + "]"
 		}
 
-		createdAt, _ := time.Parse(time.RFC3339, results[i].CreatedAt)
+		createdAt, _ := time.Parse(time.RFC3339, feed[i].CreatedAt)
 
 		posts = append(posts, ForumPost{
-			Title:           results[i].Title + tags,
+			Title:           feed[i].Title + tags,
-			DiscussionUrl:   results[i].CommentsURL,
+			DiscussionUrl:   feed[i].CommentsURL,
-			TargetUrl:       results[i].URL,
+			TargetUrl:       feed[i].URL,
-			TargetUrlDomain: extractDomainFromUrl(results[i].URL),
+			TargetUrlDomain: extractDomainFromUrl(feed[i].URL),
-			CommentCount:    results[i].CommentCount,
+			CommentCount:    feed[i].CommentCount,
-			Score:           results[i].Score,
+			Score:           feed[i].Score,
 			TimePosted:      createdAt,
 		})
 	}
@@ -116,23 +64,15 @@ func getLobstersPostsFromIds(postIds []string) (ForumPosts, error) {
 		return nil, ErrNoContent
 	}
 
-	if len(posts) != len(postIds) {
-		return posts, fmt.Errorf("%w could not fetch some lobsters posts", ErrPartialContent)
-	}
-
 	return posts, nil
 }
 
-func FetchLobstersTopPosts(feed string, limit int) (ForumPosts, error) {
+func FetchLobstersTopPosts(feedUrl string) (ForumPosts, error) {
-	postIds, err := getLobstersTopPostIds(feed)
+	posts, err := getLobstersPostsFromFeed(feedUrl)
 
 	if err != nil {
 		return nil, err
 	}
 
-	if len(postIds) > limit {
+	return posts, nil
-		postIds = postIds[:limit]
-	}
-
-	return getLobstersPostsFromIds(postIds)
 }

internal/widget/lobsters.go

@@ -11,7 +11,7 @@ import (
 
 type Lobsters struct {
 	widgetBase     `yaml:",inline"`
-	Feed           string          `yaml:"feed"`
+	FeedUrl        string          `yaml:"feed"`
 	Posts          feed.ForumPosts `yaml:"-"`
 	Limit          int             `yaml:"limit"`
 	CollapseAfter  int             `yaml:"collapse-after"`
@@ -21,8 +21,8 @@ type Lobsters struct {
 func (widget *Lobsters) Initialize() error {
 	widget.withTitle("Lobsters").withCacheDuration(30 * time.Minute)
 
-	if widget.Feed == "" {
+	if widget.FeedUrl == "" {
-		widget.Feed = "https://lobste.rs/rss"
+		widget.FeedUrl = "https://lobste.rs/active.json"
 	}
 
 	if widget.Limit <= 0 {
@@ -37,7 +37,7 @@ func (widget *Lobsters) Initialize() error {
 }
 
 func (widget *Lobsters) Update(ctx context.Context) {
-	posts, err := feed.FetchLobstersTopPosts(widget.Feed, widget.Limit)
+	posts, err := feed.FetchLobstersTopPosts(widget.FeedUrl)
 
 	if !widget.canContinueUpdateAfterHandlingErr(err) {
 		return