Allow fetching releases from multiple sources

This commit is contained in:
Svilen Markov
2024-08-27 03:26:16 +01:00
parent 7d1ede8c91
commit 01af97ddab
13 changed files with 474 additions and 367 deletions

View File

@ -0,0 +1,58 @@
package feed
import (
"fmt"
"net/http"
"strings"
)
type dockerHubRepositoryTagsResponse struct {
Results []struct {
Name string `json:"name"`
LastPushed string `json:"tag_last_pushed"`
} `json:"results"`
}
const dockerHubReleaseNotesURLFormat = "https://hub.docker.com/r/%s/tags?name=%s"
func fetchLatestDockerHubRelease(request *ReleaseRequest) (*AppRelease, error) {
parts := strings.Split(request.Repository, "/")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid repository name: %s", request.Repository)
}
httpRequest, err := http.NewRequest(
"GET",
fmt.Sprintf("https://hub.docker.com/v2/namespaces/%s/repositories/%s/tags", parts[0], parts[1]),
nil,
)
if err != nil {
return nil, err
}
if request.Token != nil {
httpRequest.Header.Add("Authorization", "Bearer "+(*request.Token))
}
response, err := decodeJsonFromRequest[dockerHubRepositoryTagsResponse](defaultClient, httpRequest)
if err != nil {
return nil, err
}
if len(response.Results) == 0 {
return nil, fmt.Errorf("no tags found for repository: %s", request.Repository)
}
tag := response.Results[0]
return &AppRelease{
Source: ReleaseSourceDockerHub,
NotesUrl: fmt.Sprintf(dockerHubReleaseNotesURLFormat, request.Repository, tag.Name),
Name: request.Repository,
Version: tag.Name,
TimeReleased: parseRFC3339Time(tag.LastPushed),
}, nil
}

View File

@ -1,344 +0,0 @@
package feed
import (
"errors"
"fmt"
"log/slog"
"net/http"
"net/url"
"sync"
"time"
)
type githubReleaseLatestResponseJson struct {
TagName string `json:"tag_name"`
PublishedAt string `json:"published_at"`
HtmlUrl string `json:"html_url"`
Reactions struct {
Downvotes int `json:"-1"`
} `json:"reactions"`
}
type gitlabReleaseResponseJson struct {
TagName string `json:"tag_name"`
PublishedAt string `json:"created_at"`
Links struct {
Self string `json:"self"`
} `json:"_links"`
Draft bool `json:"draft"`
PreRelease bool `json:"prerelease"`
Reactions struct {
Downvotes int `json:"-1"`
} `json:"reactions"`
}
func parseGithubTime(t string) time.Time {
parsedTime, err := time.Parse("2006-01-02T15:04:05Z", t)
if err != nil {
return time.Now()
}
return parsedTime
}
func FetchLatestReleasesFromGitForge(repositories []string, token string, source string) (AppReleases, error) {
switch source {
case "github":
return fetchLatestReleasesFromGithub(repositories, token)
case "gitlab":
return fetchLatestReleasesFromGitlab(repositories, token)
default:
return nil, errors.New(fmt.Sprintf("Release source %s is invalid", source))
}
}
func fetchLatestReleasesFromGitlab(repositories []string, token string) (AppReleases, error) {
appReleases := make(AppReleases, 0, len(repositories))
if len(repositories) == 0 {
return appReleases, nil
}
requests := make([]*http.Request, len(repositories))
for i, repository := range repositories {
request, _ := http.NewRequest("GET", fmt.Sprintf("https://gitlab.com/api/v4/projects/%s/releases/", url.QueryEscape(repository)), nil)
if token != "" {
request.Header.Add("PRIVATE-TOKEN", token)
}
requests[i] = request
}
task := decodeJsonFromRequestTask[[]gitlabReleaseResponseJson](defaultClient)
job := newJob(task, requests).withWorkers(15)
responses, errs, err := workerPoolDo(job)
if err != nil {
return nil, err
}
var failed int
for i := range responses {
if errs[i] != nil {
failed++
slog.Error("Failed to fetch or parse gitlab release", "error", errs[i], "url", requests[i].URL)
continue
}
releases := responses[i]
if len(releases) < 1 {
failed++
slog.Error("No releases found", "repository", repositories[i], "url", requests[i].URL)
continue
}
var liveRelease *gitlabReleaseResponseJson
for i := range releases {
release := &releases[i]
if !release.Draft && !release.PreRelease {
liveRelease = release
break
}
}
if liveRelease == nil {
slog.Error("No live release found", "repository", repositories[i], "url", requests[i].URL)
continue
}
version := liveRelease.TagName
if version[0] != 'v' {
version = "v" + version
}
appReleases = append(appReleases, AppRelease{
Name: repositories[i],
Version: version,
NotesUrl: liveRelease.Links.Self,
TimeReleased: parseGithubTime(liveRelease.PublishedAt),
Downvotes: liveRelease.Reactions.Downvotes,
})
}
if len(appReleases) == 0 {
return nil, ErrNoContent
}
appReleases.SortByNewest()
if failed > 0 {
return appReleases, fmt.Errorf("%w: could not get %d releases", ErrPartialContent, failed)
}
return appReleases, nil
}
func fetchLatestReleasesFromGithub(repositories []string, token string) (AppReleases, error) {
appReleases := make(AppReleases, 0, len(repositories))
if len(repositories) == 0 {
return appReleases, nil
}
requests := make([]*http.Request, len(repositories))
for i, repository := range repositories {
request, _ := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", repository), nil)
if token != "" {
request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
}
requests[i] = request
}
task := decodeJsonFromRequestTask[githubReleaseLatestResponseJson](defaultClient)
job := newJob(task, requests).withWorkers(15)
responses, errs, err := workerPoolDo(job)
if err != nil {
return nil, err
}
var failed int
for i := range responses {
if errs[i] != nil {
failed++
slog.Error("Failed to fetch or parse github release", "error", errs[i], "url", requests[i].URL)
continue
}
liveRelease := &responses[i]
if liveRelease == nil {
slog.Error("No live release found", "repository", repositories[i], "url", requests[i].URL)
continue
}
version := liveRelease.TagName
if version[0] != 'v' {
version = "v" + version
}
appReleases = append(appReleases, AppRelease{
Name: repositories[i],
Version: version,
NotesUrl: liveRelease.HtmlUrl,
TimeReleased: parseGithubTime(liveRelease.PublishedAt),
Downvotes: liveRelease.Reactions.Downvotes,
})
}
if len(appReleases) == 0 {
return nil, ErrNoContent
}
appReleases.SortByNewest()
if failed > 0 {
return appReleases, fmt.Errorf("%w: could not get %d releases", ErrPartialContent, failed)
}
return appReleases, nil
}
type GithubTicket struct {
Number int
CreatedAt time.Time
Title string
}
type RepositoryDetails struct {
Name string
Stars int
Forks int
OpenPullRequests int
PullRequests []GithubTicket
OpenIssues int
Issues []GithubTicket
}
type githubRepositoryDetailsResponseJson struct {
Name string `json:"full_name"`
Stars int `json:"stargazers_count"`
Forks int `json:"forks_count"`
}
type githubTicketResponseJson struct {
Count int `json:"total_count"`
Tickets []struct {
Number int `json:"number"`
CreatedAt string `json:"created_at"`
Title string `json:"title"`
} `json:"items"`
}
func FetchRepositoryDetailsFromGithub(repository string, token string, maxPRs int, maxIssues int) (RepositoryDetails, error) {
repositoryRequest, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s", repository), nil)
if err != nil {
return RepositoryDetails{}, fmt.Errorf("%w: could not create request with repository: %v", ErrNoContent, err)
}
PRsRequest, _ := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/search/issues?q=is:pr+is:open+repo:%s&per_page=%d", repository, maxPRs), nil)
issuesRequest, _ := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/search/issues?q=is:issue+is:open+repo:%s&per_page=%d", repository, maxIssues), nil)
if token != "" {
token = fmt.Sprintf("Bearer %s", token)
repositoryRequest.Header.Add("Authorization", token)
PRsRequest.Header.Add("Authorization", token)
issuesRequest.Header.Add("Authorization", token)
}
var detailsResponse githubRepositoryDetailsResponseJson
var detailsErr error
var PRsResponse githubTicketResponseJson
var PRsErr error
var issuesResponse githubTicketResponseJson
var issuesErr error
var wg sync.WaitGroup
wg.Add(1)
go (func() {
defer wg.Done()
detailsResponse, detailsErr = decodeJsonFromRequest[githubRepositoryDetailsResponseJson](defaultClient, repositoryRequest)
})()
if maxPRs > 0 {
wg.Add(1)
go (func() {
defer wg.Done()
PRsResponse, PRsErr = decodeJsonFromRequest[githubTicketResponseJson](defaultClient, PRsRequest)
})()
}
if maxIssues > 0 {
wg.Add(1)
go (func() {
defer wg.Done()
issuesResponse, issuesErr = decodeJsonFromRequest[githubTicketResponseJson](defaultClient, issuesRequest)
})()
}
wg.Wait()
if detailsErr != nil {
return RepositoryDetails{}, fmt.Errorf("%w: could not get repository details: %s", ErrNoContent, detailsErr)
}
details := RepositoryDetails{
Name: detailsResponse.Name,
Stars: detailsResponse.Stars,
Forks: detailsResponse.Forks,
PullRequests: make([]GithubTicket, 0, len(PRsResponse.Tickets)),
Issues: make([]GithubTicket, 0, len(issuesResponse.Tickets)),
}
err = nil
if maxPRs > 0 {
if PRsErr != nil {
err = fmt.Errorf("%w: could not get PRs: %s", ErrPartialContent, PRsErr)
} else {
details.OpenPullRequests = PRsResponse.Count
for i := range PRsResponse.Tickets {
details.PullRequests = append(details.PullRequests, GithubTicket{
Number: PRsResponse.Tickets[i].Number,
CreatedAt: parseGithubTime(PRsResponse.Tickets[i].CreatedAt),
Title: PRsResponse.Tickets[i].Title,
})
}
}
}
if maxIssues > 0 {
if issuesErr != nil {
// TODO: fix, overwriting the previous error
err = fmt.Errorf("%w: could not get issues: %s", ErrPartialContent, issuesErr)
} else {
details.OpenIssues = issuesResponse.Count
for i := range issuesResponse.Tickets {
details.Issues = append(details.Issues, GithubTicket{
Number: issuesResponse.Tickets[i].Number,
CreatedAt: parseGithubTime(issuesResponse.Tickets[i].CreatedAt),
Title: issuesResponse.Tickets[i].Title,
})
}
}
}
return details, err
}

184
internal/feed/github.go Normal file
View File

@ -0,0 +1,184 @@
package feed
import (
"fmt"
"net/http"
"sync"
"time"
)
type githubReleaseLatestResponseJson struct {
TagName string `json:"tag_name"`
PublishedAt string `json:"published_at"`
HtmlUrl string `json:"html_url"`
Reactions struct {
Downvotes int `json:"-1"`
} `json:"reactions"`
}
func fetchLatestGithubRelease(request *ReleaseRequest) (*AppRelease, error) {
httpRequest, err := http.NewRequest(
"GET",
fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", request.Repository),
nil,
)
if err != nil {
return nil, err
}
if request.Token != nil {
httpRequest.Header.Add("Authorization", "Bearer "+(*request.Token))
}
response, err := decodeJsonFromRequest[githubReleaseLatestResponseJson](defaultClient, httpRequest)
if err != nil {
return nil, err
}
version := response.TagName
if len(version) > 0 && version[0] != 'v' {
version = "v" + version
}
return &AppRelease{
Source: ReleaseSourceGithub,
Name: request.Repository,
Version: version,
NotesUrl: response.HtmlUrl,
TimeReleased: parseRFC3339Time(response.PublishedAt),
Downvotes: response.Reactions.Downvotes,
}, nil
}
type GithubTicket struct {
Number int
CreatedAt time.Time
Title string
}
type RepositoryDetails struct {
Name string
Stars int
Forks int
OpenPullRequests int
PullRequests []GithubTicket
OpenIssues int
Issues []GithubTicket
}
type githubRepositoryDetailsResponseJson struct {
Name string `json:"full_name"`
Stars int `json:"stargazers_count"`
Forks int `json:"forks_count"`
}
type githubTicketResponseJson struct {
Count int `json:"total_count"`
Tickets []struct {
Number int `json:"number"`
CreatedAt string `json:"created_at"`
Title string `json:"title"`
} `json:"items"`
}
func FetchRepositoryDetailsFromGithub(repository string, token string, maxPRs int, maxIssues int) (RepositoryDetails, error) {
repositoryRequest, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s", repository), nil)
if err != nil {
return RepositoryDetails{}, fmt.Errorf("%w: could not create request with repository: %v", ErrNoContent, err)
}
PRsRequest, _ := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/search/issues?q=is:pr+is:open+repo:%s&per_page=%d", repository, maxPRs), nil)
issuesRequest, _ := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/search/issues?q=is:issue+is:open+repo:%s&per_page=%d", repository, maxIssues), nil)
if token != "" {
token = fmt.Sprintf("Bearer %s", token)
repositoryRequest.Header.Add("Authorization", token)
PRsRequest.Header.Add("Authorization", token)
issuesRequest.Header.Add("Authorization", token)
}
var detailsResponse githubRepositoryDetailsResponseJson
var detailsErr error
var PRsResponse githubTicketResponseJson
var PRsErr error
var issuesResponse githubTicketResponseJson
var issuesErr error
var wg sync.WaitGroup
wg.Add(1)
go (func() {
defer wg.Done()
detailsResponse, detailsErr = decodeJsonFromRequest[githubRepositoryDetailsResponseJson](defaultClient, repositoryRequest)
})()
if maxPRs > 0 {
wg.Add(1)
go (func() {
defer wg.Done()
PRsResponse, PRsErr = decodeJsonFromRequest[githubTicketResponseJson](defaultClient, PRsRequest)
})()
}
if maxIssues > 0 {
wg.Add(1)
go (func() {
defer wg.Done()
issuesResponse, issuesErr = decodeJsonFromRequest[githubTicketResponseJson](defaultClient, issuesRequest)
})()
}
wg.Wait()
if detailsErr != nil {
return RepositoryDetails{}, fmt.Errorf("%w: could not get repository details: %s", ErrNoContent, detailsErr)
}
details := RepositoryDetails{
Name: detailsResponse.Name,
Stars: detailsResponse.Stars,
Forks: detailsResponse.Forks,
PullRequests: make([]GithubTicket, 0, len(PRsResponse.Tickets)),
Issues: make([]GithubTicket, 0, len(issuesResponse.Tickets)),
}
err = nil
if maxPRs > 0 {
if PRsErr != nil {
err = fmt.Errorf("%w: could not get PRs: %s", ErrPartialContent, PRsErr)
} else {
details.OpenPullRequests = PRsResponse.Count
for i := range PRsResponse.Tickets {
details.PullRequests = append(details.PullRequests, GithubTicket{
Number: PRsResponse.Tickets[i].Number,
CreatedAt: parseRFC3339Time(PRsResponse.Tickets[i].CreatedAt),
Title: PRsResponse.Tickets[i].Title,
})
}
}
}
if maxIssues > 0 {
if issuesErr != nil {
// TODO: fix, overwriting the previous error
err = fmt.Errorf("%w: could not get issues: %s", ErrPartialContent, issuesErr)
} else {
details.OpenIssues = issuesResponse.Count
for i := range issuesResponse.Tickets {
details.Issues = append(details.Issues, GithubTicket{
Number: issuesResponse.Tickets[i].Number,
CreatedAt: parseRFC3339Time(issuesResponse.Tickets[i].CreatedAt),
Title: issuesResponse.Tickets[i].Title,
})
}
}
}
return details, err
}

54
internal/feed/gitlab.go Normal file
View File

@ -0,0 +1,54 @@
package feed
import (
"fmt"
"net/http"
"net/url"
)
type gitlabReleaseResponseJson struct {
TagName string `json:"tag_name"`
ReleasedAt string `json:"released_at"`
Links struct {
Self string `json:"self"`
} `json:"_links"`
}
func fetchLatestGitLabRelease(request *ReleaseRequest) (*AppRelease, error) {
httpRequest, err := http.NewRequest(
"GET",
fmt.Sprintf(
"https://gitlab.com/api/v4/projects/%s/releases/permalink/latest",
url.QueryEscape(request.Repository),
),
nil,
)
if err != nil {
return nil, err
}
if request.Token != nil {
httpRequest.Header.Add("PRIVATE-TOKEN", *request.Token)
}
response, err := decodeJsonFromRequest[gitlabReleaseResponseJson](defaultClient, httpRequest)
if err != nil {
return nil, err
}
version := response.TagName
if len(version) > 0 && version[0] != 'v' {
version = "v" + version
}
return &AppRelease{
Source: ReleaseSourceGitlab,
Name: request.Repository,
Version: version,
NotesUrl: response.Links.Self,
TimeReleased: parseRFC3339Time(response.ReleasedAt),
}, nil
}

View File

@ -40,6 +40,7 @@ type Weather struct {
}
type AppRelease struct {
Source ReleaseSource
Name string
Version string
NotesUrl string

69
internal/feed/releases.go Normal file
View File

@ -0,0 +1,69 @@
package feed
import (
"errors"
"fmt"
"log/slog"
)
type ReleaseSource string
const (
ReleaseSourceGithub ReleaseSource = "github"
ReleaseSourceGitlab ReleaseSource = "gitlab"
ReleaseSourceDockerHub ReleaseSource = "dockerhub"
)
type ReleaseRequest struct {
Source ReleaseSource
Repository string
Token *string
}
func FetchLatestReleases(requests []*ReleaseRequest) (AppReleases, error) {
job := newJob(fetchLatestReleaseTask, requests).withWorkers(20)
results, errs, err := workerPoolDo(job)
if err != nil {
return nil, err
}
var failed int
releases := make(AppReleases, 0, len(requests))
for i := range results {
if errs[i] != nil {
failed++
slog.Error("Failed to fetch release", "source", requests[i].Source, "repository", requests[i].Repository, "error", errs[i])
continue
}
releases = append(releases, *results[i])
}
if failed == len(requests) {
return nil, ErrNoContent
}
releases.SortByNewest()
if failed > 0 {
return releases, fmt.Errorf("%w: could not get %d releases", ErrPartialContent, failed)
}
return releases, nil
}
func fetchLatestReleaseTask(request *ReleaseRequest) (*AppRelease, error) {
switch request.Source {
case ReleaseSourceGithub:
return fetchLatestGithubRelease(request)
case ReleaseSourceGitlab:
return fetchLatestGitLabRelease(request)
case ReleaseSourceDockerHub:
return fetchLatestDockerHubRelease(request)
}
return nil, errors.New("unsupported source")
}

View File

@ -7,6 +7,7 @@ import (
"regexp"
"slices"
"strings"
"time"
)
var (
@ -79,7 +80,6 @@ func maybeCopySliceWithoutZeroValues[T int | float64](values []T) []T {
return values
}
var urlSchemePattern = regexp.MustCompile(`^[a-z]+:\/\/`)
func stripURLScheme(url string) string {
@ -95,3 +95,13 @@ func limitStringLength(s string, max int) (string, bool) {
return s, false
}
func parseRFC3339Time(t string) time.Time {
parsed, err := time.Parse(time.RFC3339, t)
if err != nil {
return time.Now()
}
return parsed
}