mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-12-13 10:30:45 +01:00
a48cce82b9
* [chore] Bump tooling versions, bump go -> v1.23.0 * undo silly change * sign * bump go version in go.mod * allow overflow in imaging * goreleaser deprecation notices * [chore] Upgrade golangci-lint, ignore existing int overflow warnings There is a new lint for unchecked int casts. Integer overflows are bad, but the old code that triggers this lint seems to be perfectly fine. Instead of disabling the lint entirely for new code as well, grandfather in existing code. * fix golangci-lint documentation link * revert unrelated changes * revert another unrelated change * get rid of remaining nolint:gosec * swagger updates * apply review feedback * fix wrong formatting specifier thing * fix the linter for real --------- Co-authored-by: tobi <tobi.smethurst@protonmail.com>
156 lines
5.2 KiB
Go
156 lines
5.2 KiB
Go
// GoToSocial
|
|
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package media
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
|
|
"github.com/superseriousbusiness/gotosocial/internal/config"
|
|
"github.com/superseriousbusiness/gotosocial/internal/db"
|
|
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
|
"github.com/superseriousbusiness/gotosocial/internal/log"
|
|
)
|
|
|
|
type DereferenceMedia func(ctx context.Context, iri *url.URL, maxsz int64) (io.ReadCloser, error)
|
|
|
|
// RefetchEmojis iterates through remote emojis (for the given domain, or all if domain is empty string).
|
|
//
|
|
// For each emoji, the manager will check whether both the full size and static images are present in storage.
|
|
// If not, the manager will refetch and reprocess full size and static images for the emoji.
|
|
//
|
|
// The provided DereferenceMedia function will be used when it's necessary to refetch something this way.
|
|
func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceMedia DereferenceMedia) (int, error) {
|
|
// normalize domain
|
|
if domain == "" {
|
|
domain = db.EmojiAllDomains
|
|
}
|
|
|
|
var (
|
|
maxShortcodeDomain string
|
|
refetchIDs []string
|
|
)
|
|
|
|
// page through emojis 20 at a time, looking for those with missing images
|
|
for {
|
|
// Fetch next block of emojis from database
|
|
emojis, err := m.state.DB.GetEmojisBy(ctx, domain, false, true, "", maxShortcodeDomain, "", 20)
|
|
if err != nil {
|
|
if !errors.Is(err, db.ErrNoEntries) {
|
|
log.Errorf(ctx, "error fetching emojis from database: %s", err)
|
|
}
|
|
break
|
|
}
|
|
|
|
for _, emoji := range emojis {
|
|
if emoji.IsLocal() {
|
|
// never try to refetch local emojis
|
|
continue
|
|
}
|
|
|
|
if refetch, err := m.emojiRequiresRefetch(ctx, emoji); err != nil {
|
|
// an error here indicates something is wrong with storage, so we should stop
|
|
return 0, fmt.Errorf("error checking refetch requirement for emoji %s: %w", emoji.ShortcodeDomain(), err)
|
|
} else if !refetch {
|
|
continue
|
|
}
|
|
|
|
refetchIDs = append(refetchIDs, emoji.ID)
|
|
}
|
|
|
|
// Update next maxShortcodeDomain from last emoji
|
|
maxShortcodeDomain = emojis[len(emojis)-1].ShortcodeDomain()
|
|
}
|
|
|
|
// bail early if we've got nothing to do
|
|
toRefetchCount := len(refetchIDs)
|
|
if toRefetchCount == 0 {
|
|
log.Debug(ctx, "no remote emojis require a refetch")
|
|
return 0, nil
|
|
}
|
|
log.Debugf(ctx, "%d remote emoji(s) require a refetch, doing that now...", toRefetchCount)
|
|
|
|
var totalRefetched int
|
|
for _, emojiID := range refetchIDs {
|
|
emoji, err := m.state.DB.GetEmojiByID(ctx, emojiID)
|
|
if err != nil {
|
|
// this shouldn't happen--since we know we have the emoji--so return if it does
|
|
return 0, fmt.Errorf("error getting emoji %s: %w", emojiID, err)
|
|
}
|
|
shortcodeDomain := emoji.ShortcodeDomain()
|
|
|
|
if emoji.ImageRemoteURL == "" {
|
|
log.Errorf(ctx, "remote emoji %s could not be refreshed because it has no ImageRemoteURL set", shortcodeDomain)
|
|
continue
|
|
}
|
|
|
|
emojiImageIRI, err := url.Parse(emoji.ImageRemoteURL)
|
|
if err != nil {
|
|
log.Errorf(ctx, "remote emoji %s could not be refreshed because its ImageRemoteURL (%s) is not a valid uri: %s", shortcodeDomain, emoji.ImageRemoteURL, err)
|
|
continue
|
|
}
|
|
|
|
// Get max supported remote emoji media size.
|
|
maxsz := int64(config.GetMediaEmojiRemoteMaxSize()) // #nosec G115 -- Already validated.
|
|
dataFunc := func(ctx context.Context) (reader io.ReadCloser, err error) {
|
|
return dereferenceMedia(ctx, emojiImageIRI, maxsz)
|
|
}
|
|
|
|
processingEmoji, err := m.UpdateEmoji(ctx, emoji, dataFunc, AdditionalEmojiInfo{
|
|
Domain: &emoji.Domain,
|
|
ImageRemoteURL: &emoji.ImageRemoteURL,
|
|
ImageStaticRemoteURL: &emoji.ImageStaticRemoteURL,
|
|
Disabled: emoji.Disabled,
|
|
VisibleInPicker: emoji.VisibleInPicker,
|
|
})
|
|
if err != nil {
|
|
log.Errorf(ctx, "emoji %s could not be updated because of an error during processing: %s", shortcodeDomain, err)
|
|
continue
|
|
}
|
|
|
|
if _, err := processingEmoji.Load(ctx); err != nil {
|
|
log.Errorf(ctx, "emoji %s could not be updated because of an error during loading: %s", shortcodeDomain, err)
|
|
continue
|
|
}
|
|
|
|
log.Tracef(ctx, "refetched + updated emoji %s successfully from remote", shortcodeDomain)
|
|
totalRefetched++
|
|
}
|
|
|
|
return totalRefetched, nil
|
|
}
|
|
|
|
func (m *Manager) emojiRequiresRefetch(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
|
|
if has, err := m.state.Storage.Has(ctx, emoji.ImagePath); err != nil {
|
|
return false, err
|
|
} else if !has {
|
|
return true, nil
|
|
}
|
|
|
|
if has, err := m.state.Storage.Has(ctx, emoji.ImageStaticPath); err != nil {
|
|
return false, err
|
|
} else if !has {
|
|
return true, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|