[feature] Fetch + display custom emoji in statuses from remote instances (#807)

* start implementing remote emoji fetcher

* update status where pk

* aaa

* tidy up a little

* check size limits for emojis

* thank you linter, i love you <3

* update swagger docs

* add emoji dereference test

* make emoji max sizes configurable

* normalize db.ErrAlreadyExists

docs/api/swagger.yaml

@@ -2724,7 +2724,9 @@ paths:
         pattern: \w{2,30}
         required: true
         type: string
-      - description: A png or gif image of the emoji. Animated pngs work too!
+      - description: |-
+          A png or gif image of the emoji. Animated pngs work too!
+          To ensure compatibility with other fedi implementations, emoji size limit is 50kb by default.
         in: formData
         name: image
         required: true

docs/configuration/media.md

@@ -39,4 +39,20 @@ media-description-max-chars: 500
 # Examples: [30, 60, 7, 0]
 # Default: 30
 media-remote-cache-days: 30
+
+# Int. Max size in bytes of emojis uploaded to this instance via the admin API.
+# The default is the same as the Mastodon size limit for emojis (50kb), which allows
+# for good interoperability. Raising this limit may cause issues with federation
+# of your emojis to other instances, so beware.
+# Examples: [51200, 102400]
+# Default: 51200
+media-emoji-local-max-size: 51200
+
+# Int. Max size in bytes of emojis to download from other instances.
+# By default this is 100kb, or twice the size of the default for media-emoji-local-max-size.
+# This strikes a good balance between decent interoperability with instances that have
+# higher emoji size limits, and not taking up too much space in storage.
+# Examples: [51200, 102400]
+# Default: 51200
+media-emoji-remote-max-size: 102400
 ```

example/config.yaml

@@ -211,7 +211,7 @@ accounts-reason-required: true
 ##### MEDIA CONFIG #####
 ########################
 
-# Config pertaining to user media uploads (videos, image, image descriptions).
+# Config pertaining to media uploads (videos, image, image descriptions, emoji).
 
 # Int. Maximum allowed image upload size in bytes.
 # Examples: [2097152, 10485760]
@@ -244,6 +244,22 @@ media-description-max-chars: 500
 # Default: 30
 media-remote-cache-days: 30
 
+# Int. Max size in bytes of emojis uploaded to this instance via the admin API.
+# The default is the same as the Mastodon size limit for emojis (50kb), which allows
+# for good interoperability. Raising this limit may cause issues with federation
+# of your emojis to other instances, so beware.
+# Examples: [51200, 102400]
+# Default: 51200
+media-emoji-local-max-size: 51200
+
+# Int. Max size in bytes of emojis to download from other instances.
+# By default this is 100kb, or twice the size of the default for media-emoji-local-max-size.
+# This strikes a good balance between decent interoperability with instances that have
+# higher emoji size limits, and not taking up too much space in storage.
+# Examples: [51200, 102400]
+# Default: 51200
+media-emoji-remote-max-size: 102400
+
 ##########################
 ##### STORAGE CONFIG #####
 ##########################

internal/api/client/admin/emojicreate.go

@@ -26,6 +26,7 @@
 	"github.com/gin-gonic/gin"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/validate"
@@ -56,7 +57,9 @@
 //   required: true
 // - name: image
 //   in: formData
-//   description: A png or gif image of the emoji. Animated pngs work too!
+//   description: |-
+//     A png or gif image of the emoji. Animated pngs work too!
+//     To ensure compatibility with other fedi implementations, emoji size limit is 50kb by default.
 //   type: file
 //   required: true
 //
@@ -126,5 +129,10 @@ func validateCreateEmoji(form *model.EmojiCreateRequest) error {
 		return errors.New("no emoji given")
 	}
 
+	maxSize := config.GetMediaEmojiLocalMaxSize()
+	if form.Image.Size > int64(maxSize) {
+		return fmt.Errorf("emoji image too large: image is %dKB but size limit for custom emojis is %dKB", form.Image.Size/1024, maxSize/1024)
+	}
+
 	return validate.EmojiShortcode(form.Shortcode)
 }

internal/config/config.go

@@ -79,6 +79,8 @@ type Configuration struct {
 	MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"`
 	MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
 	MediaRemoteCacheDays     int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
+	MediaEmojiLocalMaxSize   int `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
+	MediaEmojiRemoteMaxSize  int `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
 
 	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
 	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`

internal/config/defaults.go

@@ -58,6 +58,8 @@
 	MediaDescriptionMinChars: 0,
 	MediaDescriptionMaxChars: 500,
 	MediaRemoteCacheDays:     30,
+	MediaEmojiLocalMaxSize:   51200,  // 50kb
+	MediaEmojiRemoteMaxSize:  102400, // 100kb
 
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",

internal/config/flags.go

@@ -75,6 +75,8 @@ func AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage"))
 		cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage"))
 		cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))
+		cmd.Flags().Int(MediaEmojiLocalMaxSizeFlag(), cfg.MediaEmojiLocalMaxSize, fieldtag("MediaEmojiLocalMaxSize", "usage"))
+		cmd.Flags().Int(MediaEmojiRemoteMaxSizeFlag(), cfg.MediaEmojiRemoteMaxSize, fieldtag("MediaEmojiRemoteMaxSize", "usage"))
 
 		// Storage
 		cmd.Flags().String(StorageBackendFlag(), cfg.StorageBackend, fieldtag("StorageBackend", "usage"))

internal/config/helpers.gen.go

@@ -793,6 +793,56 @@ func GetMediaRemoteCacheDays() int { return global.GetMediaRemoteCacheDays() }
 // SetMediaRemoteCacheDays safely sets the value for global configuration 'MediaRemoteCacheDays' field
 func SetMediaRemoteCacheDays(v int) { global.SetMediaRemoteCacheDays(v) }
 
+// GetMediaEmojiLocalMaxSize safely fetches the Configuration value for state's 'MediaEmojiLocalMaxSize' field
+func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.MediaEmojiLocalMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetMediaEmojiLocalMaxSize safely sets the Configuration value for state's 'MediaEmojiLocalMaxSize' field
+func (st *ConfigState) SetMediaEmojiLocalMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaEmojiLocalMaxSize = v
+	st.reloadToViper()
+}
+
+// MediaEmojiLocalMaxSizeFlag returns the flag name for the 'MediaEmojiLocalMaxSize' field
+func MediaEmojiLocalMaxSizeFlag() string { return "media-emoji-local-max-size" }
+
+// GetMediaEmojiLocalMaxSize safely fetches the value for global configuration 'MediaEmojiLocalMaxSize' field
+func GetMediaEmojiLocalMaxSize() int { return global.GetMediaEmojiLocalMaxSize() }
+
+// SetMediaEmojiLocalMaxSize safely sets the value for global configuration 'MediaEmojiLocalMaxSize' field
+func SetMediaEmojiLocalMaxSize(v int) { global.SetMediaEmojiLocalMaxSize(v) }
+
+// GetMediaEmojiRemoteMaxSize safely fetches the Configuration value for state's 'MediaEmojiRemoteMaxSize' field
+func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.MediaEmojiRemoteMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetMediaEmojiRemoteMaxSize safely sets the Configuration value for state's 'MediaEmojiRemoteMaxSize' field
+func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaEmojiRemoteMaxSize = v
+	st.reloadToViper()
+}
+
+// MediaEmojiRemoteMaxSizeFlag returns the flag name for the 'MediaEmojiRemoteMaxSize' field
+func MediaEmojiRemoteMaxSizeFlag() string { return "media-emoji-remote-max-size" }
+
+// GetMediaEmojiRemoteMaxSize safely fetches the value for global configuration 'MediaEmojiRemoteMaxSize' field
+func GetMediaEmojiRemoteMaxSize() int { return global.GetMediaEmojiRemoteMaxSize() }
+
+// SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
+func SetMediaEmojiRemoteMaxSize(v int) { global.SetMediaEmojiRemoteMaxSize(v) }
+
 // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field
 func (st *ConfigState) GetStorageBackend() (v string) {
 	st.mutex.Lock()

internal/config/validate.go

@@ -67,6 +67,14 @@ func Validate() error {
 		errs = append(errs, fmt.Errorf("%s must be set", WebAssetBaseDirFlag()))
 	}
 
+	if m := GetMediaEmojiLocalMaxSize(); m < 0 {
+		errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiLocalMaxSizeFlag()))
+	}
+
+	if m := GetMediaEmojiRemoteMaxSize(); m < 0 {
+		errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiRemoteMaxSizeFlag()))
+	}
+
 	if len(errs) > 0 {
 		errStrings := []string{}
 		for _, err := range errs {

internal/config/validate_test.go

@@ -141,6 +141,16 @@ func (suite *ConfigValidateTestSuite) TestValidateConfigBadProtocolNoHost() {
 	suite.EqualError(err, "host must be set; protocol must be set to either http or https, provided value was foo")
 }
 
+func (suite *ConfigValidateTestSuite) TestValidateConfigBadEmojiSizes() {
+	testrig.InitTestConfig()
+
+	config.SetMediaEmojiLocalMaxSize(-10)
+	config.SetMediaEmojiRemoteMaxSize(-50)
+
+	err := config.Validate()
+	suite.EqualError(err, "media-emoji-local-max-size must not be less than 0; media-emoji-remote-max-size must not be less than 0")
+}
+
 func TestConfigValidateTestSuite(t *testing.T) {
 	suite.Run(t, &ConfigValidateTestSuite{})
 }

internal/db/bundb/errors.go

@@ -19,7 +19,7 @@ func processPostgresError(err error) db.Error {
 	// (https://www.postgresql.org/docs/10/errcodes-appendix.html)
 	switch pgErr.Code {
 	case "23505" /* unique_violation */ :
-		return db.NewErrAlreadyExists(pgErr.Message)
+		return db.ErrAlreadyExists
 	default:
 		return err
 	}
@@ -36,7 +36,7 @@ func processSQLiteError(err error) db.Error {
 	// Handle supplied error code:
 	switch sqliteErr.Code() {
 	case sqlite3.SQLITE_CONSTRAINT_UNIQUE, sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
-		return db.NewErrAlreadyExists(err.Error())
+		return db.ErrAlreadyExists
 	default:
 		return err
 	}

internal/db/bundb/status.go

@@ -22,6 +22,7 @@
 	"container/list"
 	"context"
 	"database/sql"
+	"errors"
 	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/cache"
@@ -175,6 +176,57 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er
 	})
 }
 
+func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status) (*gtsmodel.Status, db.Error) {
+	err := s.conn.RunInTx(ctx, func(tx bun.Tx) error {
+		// create links between this status and any emojis it uses
+		for _, i := range status.EmojiIDs {
+			if _, err := tx.NewInsert().Model(&gtsmodel.StatusToEmoji{
+				StatusID: status.ID,
+				EmojiID:  i,
+			}).Exec(ctx); err != nil {
+				err = s.conn.errProc(err)
+				if !errors.Is(err, db.ErrAlreadyExists) {
+					return err
+				}
+			}
+		}
+
+		// create links between this status and any tags it uses
+		for _, i := range status.TagIDs {
+			if _, err := tx.NewInsert().Model(&gtsmodel.StatusToTag{
+				StatusID: status.ID,
+				TagID:    i,
+			}).Exec(ctx); err != nil {
+				err = s.conn.errProc(err)
+				if !errors.Is(err, db.ErrAlreadyExists) {
+					return err
+				}
+			}
+		}
+
+		// change the status ID of the media attachments to this status
+		for _, a := range status.Attachments {
+			a.StatusID = status.ID
+			a.UpdatedAt = time.Now()
+			if _, err := tx.NewUpdate().Model(a).
+				Where("id = ?", a.ID).
+				Exec(ctx); err != nil {
+				return err
+			}
+		}
+
+		// Finally, update the status itself
+		if _, err := tx.NewUpdate().Model(status).WherePK().Exec(ctx); err != nil {
+			return err
+		}
+
+		s.cache.Put(status)
+		return nil
+	})
+
+	return status, err
+}
+
 func (s *statusDB) GetStatusParents(ctx context.Context, status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, db.Error) {
 	parents := []*gtsmodel.Status{}
 	s.statusParent(ctx, status, &parents, onlyDirect)

internal/db/emoji.go

@@ -35,4 +35,6 @@ type Emoji interface {
 	// GetEmojiByShortcodeDomain gets an emoji based on its shortcode and domain.
 	// For local emoji, domain should be an empty string.
 	GetEmojiByShortcodeDomain(ctx context.Context, shortcode string, domain string) (*gtsmodel.Emoji, Error)
+	// GetEmojiByURI returns one emoji based on its ActivityPub URI.
+	GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoji, Error)
 }

internal/db/error.go

@@ -28,19 +28,8 @@
 	ErrNoEntries Error = fmt.Errorf("no entries")
 	// ErrMultipleEntries is returned when a caller expected ONE entry for a query, but multiples were found.
 	ErrMultipleEntries Error = fmt.Errorf("multiple entries")
+	// ErrAlreadyExists is returned when a conflict was encountered in the db when doing an insert.
+	ErrAlreadyExists Error = fmt.Errorf("already exists")
 	// ErrUnknown denotes an unknown database error.
 	ErrUnknown Error = fmt.Errorf("unknown error")
 )
-
-// ErrAlreadyExists is returned when a caller tries to insert a database entry that already exists in the db.
-type ErrAlreadyExists struct {
-	message string
-}
-
-func (e *ErrAlreadyExists) Error() string {
-	return e.message
-}
-
-func NewErrAlreadyExists(msg string) error {
-	return &ErrAlreadyExists{message: msg}
-}

internal/db/status.go

@@ -38,6 +38,9 @@ type Status interface {
 	// PutStatus stores one status in the database.
 	PutStatus(ctx context.Context, status *gtsmodel.Status) Error
 
+	// UpdateStatus updates one status in the database and returns it to the caller.
+	UpdateStatus(ctx context.Context, status *gtsmodel.Status) (*gtsmodel.Status, Error)
+
 	// CountStatusReplies returns the amount of replies recorded for a status, or an error if something goes wrong
 	CountStatusReplies(ctx context.Context, status *gtsmodel.Status) (int, Error)
 

internal/federation/dereferencing/dereferencer.go

@@ -41,6 +41,7 @@ type Dereferencer interface {
 	GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
 
 	GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error)
+	GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error)
 
 	DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
 	DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error

internal/federation/dereferencing/emoji.go

@@ -0,0 +1,51 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+)
+
+func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error) {
+	t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
+	if err != nil {
+		return nil, fmt.Errorf("GetRemoteEmoji: error creating transport: %s", err)
+	}
+
+	derefURI, err := url.Parse(remoteURL)
+	if err != nil {
+		return nil, fmt.Errorf("GetRemoteEmoji: error parsing url: %s", err)
+	}
+
+	dataFunc := func(innerCtx context.Context) (io.Reader, int, error) {
+		return t.DereferenceMedia(innerCtx, derefURI)
+	}
+
+	processingMedia, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai)
+	if err != nil {
+		return nil, fmt.Errorf("GetRemoteEmoji: error processing emoji: %s", err)
+	}
+
+	return processingMedia, nil
+}

internal/federation/dereferencing/emoji_test.go

@@ -0,0 +1,95 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+)
+
+type EmojiTestSuite struct {
+	DereferencerStandardTestSuite
+}
+
+func (suite *EmojiTestSuite) TestDereferenceEmojiBlocking() {
+	ctx := context.Background()
+	fetchingAccount := suite.testAccounts["local_account_1"]
+	emojiImageRemoteURL := "http://example.org/media/emojis/1781772.gif"
+	emojiImageStaticRemoteURL := "http://example.org/media/emojis/1781772.gif"
+	emojiURI := "http://example.org/emojis/1781772"
+	emojiShortcode := "peglin"
+	emojiID := "01GCBMGNZBKMEE1KTZ6PMJEW5D"
+	emojiDomain := "example.org"
+	emojiDisabled := false
+	emojiVisibleInPicker := false
+
+	ai := &media.AdditionalEmojiInfo{
+		Domain:               &emojiDomain,
+		ImageRemoteURL:       &emojiImageRemoteURL,
+		ImageStaticRemoteURL: &emojiImageStaticRemoteURL,
+		Disabled:             &emojiDisabled,
+		VisibleInPicker:      &emojiVisibleInPicker,
+	}
+
+	processingEmoji, err := suite.dereferencer.GetRemoteEmoji(ctx, fetchingAccount.Username, emojiImageRemoteURL, emojiShortcode, emojiID, emojiURI, ai)
+	suite.NoError(err)
+
+	// make a blocking call to load the emoji from the in-process media
+	emoji, err := processingEmoji.LoadEmoji(ctx)
+	suite.NoError(err)
+	suite.NotNil(emoji)
+
+	suite.Equal(emojiID, emoji.ID)
+	suite.WithinDuration(time.Now(), emoji.CreatedAt, 10*time.Second)
+	suite.WithinDuration(time.Now(), emoji.UpdatedAt, 10*time.Second)
+	suite.Equal(emojiShortcode, emoji.Shortcode)
+	suite.Equal(emojiDomain, emoji.Domain)
+	suite.Equal(emojiImageRemoteURL, emoji.ImageRemoteURL)
+	suite.Equal(emojiImageStaticRemoteURL, emoji.ImageStaticRemoteURL)
+	suite.Contains(emoji.ImageURL, "/emoji/original/01GCBMGNZBKMEE1KTZ6PMJEW5D.gif")
+	suite.Contains(emoji.ImageStaticURL, "emoji/static/01GCBMGNZBKMEE1KTZ6PMJEW5D.png")
+	suite.Contains(emoji.ImagePath, "/emoji/original/01GCBMGNZBKMEE1KTZ6PMJEW5D.gif")
+	suite.Contains(emoji.ImageStaticPath, "/emoji/static/01GCBMGNZBKMEE1KTZ6PMJEW5D.png")
+	suite.Equal("image/gif", emoji.ImageContentType)
+	suite.Equal("image/png", emoji.ImageStaticContentType)
+	suite.Equal(37796, emoji.ImageFileSize)
+	suite.Equal(7951, emoji.ImageStaticFileSize)
+	suite.WithinDuration(time.Now(), emoji.ImageUpdatedAt, 10*time.Second)
+	suite.False(*emoji.Disabled)
+	suite.Equal(emojiURI, emoji.URI)
+	suite.False(*emoji.VisibleInPicker)
+	suite.Empty(emoji.CategoryID)
+
+	// ensure that emoji is now in storage
+	stored, err := suite.storage.Get(ctx, emoji.ImagePath)
+	suite.NoError(err)
+	suite.Len(stored, emoji.ImageFileSize)
+
+	storedStatic, err := suite.storage.Get(ctx, emoji.ImageStaticPath)
+	suite.NoError(err)
+	suite.Len(storedStatic, emoji.ImageStaticFileSize)
+}
+
+func TestEmojiTestSuite(t *testing.T) {
+	suite.Run(t, new(EmojiTestSuite))
+}

internal/federation/dereferencing/status.go

@@ -26,10 +26,10 @@
 	"net/url"
 	"strings"
 
-	"codeberg.org/gruf/go-kv"
 	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -46,11 +46,7 @@ func (d *deref) EnrichRemoteStatus(ctx context.Context, username string, status
 		return nil, err
 	}
 
-	if err := d.db.UpdateByPrimaryKey(ctx, status); err != nil {
+	return d.db.UpdateStatus(ctx, status)
-		return nil, fmt.Errorf("EnrichRemoteStatus: error updating status: %s", err)
-	}
-
-	return status, nil
 }
 
 // GetRemoteStatus completely dereferences a remote status, converts it to a GtS model status,
@@ -225,12 +221,6 @@ func (d *deref) dereferenceStatusable(ctx context.Context, username string, remo
 // and attach them to the status. The status itself will not be added to the database yet,
 // that's up the caller to do.
 func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Status, requestingUsername string, includeParent bool) error {
-	l := log.WithFields(kv.Fields{
-
-		{"status", status},
-	}...)
-	l.Debug("entering function")
-
 	statusIRI, err := url.Parse(status.URI)
 	if err != nil {
 		return fmt.Errorf("populateStatusFields: couldn't parse status URI %s: %s", status.URI, err)
@@ -262,7 +252,9 @@ func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Statu
 	// TODO
 
 	// 3. Emojis
-	// TODO
+	if err := d.populateStatusEmojis(ctx, status, requestingUsername); err != nil {
+		return fmt.Errorf("populateStatusFields: error populating status emojis: %s", err)
+	}
 
 	// 4. Mentions
 	// TODO: do we need to handle removing empty mention objects and just using mention IDs slice?
@@ -413,6 +405,64 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.
 	return nil
 }
 
+func (d *deref) populateStatusEmojis(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
+	// At this point we should know:
+	// * the AP uri of the emoji
+	// * the domain of the emoji
+	// * the shortcode of the emoji
+	// * the remote URL of the image
+	// This should be enough to dereference the emoji
+
+	gotEmojis := make([]*gtsmodel.Emoji, 0, len(status.Emojis))
+	emojiIDs := make([]string, 0, len(status.Emojis))
+
+	for _, e := range status.Emojis {
+		var gotEmoji *gtsmodel.Emoji
+		var err error
+
+		// check if we've already got this emoji in the db
+		if gotEmoji, err = d.db.GetEmojiByURI(ctx, e.URI); err != nil && err != db.ErrNoEntries {
+			log.Errorf("populateStatusEmojis: error checking database for emoji %s: %s", e.URI, err)
+			continue
+		}
+
+		if gotEmoji == nil {
+			// it's new! go get it!
+			newEmojiID, err := id.NewRandomULID()
+			if err != nil {
+				log.Errorf("populateStatusEmojis: error generating id for remote emoji %s: %s", e.URI, err)
+				continue
+			}
+
+			processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, newEmojiID, e.URI, &media.AdditionalEmojiInfo{
+				Domain:               &e.Domain,
+				ImageRemoteURL:       &e.ImageRemoteURL,
+				ImageStaticRemoteURL: &e.ImageRemoteURL,
+				Disabled:             e.Disabled,
+				VisibleInPicker:      e.VisibleInPicker,
+			})
+
+			if err != nil {
+				log.Errorf("populateStatusEmojis: couldn't get remote emoji %s: %s", e.URI, err)
+				continue
+			}
+
+			if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil {
+				log.Errorf("populateStatusEmojis: couldn't load remote emoji %s: %s", e.URI, err)
+				continue
+			}
+		}
+
+		// if we get here, we either had the emoji already or we successfully fetched it
+		gotEmojis = append(gotEmojis, gotEmoji)
+		emojiIDs = append(emojiIDs, gotEmoji.ID)
+	}
+
+	status.Emojis = gotEmojis
+	status.EmojiIDs = emojiIDs
+	return nil
+}
+
 func (d *deref) populateStatusRepliedTo(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
 	if status.InReplyToURI != "" && status.InReplyToID == "" {
 		statusURI, err := url.Parse(status.InReplyToURI)

internal/federation/federatingdb/create.go

@@ -226,8 +226,7 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream
 	status.ID = statusID
 
 	if err := f.db.PutStatus(ctx, status); err != nil {
-		var alreadyExistsError *db.ErrAlreadyExists
+		if errors.Is(err, db.ErrAlreadyExists) {
-		if errors.As(err, &alreadyExistsError) {
 			// the status already exists in the database, which means we've already handled everything else,
 			// so we can just return nil here and be done with it.
 			return nil

internal/gtsmodel/emoji.go

@@ -20,7 +20,7 @@
 
 import "time"
 
-// Emoji represents a custom emoji that's been uploaded through the admin UI, and is useable by instance denizens.
+// Emoji represents a custom emoji that's been uploaded through the admin UI or downloaded from a remote instance.
 type Emoji struct {
 	ID                     string    `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"`                                // id of this item in the database
 	CreatedAt              time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                         // when was item created

internal/media/processingemoji.go

@@ -28,6 +28,7 @@
 	"sync/atomic"
 	"time"
 
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -170,6 +171,11 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 		return fmt.Errorf("store: error executing data function: %s", err)
 	}
 
+	maxSize := config.GetMediaEmojiRemoteMaxSize()
+	if fileSize > maxSize {
+		return fmt.Errorf("store: emoji size (%db) is larger than allowed emojiRemoteMaxSize (%db)", fileSize, maxSize)
+	}
+
 	// defer closing the reader when we're done with it
 	defer func() {
 		if rc, ok := reader.(io.ReadCloser); ok {

internal/processing/status/util.go

@@ -234,8 +234,7 @@ func (p *processor) ProcessTags(ctx context.Context, form *apimodel.AdvancedStat
 	}
 	for _, tag := range gtsTags {
 		if err := p.db.Put(ctx, tag); err != nil {
-			var alreadyExistsError *db.ErrAlreadyExists
+			if !errors.Is(err, db.ErrAlreadyExists) {
-			if !errors.As(err, &alreadyExistsError) {
 				return fmt.Errorf("error putting tags in db: %s", err)
 			}
 		}

test/cliparsing.sh

@@ -5,7 +5,7 @@ set -e
 echo "STARTING CLI TESTS"
 
 echo "TEST_1 Make sure defaults are set correctly."
-TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_1="$(go run ./cmd/gotosocial/... debug config)"
 if [ "${TEST_1}" != "${TEST_1_EXPECTED}" ]; then
     echo "TEST_1 not equal TEST_1_EXPECTED"
@@ -15,7 +15,7 @@ else
 fi
 
 echo "TEST_2 Override db-address from default using cli flag."
-TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_2="$(go run ./cmd/gotosocial/... --db-address some.db.address debug config)"
 if [ "${TEST_2}" != "${TEST_2_EXPECTED}" ]; then
     echo "TEST_2 not equal TEST_2_EXPECTED"
@@ -25,7 +25,7 @@ else
 fi
 
 echo "TEST_3 Override db-address from default using env var."
-TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_3="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... debug config)"
 if [ "${TEST_3}" != "${TEST_3_EXPECTED}" ]; then
     echo "TEST_3 not equal TEST_3_EXPECTED"
@@ -35,7 +35,7 @@ else
 fi
 
 echo "TEST_4 Override db-address from default using both env var and cli flag. The cli flag should take priority."
-TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_4="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... --db-address some.other.db.address debug config)"
 if [ "${TEST_4}" != "${TEST_4_EXPECTED}" ]; then
     echo "TEST_4 not equal TEST_4_EXPECTED"
@@ -45,7 +45,7 @@ else
 fi
 
 echo "TEST_5 Test loading a config file by passing an env var."
-TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_5="$(GTS_CONFIG_PATH=./test/test.yaml go run ./cmd/gotosocial/... debug config)"
 if [ "${TEST_5}" != "${TEST_5_EXPECTED}" ]; then
     echo "TEST_5 not equal TEST_5_EXPECTED"
@@ -55,7 +55,7 @@ else
 fi
 
 echo "TEST_6 Test loading a config file by passing cli flag."
-TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_6="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)"
 if [ "${TEST_6}" != "${TEST_6_EXPECTED}" ]; then
     echo "TEST_6 not equal TEST_6_EXPECTED"
@@ -65,7 +65,7 @@ else
 fi
 
 echo "TEST_7 Test loading a config file and overriding one of the variables with a cli flag."
-TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_7="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)"
 if [ "${TEST_7}" != "${TEST_7_EXPECTED}" ]; then
     echo "TEST_7 not equal TEST_7_EXPECTED"
@@ -75,7 +75,7 @@ else
 fi
 
 echo "TEST_8 Test loading a config file and overriding one of the variables with an env var."
-TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_8="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)"
 if [ "${TEST_8}" != "${TEST_8_EXPECTED}" ]; then
     echo "TEST_8 not equal TEST_8_EXPECTED"
@@ -85,7 +85,7 @@ else
 fi
 
 echo "TEST_9 Test loading a config file and overriding one of the variables with both an env var and a cli flag. The cli flag should have priority."
-TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_9="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)"
 if [ "${TEST_9}" != "${TEST_9_EXPECTED}" ]; then
     echo "TEST_9 not equal TEST_9_EXPECTED"
@@ -95,7 +95,7 @@ else
 fi
 
 echo "TEST_10 Test loading a config file from json."
-TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_10="$(go run ./cmd/gotosocial/... --config-path ./test/test.json debug config)"
 if [ "${TEST_10}" != "${TEST_10_EXPECTED}" ]; then
     echo "TEST_10 not equal TEST_10_EXPECTED"
@@ -105,7 +105,7 @@ else
 fi
 
 echo "TEST_11 Test loading a partial config file. Default values should be used apart from those set in the config file."
-TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_11="$(go run ./cmd/gotosocial/... --config-path ./test/test2.yaml debug config)"
 if [ "${TEST_11}" != "${TEST_11_EXPECTED}" ]; then
     echo "TEST_11 not equal TEST_11_EXPECTED"

test/envparsing.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-EXPECTED='{"account-domain":"peepee","accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-expose-peers":true,"instance-expose-suspended":true,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
+EXPECTED='{"account-domain":"peepee","accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-expose-peers":true,"instance-expose-suspended":true,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
 
 # Set all the environment variables to 
 # ensure that these are parsed without panic
@@ -35,6 +35,8 @@ GTS_MEDIA_VIDEO_MAX_SIZE=420 \
 GTS_MEDIA_DESCRIPTION_MIN_CHARS=69 \
 GTS_MEDIA_DESCRIPTION_MAX_CHARS=5000 \
 GTS_MEDIA_REMOTE_CACHE_DAYS=30 \
+GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \
+GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \
 GTS_STORAGE_BACKEND='local' \
 GTS_STORAGE_LOCAL_BASE_PATH='/root/store' \
 GTS_STORAGE_S3_ACCESS_KEY='minio' \

testrig/config.go

@@ -64,6 +64,8 @@ func InitTestConfig() {
 	MediaDescriptionMinChars: 0,
 	MediaDescriptionMaxChars: 500,
 	MediaRemoteCacheDays:     30,
+	MediaEmojiLocalMaxSize:   51200,  // 50kb
+	MediaEmojiRemoteMaxSize:  102400, // 100kb
 
 	// the testrig only uses in-memory storage, so we can
 	// safely set this value to 'test' to avoid running storage

testrig/testmodels.go

@@ -1954,6 +1954,11 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
 		panic(err)
 	}
 
+	peglinBytes, err := os.ReadFile(fmt.Sprintf("%s/peglin.gif", relativePath))
+	if err != nil {
+		panic(err)
+	}
+
 	return map[string]RemoteAttachmentFile{
 		"https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg": {
 			Data:        beeBytes,
@@ -1967,6 +1972,10 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
 			Data:        massiveFuckingTurnipBytes,
 			ContentType: "image/jpeg",
 		},
+		"http://example.org/media/emojis/1781772.gif": {
+			Data:        peglinBytes,
+			ContentType: "image/gif",
+		},
 	}
 }