[chore] Add media-ffmpeg-pool-size config var (#3164)

cmd/gotosocial/action/server/server.go

@@ -487,16 +487,24 @@ func setLimits(ctx context.Context) {
 }
 
 func precompileWASM(ctx context.Context) error {
-	// TODO: make max number instances configurable
-	maxprocs := runtime.GOMAXPROCS(0)
 	if err := sqlite3.Initialize(); err != nil {
 		return gtserror.Newf("error compiling sqlite3: %w", err)
 	}
-	if err := ffmpeg.InitFfmpeg(ctx, maxprocs); err != nil {
+
+	// Use admin-set ffmpeg pool size, and fall
+	// back to GOMAXPROCS if number 0 or less.
+	ffPoolSize := config.GetMediaFfmpegPoolSize()
+	if ffPoolSize <= 0 {
+		ffPoolSize = runtime.GOMAXPROCS(0)
+	}
+
+	if err := ffmpeg.InitFfmpeg(ctx, ffPoolSize); err != nil {
 		return gtserror.Newf("error compiling ffmpeg: %w", err)
 	}
-	if err := ffmpeg.InitFfprobe(ctx, maxprocs); err != nil {
+
+	if err := ffmpeg.InitFfprobe(ctx, ffPoolSize); err != nil {
 		return gtserror.Newf("error compiling ffprobe: %w", err)
 	}
+
 	return nil
 }

docs/configuration/media.md

@@ -56,6 +56,24 @@ media-emoji-local-max-size: 50KiB
 # Default: 100KiB (102400 bytes)
 media-emoji-remote-max-size: 100KiB
 
+# Int. Number of instances of ffmpeg+ffprobe to add to the media processing pool.
+#
+# Increasing this number will lead to faster concurrent media processing,
+# but at the cost of up to about 250MB of (spiking) memory usage per increment.
+#
+# You'll want to increase this number if you have RAM to spare, and/or if you're
+# hosting an instance for more than 50 or so people who post/view lots of media,
+# but you should leave it at 1 for single-user instances or when running GoToSocial
+# in a constrained (low-memory) environment.
+#
+# If you set this number to 0 or less, then instead of a fixed number of instances,
+# it will scale with GOMAXPROCS x 1, yielding (usually) one ffmpeg instance and one
+# ffprobe instance per CPU core on the host machine.
+#
+# Examples: [1, 2, -1, 8]
+# Default: 1
+media-ffmpeg-pool-size: 1
+
 # The below media cleanup settings allow admins to customize when and
 # how often media cleanup + prune jobs run, while being set to a fairly
 # sensible default (every night @ midnight). For more information on exactly

example/config.yaml

@@ -493,6 +493,24 @@ media-emoji-local-max-size: 50KiB
 # Default: 100KiB (102400 bytes)
 media-emoji-remote-max-size: 100KiB
 
+# Int. Number of instances of ffmpeg+ffprobe to add to the media processing pool.
+#
+# Increasing this number will lead to faster concurrent media processing,
+# but at the cost of up to about 250MB of (spiking) memory usage per increment.
+#
+# You'll want to increase this number if you have RAM to spare, and/or if you're
+# hosting an instance for more than 50 or so people who post/view lots of media,
+# but you should leave it at 1 for single-user instances or when running GoToSocial
+# in a constrained (low-memory) environment.
+#
+# If you set this number to 0 or less, then instead of a fixed number of instances,
+# it will scale with GOMAXPROCS x 1, yielding (usually) one ffmpeg instance and one
+# ffprobe instance per CPU core on the host machine.
+#
+# Examples: [1, 2, -1, 8]
+# Default: 1
+media-ffmpeg-pool-size: 1
+
 # The below media cleanup settings allow admins to customize when and
 # how often media cleanup + prune jobs run, while being set to a fairly
 # sensible default (every night @ midnight). For more information on exactly

internal/config/config.go

@@ -101,6 +101,7 @@ type Configuration struct {
 	MediaRemoteMaxSize       bytesize.Size `name:"media-remote-max-size" usage:"Max size in bytes of media to download from other instances"`
 	MediaCleanupFrom         string        `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
 	MediaCleanupEvery        time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`
+	MediaFfmpegPoolSize      int           `name:"media-ffmpeg-pool-size" usage:"Number of instances of the embedded ffmpeg WASM binary to add to the media processing pool. 0 or less uses GOMAXPROCS."`
 
 	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
 	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`

internal/config/defaults.go

@@ -80,6 +80,7 @@ var Defaults = Configuration{
 	MediaEmojiRemoteMaxSize:  100 * bytesize.KiB,
 	MediaCleanupFrom:         "00:00",        // Midnight.
 	MediaCleanupEvery:        24 * time.Hour, // 1/day.
+	MediaFfmpegPoolSize:      1,
 
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",

internal/config/helpers.gen.go

@@ -1300,6 +1300,31 @@ func GetMediaCleanupEvery() time.Duration { return global.GetMediaCleanupEvery()
 // SetMediaCleanupEvery safely sets the value for global configuration 'MediaCleanupEvery' field
 func SetMediaCleanupEvery(v time.Duration) { global.SetMediaCleanupEvery(v) }
 
+// GetMediaFfmpegPoolSize safely fetches the Configuration value for state's 'MediaFfmpegPoolSize' field
+func (st *ConfigState) GetMediaFfmpegPoolSize() (v int) {
+	st.mutex.RLock()
+	v = st.config.MediaFfmpegPoolSize
+	st.mutex.RUnlock()
+	return
+}
+
+// SetMediaFfmpegPoolSize safely sets the Configuration value for state's 'MediaFfmpegPoolSize' field
+func (st *ConfigState) SetMediaFfmpegPoolSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaFfmpegPoolSize = v
+	st.reloadToViper()
+}
+
+// MediaFfmpegPoolSizeFlag returns the flag name for the 'MediaFfmpegPoolSize' field
+func MediaFfmpegPoolSizeFlag() string { return "media-ffmpeg-pool-size" }
+
+// GetMediaFfmpegPoolSize safely fetches the value for global configuration 'MediaFfmpegPoolSize' field
+func GetMediaFfmpegPoolSize() int { return global.GetMediaFfmpegPoolSize() }
+
+// SetMediaFfmpegPoolSize safely sets the value for global configuration 'MediaFfmpegPoolSize' field
+func SetMediaFfmpegPoolSize(v int) { global.SetMediaFfmpegPoolSize(v) }
+
 // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field
 func (st *ConfigState) GetStorageBackend() (v string) {
 	st.mutex.RLock()

test/envparsing.sh

@@ -126,6 +126,7 @@ EXPECT=$(cat << "EOF"
     "media-description-min-chars": 69,
     "media-emoji-local-max-size": 420,
     "media-emoji-remote-max-size": 420,
+    "media-ffmpeg-pool-size": 8,
     "media-local-max-size": 420,
     "media-remote-cache-days": 30,
     "media-remote-max-size": 420,
@@ -245,6 +246,7 @@ GTS_MEDIA_REMOTE_MAX_SIZE=420 \
 GTS_MEDIA_REMOTE_CACHE_DAYS=30 \
 GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \
 GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \
+GTS_MEDIA_FFMPEG_POOL_SIZE=8 \
 GTS_METRICS_AUTH_ENABLED=false \
 GTS_METRICS_ENABLED=false \
 GTS_STORAGE_BACKEND='local' \