cock

cmd/gotosocial/action/admin/account/account.go

@@ -40,7 +40,8 @@ func initState(ctx context.Context) (*state.State, error) {
 	state.Caches.Init()
 	state.Caches.Start()
 
-	// Set the state DB connection
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this (yet).
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbConn: %w", err)

cmd/gotosocial/action/admin/media/list.go

@@ -127,6 +127,8 @@ func setupList(ctx context.Context) (*list, error) {
 	state.Caches.Init()
 	state.Caches.Start()
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbService, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)

cmd/gotosocial/action/admin/media/prune/common.go

@@ -45,10 +45,12 @@ func setupPrune(ctx context.Context) (*prune, error) {
 	state.Caches.Start()
 
 	// Scheduler is required for the
-	// claner, but no other workers
+	// cleaner, but no other workers
 	// are needed for this CLI action.
 	state.Workers.StartScheduler()
 
+	// Set state DB connection.
+	// Don't need Actions for this.
 	dbService, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)

cmd/gotosocial/action/admin/trans/export.go

@@ -33,12 +33,12 @@
 var Export action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
-
-	// Set the state DB connection
 	state.DB = dbConn
 
 	exporter := trans.NewExporter(dbConn)

cmd/gotosocial/action/admin/trans/import.go

@@ -33,12 +33,12 @@
 var Import action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
-
-	// Set the state DB connection
 	state.DB = dbConn
 
 	importer := trans.NewImporter(dbConn)

cmd/gotosocial/action/server/server.go

@@ -32,6 +32,7 @@
 	"github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/gin-gonic/gin"
 	"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
@@ -44,6 +45,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/metrics"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
 	tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"go.uber.org/automaxprocs/maxprocs"
@@ -164,6 +166,10 @@
 	// Set DB on state.
 	state.DB = dbService
 
+	// Set Actions on state, providing workers to
+	// Actions as well for triggering side effects.
+	state.Actions = actions.New(dbService, &state.Workers)
+
 	// Ensure necessary database instance prerequisites exist.
 	if err := dbService.CreateInstanceAccount(ctx); err != nil {
 		return fmt.Errorf("error creating instance account: %s", err)
@@ -283,15 +289,18 @@ func(context.Context, time.Time) {
 	// Create background cleaner.
 	cleaner := cleaner.New(state)
 
-	// Now schedule background cleaning tasks.
+	// Create subscriptions fetcher.
-	if err := cleaner.ScheduleJobs(); err != nil {
+	subscriptions := subscriptions.New(
-		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
+		state,
-	}
+		transportController,
+		typeConverter,
+	)
 
 	// Create the processor using all the
 	// other services we've created so far.
 	process = processing.NewProcessor(
 		cleaner,
+		subscriptions,
 		typeConverter,
 		federator,
 		oauthServer,
@@ -302,6 +311,16 @@ func(context.Context, time.Time) {
 		intFilter,
 	)
 
+	// Schedule background cleaning tasks.
+	if err := cleaner.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
+	}
+
+	// Schedule background subscriptions updating.
+	if err := subscriptions.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
+	}
+
 	// Initialize the specialized workers pools.
 	state.Workers.Client.Init(messages.ClientMsgIndices())
 	state.Workers.Federator.Init(messages.FederatorMsgIndices())

cmd/gotosocial/action/testrig/testrig.go

@@ -47,6 +47,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/router"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -314,11 +315,23 @@
 	// Create background cleaner.
 	cleaner := cleaner.New(state)
 
-	// Now schedule background cleaning tasks.
+	// Schedule background cleaning tasks.
 	if err := cleaner.ScheduleJobs(); err != nil {
 		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
 	}
 
+	// Create subscriptions fetcher.
+	subscriptions := subscriptions.New(
+		state,
+		transportController,
+		typeConverter,
+	)
+
+	// Schedule background subscriptions updating.
+	if err := subscriptions.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
+	}
+
 	// Finally start the main http server!
 	if err := route.Start(); err != nil {
 		return fmt.Errorf("error starting router: %w", err)

internal/actions/actions.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package admin
+package actions
 
 import (
 	"context"
@@ -23,11 +23,12 @@
 	"sync"
 	"time"
 
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
-	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/workers"
 )
 
 func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode {
@@ -42,15 +43,34 @@ func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode {
 }
 
 type Actions struct {
-	r     map[string]*gtsmodel.AdminAction
+	// Map of running actions.
-	state *state.State
+	running map[string]*gtsmodel.AdminAction
 
-	// Not embedded struct,
+	// Lock for running admin actions.
-	// to shield from access
+	//
-	// by outside packages.
+	// Not embedded struct, to shield
+	// from access by outside packages.
 	m sync.Mutex
+
+	// DB for storing, updating,
+	// deleting admin actions etc.
+	db db.DB
+
+	// Workers for queuing
+	// admin action side effects.
+	workers *workers.Workers
 }
 
+func New(db db.DB, workers *workers.Workers) *Actions {
+	return &Actions{
+		running: make(map[string]*gtsmodel.AdminAction),
+		db:      db,
+		workers: workers,
+	}
+}
+
+type AdminActionF func(context.Context) gtserror.MultiError
+
 // Run runs the given admin action by executing the supplied function.
 //
 // Run handles locking, action insertion and updating, so you don't have to!
@@ -62,10 +82,10 @@ type Actions struct {
 // will be updated on the provided admin action in the database.
 func (a *Actions) Run(
 	ctx context.Context,
-	action *gtsmodel.AdminAction,
+	adminAction *gtsmodel.AdminAction,
-	f func(context.Context) gtserror.MultiError,
+	f AdminActionF,
 ) gtserror.WithCode {
-	actionKey := action.Key()
+	actionKey := adminAction.Key()
 
 	// LOCK THE MAP HERE, since we're
 	// going to do some operations on it.
@@ -73,7 +93,7 @@ func (a *Actions) Run(
 
 	// Bail if an action with
 	// this key is already running.
-	running, ok := a.r[actionKey]
+	running, ok := a.running[actionKey]
 	if ok {
 		a.m.Unlock()
 		return errActionConflict(running)
@@ -81,7 +101,7 @@ func (a *Actions) Run(
 
 	// Action with this key not
 	// yet running, create it.
-	if err := a.state.DB.PutAdminAction(ctx, action); err != nil {
+	if err := a.db.PutAdminAction(ctx, adminAction); err != nil {
 		err = gtserror.Newf("db error putting admin action %s: %w", actionKey, err)
 
 		// Don't store in map
@@ -92,7 +112,7 @@ func (a *Actions) Run(
 
 	// Action was inserted,
 	// store in map.
-	a.r[actionKey] = action
+	a.running[actionKey] = adminAction
 
 	// UNLOCK THE MAP HERE, since
 	// we're done modifying it for now.
@@ -104,22 +124,22 @@ func (a *Actions) Run(
 
 		// Run the thing and collect errors.
 		if errs := f(ctx); errs != nil {
-			action.Errors = make([]string, 0, len(errs))
+			adminAction.Errors = make([]string, 0, len(errs))
 			for _, err := range errs {
-				action.Errors = append(action.Errors, err.Error())
+				adminAction.Errors = append(adminAction.Errors, err.Error())
 			}
 		}
 
 		// Action is no longer running:
 		// remove from running map.
 		a.m.Lock()
-		delete(a.r, actionKey)
+		delete(a.running, actionKey)
 		a.m.Unlock()
 
 		// Mark as completed in the db,
 		// storing errors for later review.
-		action.CompletedAt = time.Now()
+		adminAction.CompletedAt = time.Now()
-		if err := a.state.DB.UpdateAdminAction(ctx, action, "completed_at", "errors"); err != nil {
+		if err := a.db.UpdateAdminAction(ctx, adminAction, "completed_at", "errors"); err != nil {
 			log.Errorf(ctx, "db error marking action %s as completed: %q", actionKey, err)
 		}
 	}()
@@ -135,8 +155,8 @@ func (a *Actions) GetRunning() []*gtsmodel.AdminAction {
 	defer a.m.Unlock()
 
 	// Assemble all currently running actions.
-	running := make([]*gtsmodel.AdminAction, 0, len(a.r))
+	running := make([]*gtsmodel.AdminAction, 0, len(a.running))
-	for _, action := range a.r {
+	for _, action := range a.running {
 		running = append(running, action)
 	}
 
@@ -166,5 +186,5 @@ func (a *Actions) TotalRunning() int {
 	a.m.Lock()
 	defer a.m.Unlock()
 
-	return len(a.r)
+	return len(a.running)
 }

internal/actions/actions_test.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package admin_test
+package actions_test
 
 import (
 	"context"
@@ -32,12 +32,26 @@
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
+const (
+	rMediaPath    = "../../testrig/media"
+	rTemplatePath = "../../web/template"
+)
+
 type ActionsTestSuite struct {
-	AdminStandardTestSuite
+	suite.Suite
+}
+
+func (suite *ActionsTestSuite) SetupSuite() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
 }
 
 func (suite *ActionsTestSuite) TestActionOverlap() {
-	ctx := context.Background()
+	var (
+		testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		ctx         = context.Background()
+	)
+	defer testrig.TearDownTestStructs(testStructs)
 
 	// Suspend account.
 	action1 := &gtsmodel.AdminAction{
@@ -61,7 +75,7 @@ func (suite *ActionsTestSuite) TestActionOverlap() {
 	key2 := action2.Key()
 	suite.Equal("account/01H90S1CXQ97J9625C5YBXZWGT", key2)
 
-	errWithCode := suite.adminProcessor.Actions().Run(
+	errWithCode := testStructs.State.Actions.Run(
 		ctx,
 		action1,
 		func(ctx context.Context) gtserror.MultiError {
@@ -74,7 +88,7 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// While first action is sleeping, try to
 	// process another with the same key.
-	errWithCode = suite.adminProcessor.Actions().Run(
+	errWithCode = testStructs.State.Actions.Run(
 		ctx,
 		action2,
 		func(ctx context.Context) gtserror.MultiError {
@@ -90,13 +104,13 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return testStructs.State.Actions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}
 
 	// Try again.
-	errWithCode = suite.adminProcessor.Actions().Run(
+	errWithCode = testStructs.State.Actions.Run(
 		ctx,
 		action2,
 		func(ctx context.Context) gtserror.MultiError {
@@ -107,14 +121,18 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return testStructs.State.Actions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}
 }
 
 func (suite *ActionsTestSuite) TestActionWithErrors() {
-	ctx := context.Background()
+	var (
+		testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		ctx         = context.Background()
+	)
+	defer testrig.TearDownTestStructs(testStructs)
 
 	// Suspend a domain.
 	action := &gtsmodel.AdminAction{
@@ -125,7 +143,7 @@ func (suite *ActionsTestSuite) TestActionWithErrors() {
 		AccountID:      "01H90S1ZZXP4N74H4A9RVW1MRP",
 	}
 
-	errWithCode := suite.adminProcessor.Actions().Run(
+	errWithCode := testStructs.State.Actions.Run(
 		ctx,
 		action,
 		func(ctx context.Context) gtserror.MultiError {
@@ -140,13 +158,13 @@ func(ctx context.Context) gtserror.MultiError {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return testStructs.State.Actions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}
 
 	// Get action from the db.
-	dbAction, err := suite.db.GetAdminAction(ctx, action.ID)
+	dbAction, err := testStructs.State.DB.GetAdminAction(ctx, action.ID)
 	if err != nil {
 		suite.FailNow(err.Error())
 	}

internal/actions/domainkeys.go

@@ -0,0 +1,51 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package actions
+
+import (
+	"context"
+	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (a *Actions) DomainKeysExpireF(domain string) AdminActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		var (
+			expiresAt = time.Now()
+			errs      gtserror.MultiError
+		)
+
+		// For each account on this domain, expire
+		// the public key and update the account.
+		if err := a.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) {
+			account.PublicKeyExpiresAt = expiresAt
+			if err := a.db.UpdateAccount(ctx,
+				account,
+				"public_key_expires_at",
+			); err != nil {
+				errs.Appendf("db error updating account: %w", err)
+			}
+		}); err != nil {
+			errs.Appendf("db error ranging through accounts: %w", err)
+		}
+
+		return errs
+	}
+}

internal/actions/domainperms.go

@@ -0,0 +1,387 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package actions
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"codeberg.org/gruf/go-kv"
+
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+)
+
+// Returns an AdminActionF for
+// domain allow side effects.
+func (a *Actions) DomainAllowF(
+	actionID string,
+	domainAllow *gtsmodel.DomainAllow,
+) AdminActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "allow"},
+				{"actionID", actionID},
+				{"domain", domainAllow.Domain},
+			}...)
+
+		// Log start + finish.
+		l.Info("processing side effects")
+		errs := a.domainAllowSideEffects(ctx, domainAllow)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+func (a *Actions) domainAllowSideEffects(
+	ctx context.Context,
+	allow *gtsmodel.DomainAllow,
+) gtserror.MultiError {
+	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
+		// We're running in allowlist mode,
+		// so there are no side effects to
+		// process here.
+		return nil
+	}
+
+	// We're running in blocklist mode or
+	// some similar mode which necessitates
+	// domain allow side effects if a block
+	// was in place when the allow was created.
+	//
+	// So, check if there's a block.
+	block, err := a.db.GetDomainBlock(ctx, allow.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs := gtserror.NewMultiError(1)
+		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
+		return errs
+	}
+
+	if block == nil {
+		// No block?
+		// No problem!
+		return nil
+	}
+
+	// There was a block, over which the new
+	// allow ought to take precedence. To account
+	// for this, just run side effects as though
+	// the domain was being unblocked, while
+	// leaving the existing block in place.
+	//
+	// Any accounts that were suspended by
+	// the block will be unsuspended and be
+	// able to interact with the instance again.
+	return a.domainUnblockSideEffects(ctx, block)
+}
+
+// Returns an AdminActionF for
+// domain unallow side effects.
+func (a *Actions) DomainUnallowF(
+	actionID string,
+	domainAllow *gtsmodel.DomainAllow,
+) AdminActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "unallow"},
+				{"actionID", actionID},
+				{"domain", domainAllow.Domain},
+			}...)
+
+		// Log start + finish.
+		l.Info("processing side effects")
+		errs := a.domainUnallowSideEffects(ctx, domainAllow)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+func (a *Actions) domainUnallowSideEffects(
+	ctx context.Context,
+	allow *gtsmodel.DomainAllow,
+) gtserror.MultiError {
+	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
+		// We're running in allowlist mode,
+		// so there are no side effects to
+		// process here.
+		return nil
+	}
+
+	// We're running in blocklist mode or
+	// some similar mode which necessitates
+	// domain allow side effects if a block
+	// was in place when the allow was removed.
+	//
+	// So, check if there's a block.
+	block, err := a.db.GetDomainBlock(ctx, allow.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs := gtserror.NewMultiError(1)
+		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
+		return errs
+	}
+
+	if block == nil {
+		// No block?
+		// No problem!
+		return nil
+	}
+
+	// There was a block, over which the previous
+	// allow was taking precedence. Now that the
+	// allow has been removed, we should put the
+	// side effects of the block back in place.
+	//
+	// To do this, process the block side effects
+	// again as though the block were freshly
+	// created. This will mark all accounts from
+	// the blocked domain as suspended, and clean
+	// up their follows/following, media, etc.
+	return a.domainBlockSideEffects(ctx, block)
+}
+
+func (a *Actions) DomainBlockF(
+	actionID string,
+	domainBlock *gtsmodel.DomainBlock,
+) AdminActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "block"},
+				{"actionID", actionID},
+				{"domain", domainBlock.Domain},
+			}...)
+
+		skip, err := a.skipBlockSideEffects(ctx, domainBlock.Domain)
+		if err != nil {
+			return err
+		}
+
+		if skip != "" {
+			l.Infof("skipping side effects: %s", skip)
+			return nil
+		}
+
+		l.Info("processing side effects")
+		errs := a.domainBlockSideEffects(ctx, domainBlock)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+// domainBlockSideEffects processes the side effects of a domain block:
+//
+//  1. Strip most info away from the instance entry for the domain.
+//  2. Pass each account from the domain to the processor for deletion.
+//
+// It should be called asynchronously, since it can take a while when
+// there are many accounts present on the given domain.
+func (a *Actions) domainBlockSideEffects(
+	ctx context.Context,
+	block *gtsmodel.DomainBlock,
+) gtserror.MultiError {
+	var errs gtserror.MultiError
+
+	// If we have an instance entry for this domain,
+	// update it with the new block ID and clear all fields
+	instance, err := a.db.GetInstance(ctx, block.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
+		return errs
+	}
+
+	if instance != nil {
+		// We had an entry for this domain.
+		columns := stubbifyInstance(instance, block.ID)
+		if err := a.db.UpdateInstance(ctx, instance, columns...); err != nil {
+			errs.Appendf("db error updating instance: %w", err)
+			return errs
+		}
+	}
+
+	// For each account that belongs to this domain,
+	// process an account delete message to remove
+	// that account's posts, media, etc.
+	if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
+		if err := a.workers.Client.Process(ctx, &messages.FromClientAPI{
+			APObjectType:   ap.ActorPerson,
+			APActivityType: ap.ActivityDelete,
+			GTSModel:       block,
+			Origin:         account,
+			Target:         account,
+		}); err != nil {
+			errs.Append(err)
+		}
+	}); err != nil {
+		errs.Appendf("db error ranging through accounts: %w", err)
+	}
+
+	return errs
+}
+
+func (a *Actions) DomainUnblockF(
+	actionID string,
+	domainBlock *gtsmodel.DomainBlock,
+) AdminActionF {
+	return func(ctx context.Context) gtserror.MultiError {
+		l := log.
+			WithContext(ctx).
+			WithFields(kv.Fields{
+				{"action", "unblock"},
+				{"actionID", actionID},
+				{"domain", domainBlock.Domain},
+			}...)
+
+		l.Info("processing side effects")
+		errs := a.domainUnblockSideEffects(ctx, domainBlock)
+		l.Info("finished processing side effects")
+
+		return errs
+	}
+}
+
+// domainUnblockSideEffects processes the side effects of undoing a
+// domain block:
+//
+//  1. Mark instance entry as no longer suspended.
+//  2. Mark each account from the domain as no longer suspended, if the
+//     suspension origin corresponds to the ID of the provided domain block.
+//
+// It should be called asynchronously, since it can take a while when
+// there are many accounts present on the given domain.
+func (a *Actions) domainUnblockSideEffects(
+	ctx context.Context,
+	block *gtsmodel.DomainBlock,
+) gtserror.MultiError {
+	var errs gtserror.MultiError
+
+	// Update instance entry for this domain, if we have it.
+	instance, err := a.db.GetInstance(ctx, block.Domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
+	}
+
+	if instance != nil {
+		// We had an entry, update it to signal
+		// that it's no longer suspended.
+		instance.SuspendedAt = time.Time{}
+		instance.DomainBlockID = ""
+		if err := a.db.UpdateInstance(
+			ctx,
+			instance,
+			"suspended_at",
+			"domain_block_id",
+		); err != nil {
+			errs.Appendf("db error updating instance: %w", err)
+			return errs
+		}
+	}
+
+	// Unsuspend all accounts whose suspension origin was this domain block.
+	if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
+		if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() {
+			// Account wasn't suspended, nothing to do.
+			return
+		}
+
+		if account.SuspensionOrigin != block.ID {
+			// Account was suspended, but not by
+			// this domain block, leave it alone.
+			return
+		}
+
+		// Account was suspended by this domain
+		// block, mark it as unsuspended.
+		account.SuspendedAt = time.Time{}
+		account.SuspensionOrigin = ""
+
+		if err := a.db.UpdateAccount(
+			ctx,
+			account,
+			"suspended_at",
+			"suspension_origin",
+		); err != nil {
+			errs.Appendf("db error updating account %s: %w", account.Username, err)
+		}
+	}); err != nil {
+		errs.Appendf("db error ranging through accounts: %w", err)
+	}
+
+	return errs
+}
+
+// skipBlockSideEffects checks if side effects of block creation
+// should be skipped for the given domain, taking account of
+// instance federation mode, and existence of any allows
+// which ought to "shield" this domain from being blocked.
+//
+// If the caller should skip, the returned string will be non-zero
+// and will be set to a reason why side effects should be skipped.
+//
+//   - blocklist mode + allow exists: "..." (skip)
+//   - blocklist mode + no allow:     ""    (don't skip)
+//   - allowlist mode + allow exists: ""    (don't skip)
+//   - allowlist mode + no allow:     ""    (don't skip)
+func (a *Actions) skipBlockSideEffects(
+	ctx context.Context,
+	domain string,
+) (string, gtserror.MultiError) {
+	var (
+		skip string // Assume "" (don't skip).
+		errs gtserror.MultiError
+	)
+
+	// Never skip block side effects in allowlist mode.
+	fediMode := config.GetInstanceFederationMode()
+	if fediMode == config.InstanceFederationModeAllowlist {
+		return skip, errs
+	}
+
+	// We know we're in blocklist mode.
+	//
+	// We want to skip domain block side
+	// effects if an allow is already
+	// in place which overrides the block.
+
+	// Check if an explicit allow exists for this domain.
+	domainAllow, err := a.db.GetDomainAllow(ctx, domain)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		errs.Appendf("error getting domain allow: %w", err)
+		return skip, errs
+	}
+
+	if domainAllow != nil {
+		skip = "running in blocklist mode, and an explicit allow exists for this domain"
+		return skip, errs
+	}
+
+	return skip, errs
+}

internal/actions/util.go

@@ -0,0 +1,99 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package actions
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// stubbifyInstance renders the given instance as a stub,
+// removing most information from it and marking it as
+// suspended.
+//
+// For caller's convenience, this function returns the db
+// names of all columns that are updated by it.
+func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string {
+	instance.Title = ""
+	instance.SuspendedAt = time.Now()
+	instance.DomainBlockID = domainBlockID
+	instance.ShortDescription = ""
+	instance.Description = ""
+	instance.Terms = ""
+	instance.ContactEmail = ""
+	instance.ContactAccountUsername = ""
+	instance.ContactAccountID = ""
+	instance.Version = ""
+
+	return []string{
+		"title",
+		"suspended_at",
+		"domain_block_id",
+		"short_description",
+		"description",
+		"terms",
+		"contact_email",
+		"contact_account_username",
+		"contact_account_id",
+		"version",
+	}
+}
+
+// rangeDomainAccounts iterates through all accounts
+// originating from the given domain, and calls the
+// provided range function on each account.
+//
+// If an error is returned while selecting accounts,
+// the loop will stop and return the error.
+func (a *Actions) rangeDomainAccounts(
+	ctx context.Context,
+	domain string,
+	rangeF func(*gtsmodel.Account),
+) error {
+	var (
+		limit = 50   // Limit selection to avoid spiking mem/cpu.
+		maxID string // Start with empty string to select from top.
+	)
+
+	for {
+		// Get (next) page of accounts.
+		accounts, err := a.db.GetInstanceAccounts(ctx, domain, maxID, limit)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			// Real db error.
+			return gtserror.Newf("db error getting instance accounts: %w", err)
+		}
+
+		if len(accounts) == 0 {
+			// No accounts left, we're done.
+			return nil
+		}
+
+		// Set next max ID for paging down.
+		maxID = accounts[len(accounts)-1].ID
+
+		// Call provided range function.
+		for _, account := range accounts {
+			rangeF(account)
+		}
+	}
+}

internal/api/activitypub/emoji/emojiget_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/emoji"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -73,6 +74,7 @@ func (suite *EmojiGetTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 	suite.tc = typeutils.NewConverter(&suite.state)

internal/api/activitypub/users/user_test.go

@@ -20,6 +20,7 @@
 import (
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -84,6 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(

internal/api/auth/auth_test.go

@@ -26,6 +26,7 @@
 	"github.com/gin-contrib/sessions/memstore"
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/auth"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -84,6 +85,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 	suite.mediaManager = testrig.NewTestMediaManager(&suite.state)

internal/api/client/accounts/account_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/accounts"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -85,6 +86,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/admin/admin.go

@@ -46,6 +46,7 @@
 	DomainPermissionSubscriptionsPathWithID  = DomainPermissionSubscriptionsPath + "/:" + apiutil.IDKey
 	DomainPermissionSubscriptionsPreviewPath = DomainPermissionSubscriptionsPath + "/preview"
 	DomainPermissionSubscriptionRemovePath   = DomainPermissionSubscriptionsPathWithID + "/remove"
+	DomainPermissionSubscriptionTestPath     = DomainPermissionSubscriptionsPathWithID + "/test"
 	DomainKeysExpirePath                     = BasePath + "/domain_keys_expire"
 	HeaderAllowsPath                         = BasePath + "/header_allows"
 	HeaderAllowsPathWithID                   = HeaderAllowsPath + "/:" + apiutil.IDKey
@@ -129,6 +130,7 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
 	attachHandler(http.MethodGet, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionGETHandler)
 	attachHandler(http.MethodPatch, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionPATCHHandler)
 	attachHandler(http.MethodPost, DomainPermissionSubscriptionRemovePath, m.DomainPermissionSubscriptionRemovePOSTHandler)
+	attachHandler(http.MethodPost, DomainPermissionSubscriptionTestPath, m.DomainPermissionSubscriptionTestPOSTHandler)
 
 	// header filtering administration routes
 	attachHandler(http.MethodGet, HeaderAllowsPathWithID, m.HeaderFilterAllowGET)

internal/api/client/admin/admin_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -91,6 +92,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/admin/domainpermissionsubscriptiontest.go

@@ -0,0 +1,118 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// DomainPermissionSubscriptionTestPOSTHandler swagger:operation POST /api/v1/admin/domain_permission_subscriptions/{id}/test domainPermissionSubscriptionTest
+//
+// Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*.
+//
+// The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message.
+//
+// This is useful in cases where you want to check that your instance can actually fetch + parse a list.
+//
+//	---
+//	tags:
+//	- admin
+//
+//	produces:
+//	- application/json
+//
+//	parameters:
+//	-
+//		name: id
+//		required: true
+//		in: path
+//		description: ID of the domain permission draft.
+//		type: string
+//
+//	security:
+//	- OAuth2 Bearer:
+//		- admin
+//
+//	responses:
+//		'200':
+//			description: >-
+//				Either an array of domain permissions, OR an error message of the form
+//				`{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched.
+//			schema:
+//				type: array
+//				items:
+//					"$ref": "#/definitions/domain"
+//		'400':
+//			description: bad request
+//		'401':
+//			description: unauthorized
+//		'403':
+//			description: forbidden
+//		'406':
+//			description: not acceptable
+//		'409':
+//			description: conflict
+//		'500':
+//			description: internal server error
+func (m *Module) DomainPermissionSubscriptionTestPOSTHandler(c *gin.Context) {
+	authed, err := oauth.Authed(c, true, true, true, true)
+	if err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if !*authed.User.Admin {
+		err := fmt.Errorf("user %s not an admin", authed.User.ID)
+		apiutil.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if authed.Account.IsMoving() {
+		apiutil.ForbiddenAfterMove(c)
+		return
+	}
+
+	if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey))
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	resp, errWithCode := m.processor.Admin().DomainPermissionSubscriptionTest(
+		c.Request.Context(),
+		authed.Account,
+		id,
+	)
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	apiutil.JSON(c, http.StatusOK, resp)
+}

internal/api/client/admin/domainpermissionsubscruptiontest_test.go

@@ -0,0 +1,125 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+type DomainPermissionSubscriptionTestTestSuite struct {
+	AdminStandardTestSuite
+}
+
+func (suite *DomainPermissionSubscriptionTestTestSuite) TestDomainPermissionSubscriptionTest() {
+	var (
+		ctx         = context.Background()
+		testAccount = suite.testAccounts["admin_account"]
+		permSub     = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+
+	// Create a subscription for a CSV list of baddies.
+	err := suite.state.DB.PutDomainPermissionSubscription(ctx, permSub)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Prepare the request to the /test endpoint.
+	subPath := strings.ReplaceAll(
+		admin.DomainPermissionSubscriptionTestPath,
+		":id", permSub.ID,
+	)
+	path := "/api" + subPath
+	recorder := httptest.NewRecorder()
+	ginCtx := suite.newContext(recorder, http.MethodPost, nil, path, "application/json")
+	ginCtx.Params = gin.Params{
+		gin.Param{
+			Key:   apiutil.IDKey,
+			Value: permSub.ID,
+		},
+	}
+
+	// Trigger the handler.
+	suite.adminModule.DomainPermissionSubscriptionTestPOSTHandler(ginCtx)
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// Read the body back.
+	b, err := io.ReadAll(recorder.Body)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	dst := new(bytes.Buffer)
+	if err := json.Indent(dst, b, "", "  "); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Ensure expected.
+	suite.Equal(`[
+  {
+    "domain": "bumfaces.net",
+    "public_comment": "big jerks"
+  },
+  {
+    "domain": "peepee.poopoo",
+    "public_comment": "harassment"
+  },
+  {
+    "domain": "nothanks.com"
+  }
+]`, dst.String())
+
+	// No permissions should be created
+	// since this is a dry run / test.
+	blocked, err := suite.state.DB.AreDomainsBlocked(
+		ctx,
+		[]string{"bumfaces.net", "peepee.poopoo", "nothanks.com"},
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+	suite.False(blocked)
+}
+
+func TestDomainPermissionSubscriptionTestTestSuite(t *testing.T) {
+	suite.Run(t, &DomainPermissionSubscriptionTestTestSuite{})
+}

internal/api/client/bookmarks/bookmarks_test.go

@@ -28,6 +28,7 @@
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/bookmarks"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
@@ -95,6 +96,7 @@ func (suite *BookmarkTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/favourites/favourites_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/favourites"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -79,6 +80,7 @@ func (suite *FavouritesStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/filters/v1/filter_test.go

@@ -23,6 +23,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	filtersV1 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v1"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/filters/v2/filter_test.go

@@ -23,6 +23,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	filtersV2 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v2"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/followedtags/followedtags_test.go

@@ -21,6 +21,7 @@
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/followedtags"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -79,6 +80,7 @@ func (suite *FollowedTagsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/followrequests/followrequest_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequests"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -82,6 +83,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/instance/instance_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -84,6 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/lists/lists_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/lists"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -85,6 +86,7 @@ func (suite *ListsStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/mutes/mutes_test.go

@@ -25,6 +25,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/mutes"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -81,6 +82,7 @@ func (suite *MutesTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/notifications/notifications_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/notifications"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -81,6 +82,7 @@ func (suite *NotificationsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/polls/polls_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/polls"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -76,6 +77,7 @@ func (suite *PollsStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/reports/reports_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/reports"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -76,6 +77,7 @@ func (suite *ReportsStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/search/search_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/search"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -80,6 +81,7 @@ func (suite *SearchStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/statuses/status_test.go

@@ -25,6 +25,7 @@
 	"strings"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -192,6 +193,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/streaming/streaming_test.go

@@ -31,6 +31,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/streaming"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -92,6 +93,7 @@ func (suite *StreamingTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/tags/tags_test.go

@@ -26,6 +26,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/tags"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
@@ -87,6 +88,7 @@ func (suite *TagsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/user/user_test.go

@@ -24,6 +24,7 @@
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/user"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -72,6 +73,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/fileserver/fileserver_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -98,6 +99,7 @@ func (suite *FileserverTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")

internal/api/wellknown/webfinger/webfinger_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/api/wellknown/webfinger"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -79,6 +80,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(

internal/api/wellknown/webfinger/webfingerget_test.go

@@ -39,6 +39,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -90,6 +91,7 @@ func (suite *WebfingerGetTestSuite) funkifyAccountDomain(host string, accountDom
 
 	suite.processor = processing.NewProcessor(
 		cleaner.New(&suite.state),
+		subscriptions.New(&suite.state, suite.federator.TransportController(), suite.tc),
 		suite.tc,
 		suite.federator,
 		testrig.NewTestOauthServer(suite.db),

internal/cleaner/media_test.go

@@ -26,6 +26,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
@@ -67,6 +68,7 @@ func (suite *MediaTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.state.Storage = suite.storage
 
 	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

internal/config/config.go

@@ -78,15 +78,17 @@ type Configuration struct {
 	WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
 	WebAssetBaseDir    string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`
 
-	InstanceFederationMode         string             `name:"instance-federation-mode" usage:"Set instance federation mode."`
+	InstanceFederationMode            string             `name:"instance-federation-mode" usage:"Set instance federation mode."`
-	InstanceFederationSpamFilter   bool               `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"`
+	InstanceFederationSpamFilter      bool               `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"`
-	InstanceExposePeers            bool               `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
+	InstanceExposePeers               bool               `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
-	InstanceExposeSuspended        bool               `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
+	InstanceExposeSuspended           bool               `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
-	InstanceExposeSuspendedWeb     bool               `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"`
+	InstanceExposeSuspendedWeb        bool               `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"`
-	InstanceExposePublicTimeline   bool               `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"`
+	InstanceExposePublicTimeline      bool               `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"`
-	InstanceDeliverToSharedInboxes bool               `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."`
+	InstanceDeliverToSharedInboxes    bool               `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."`
-	InstanceInjectMastodonVersion  bool               `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"`
+	InstanceInjectMastodonVersion     bool               `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"`
-	InstanceLanguages              language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."`
+	InstanceLanguages                 language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."`
+	InstanceSubscriptionsProcessFrom  string             `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
+	InstanceSubscriptionsProcessEvery time.Duration      `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."`
 
 	AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."`
 	AccountsReasonRequired   bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`

internal/config/defaults.go

@@ -58,13 +58,15 @@
 	WebTemplateBaseDir: "./web/template/",
 	WebAssetBaseDir:    "./web/assets/",
 
-	InstanceFederationMode:         InstanceFederationModeDefault,
+	InstanceFederationMode:            InstanceFederationModeDefault,
-	InstanceFederationSpamFilter:   false,
+	InstanceFederationSpamFilter:      false,
-	InstanceExposePeers:            false,
+	InstanceExposePeers:               false,
-	InstanceExposeSuspended:        false,
+	InstanceExposeSuspended:           false,
-	InstanceExposeSuspendedWeb:     false,
+	InstanceExposeSuspendedWeb:        false,
-	InstanceDeliverToSharedInboxes: true,
+	InstanceDeliverToSharedInboxes:    true,
-	InstanceLanguages:              make(language.Languages, 0),
+	InstanceLanguages:                 make(language.Languages, 0),
+	InstanceSubscriptionsProcessFrom:  "23:00",        // 11pm,
+	InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
 
 	AccountsRegistrationOpen: false,
 	AccountsReasonRequired:   true,

internal/config/flags.go

@@ -90,6 +90,8 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().Bool(InstanceExposeSuspendedWebFlag(), cfg.InstanceExposeSuspendedWeb, fieldtag("InstanceExposeSuspendedWeb", "usage"))
 		cmd.Flags().Bool(InstanceDeliverToSharedInboxesFlag(), cfg.InstanceDeliverToSharedInboxes, fieldtag("InstanceDeliverToSharedInboxes", "usage"))
 		cmd.Flags().StringSlice(InstanceLanguagesFlag(), cfg.InstanceLanguages.TagStrs(), fieldtag("InstanceLanguages", "usage"))
+		cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage"))
+		cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage"))
 
 		// Accounts
 		cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage"))

internal/config/helpers.gen.go

@@ -1000,6 +1000,62 @@ func GetInstanceLanguages() language.Languages { return global.GetInstanceLangua
 // SetInstanceLanguages safely sets the value for global configuration 'InstanceLanguages' field
 func SetInstanceLanguages(v language.Languages) { global.SetInstanceLanguages(v) }
 
+// GetInstanceSubscriptionsProcessFrom safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field
+func (st *ConfigState) GetInstanceSubscriptionsProcessFrom() (v string) {
+	st.mutex.RLock()
+	v = st.config.InstanceSubscriptionsProcessFrom
+	st.mutex.RUnlock()
+	return
+}
+
+// SetInstanceSubscriptionsProcessFrom safely sets the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field
+func (st *ConfigState) SetInstanceSubscriptionsProcessFrom(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.InstanceSubscriptionsProcessFrom = v
+	st.reloadToViper()
+}
+
+// InstanceSubscriptionsProcessFromFlag returns the flag name for the 'InstanceSubscriptionsProcessFrom' field
+func InstanceSubscriptionsProcessFromFlag() string { return "instance-subscriptions-process-from" }
+
+// GetInstanceSubscriptionsProcessFrom safely fetches the value for global configuration 'InstanceSubscriptionsProcessFrom' field
+func GetInstanceSubscriptionsProcessFrom() string {
+	return global.GetInstanceSubscriptionsProcessFrom()
+}
+
+// SetInstanceSubscriptionsProcessFrom safely sets the value for global configuration 'InstanceSubscriptionsProcessFrom' field
+func SetInstanceSubscriptionsProcessFrom(v string) { global.SetInstanceSubscriptionsProcessFrom(v) }
+
+// GetInstanceSubscriptionsProcessEvery safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field
+func (st *ConfigState) GetInstanceSubscriptionsProcessEvery() (v time.Duration) {
+	st.mutex.RLock()
+	v = st.config.InstanceSubscriptionsProcessEvery
+	st.mutex.RUnlock()
+	return
+}
+
+// SetInstanceSubscriptionsProcessEvery safely sets the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field
+func (st *ConfigState) SetInstanceSubscriptionsProcessEvery(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.InstanceSubscriptionsProcessEvery = v
+	st.reloadToViper()
+}
+
+// InstanceSubscriptionsProcessEveryFlag returns the flag name for the 'InstanceSubscriptionsProcessEvery' field
+func InstanceSubscriptionsProcessEveryFlag() string { return "instance-subscriptions-process-every" }
+
+// GetInstanceSubscriptionsProcessEvery safely fetches the value for global configuration 'InstanceSubscriptionsProcessEvery' field
+func GetInstanceSubscriptionsProcessEvery() time.Duration {
+	return global.GetInstanceSubscriptionsProcessEvery()
+}
+
+// SetInstanceSubscriptionsProcessEvery safely sets the value for global configuration 'InstanceSubscriptionsProcessEvery' field
+func SetInstanceSubscriptionsProcessEvery(v time.Duration) {
+	global.SetInstanceSubscriptionsProcessEvery(v)
+}
+
 // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field
 func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) {
 	st.mutex.RLock()

internal/federation/dereferencing/dereferencer_test.go

@@ -20,6 +20,7 @@
 import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@@ -77,6 +78,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
 	suite.client = testrig.NewMockHTTPClient(nil, "../../../testrig/media")
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.state.Storage = suite.storage
 
 	visFilter := visibility.NewFilter(&suite.state)

internal/federation/federatingdb/federatingdb_test.go

@@ -22,6 +22,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
@@ -91,6 +92,7 @@ func (suite *FederatingDBTestSuite) SetupTest() {
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 }
 
 func (suite *FederatingDBTestSuite) TearDownTest() {

internal/media/media_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -53,6 +54,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.state.Storage = suite.storage
 
 	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

internal/oauth/clientstore_test.go

@@ -22,6 +22,7 @@
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
@@ -54,6 +55,7 @@ func (suite *PgClientStoreTestSuite) SetupTest() {
 	testrig.InitTestConfig()
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	testrig.StandardDBSetup(suite.db, nil)
 }
 

internal/processing/account/account_test.go

@@ -22,6 +22,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -93,6 +94,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(

internal/processing/admin/account_test.go

@@ -53,7 +53,7 @@ func (suite *AccountTestSuite) TestAccountActionSuspend() {
 
 	// Wait for action to finish.
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return suite.state.Actions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}

internal/processing/admin/accountaction.go

@@ -68,7 +68,7 @@ func (p *Processor) accountActionSuspend(
 ) (string, gtserror.WithCode) {
 	actionID := id.NewULID()
 
-	errWithCode := p.actions.Run(
+	errWithCode := p.state.Actions.Run(
 		ctx,
 		&gtsmodel.AdminAction{
 			ID:             actionID,

internal/processing/admin/admin.go

@@ -21,10 +21,10 @@
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/common"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -33,21 +33,14 @@ type Processor struct {
 	// common processor logic
 	c *common.Processor
 
-	state     *state.State
+	state         *state.State
-	cleaner   *cleaner.Cleaner
+	cleaner       *cleaner.Cleaner
-	converter *typeutils.Converter
+	subscriptions *subscriptions.Subscriptions
-	federator *federation.Federator
+	converter     *typeutils.Converter
-	media     *media.Manager
+	federator     *federation.Federator
-	transport transport.Controller
+	media         *media.Manager
-	email     email.Sender
+	transport     transport.Controller
-
+	email         email.Sender
-	// admin Actions currently
-	// undergoing processing
-	actions *Actions
-}
-
-func (p *Processor) Actions() *Actions {
-	return p.actions
 }
 
 // New returns a new admin processor.
@@ -55,6 +48,7 @@ func New(
 	common *common.Processor,
 	state *state.State,
 	cleaner *cleaner.Cleaner,
+	subscriptions *subscriptions.Subscriptions,
 	federator *federation.Federator,
 	converter *typeutils.Converter,
 	mediaManager *media.Manager,
@@ -62,17 +56,14 @@ func New(
 	emailSender email.Sender,
 ) Processor {
 	return Processor{
-		c:         common,
+		c:             common,
-		state:     state,
+		state:         state,
-		cleaner:   cleaner,
+		cleaner:       cleaner,
-		converter: converter,
+		subscriptions: subscriptions,
-		federator: federator,
+		converter:     converter,
-		media:     mediaManager,
+		federator:     federator,
-		transport: transportController,
+		media:         mediaManager,
-		email:     emailSender,
+		transport:     transportController,
-		actions: &Actions{
+		email:         emailSender,
-			r:     make(map[string]*gtsmodel.AdminAction),
-			state: state,
-		},
 	}
 }

internal/processing/admin/admin_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -33,6 +34,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/testrig"
@@ -89,6 +91,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(
@@ -109,6 +112,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 
 	suite.processor = processing.NewProcessor(
 		cleaner.New(&suite.state),
+		subscriptions.New(&suite.state, suite.transportController, suite.tc),
 		suite.tc,
 		suite.federator,
 		suite.oauthServer,

internal/processing/admin/domainallow.go

@@ -22,14 +22,11 @@
 	"errors"
 	"fmt"
 
-	"codeberg.org/gruf/go-kv"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 )
 
@@ -69,84 +66,30 @@ func (p *Processor) createDomainAllow(
 		}
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of allow.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domainAllow.Domain,
+		Type:           gtsmodel.AdminActionUnsuspend,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process domain allow side
+	if errWithCode := p.state.Actions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.Actions.DomainAllowF(action.ID, domainAllow),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domain,
-			Type:           gtsmodel.AdminActionSuspend,
-			AccountID:      adminAcct.ID,
-			Text:           domainAllow.PrivateComment,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			l.Info("processing domain allow side effects")
-			defer func() { l.Info("finished processing domain allow side effects") }()
-
-			return p.domainAllowSideEffects(ctx, domainAllow)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
 	apiDomainAllow, errWithCode := p.apiDomainPerm(ctx, domainAllow, false)
 	if errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainAllow, actionID, nil
+	return apiDomainAllow, action.ID, nil
-}
-
-func (p *Processor) domainAllowSideEffects(
-	ctx context.Context,
-	allow *gtsmodel.DomainAllow,
-) gtserror.MultiError {
-	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
-		// We're running in allowlist mode,
-		// so there are no side effects to
-		// process here.
-		return nil
-	}
-
-	// We're running in blocklist mode or
-	// some similar mode which necessitates
-	// domain allow side effects if a block
-	// was in place when the allow was created.
-	//
-	// So, check if there's a block.
-	block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs := gtserror.NewMultiError(1)
-		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
-		return errs
-	}
-
-	if block == nil {
-		// No block?
-		// No problem!
-		return nil
-	}
-
-	// There was a block, over which the new
-	// allow ought to take precedence. To account
-	// for this, just run side effects as though
-	// the domain was being unblocked, while
-	// leaving the existing block in place.
-	//
-	// Any accounts that were suspended by
-	// the block will be unsuspended and be
-	// able to interact with the instance again.
-	return p.domainUnblockSideEffects(ctx, block)
 }
 
 func (p *Processor) deleteDomainAllow(
@@ -179,77 +122,23 @@ func (p *Processor) deleteDomainAllow(
 		return nil, "", gtserror.NewErrorInternalError(err)
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of unallow.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domainAllow.Domain,
+		Type:           gtsmodel.AdminActionUnsuspend,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process domain unallow side
+	if errWithCode := p.state.Actions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.Actions.DomainUnallowF(action.ID, domainAllow),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domainAllow.Domain,
-			Type:           gtsmodel.AdminActionUnsuspend,
-			AccountID:      adminAcct.ID,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domainAllow.Domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			l.Info("processing domain unallow side effects")
-			defer func() { l.Info("finished processing domain unallow side effects") }()
-
-			return p.domainUnallowSideEffects(ctx, domainAllow)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainAllow, actionID, nil
+	return apiDomainAllow, action.ID, nil
-}
-
-func (p *Processor) domainUnallowSideEffects(
-	ctx context.Context,
-	allow *gtsmodel.DomainAllow,
-) gtserror.MultiError {
-	if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
-		// We're running in allowlist mode,
-		// so there are no side effects to
-		// process here.
-		return nil
-	}
-
-	// We're running in blocklist mode or
-	// some similar mode which necessitates
-	// domain allow side effects if a block
-	// was in place when the allow was removed.
-	//
-	// So, check if there's a block.
-	block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs := gtserror.NewMultiError(1)
-		errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
-		return errs
-	}
-
-	if block == nil {
-		// No block?
-		// No problem!
-		return nil
-	}
-
-	// There was a block, over which the previous
-	// allow was taking precedence. Now that the
-	// allow has been removed, we should put the
-	// side effects of the block back in place.
-	//
-	// To do this, process the block side effects
-	// again as though the block were freshly
-	// created. This will mark all accounts from
-	// the blocked domain as suspended, and clean
-	// up their follows/following, media, etc.
-	return p.domainBlockSideEffects(ctx, block)
 }

internal/processing/admin/domainblock.go

@@ -21,18 +21,12 @@
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
-	"codeberg.org/gruf/go-kv"
-	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/log"
-	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 )
 
@@ -72,149 +66,31 @@ func (p *Processor) createDomainBlock(
 		}
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of block.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domain,
+		Type:           gtsmodel.AdminActionSuspend,
+		AccountID:      adminAcct.ID,
+		Text:           domainBlock.PrivateComment,
+	}
 
-	// Process domain block side
+	if errWithCode := p.state.Actions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.Actions.DomainBlockF(action.ID, domainBlock),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domain,
-			Type:           gtsmodel.AdminActionSuspend,
-			AccountID:      adminAcct.ID,
-			Text:           domainBlock.PrivateComment,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			skip, err := p.skipBlockSideEffects(ctx, domain)
-			if err != nil {
-				return err
-			}
-			if skip != "" {
-				l.Infof("skipping domain block side effects: %s", skip)
-				return nil
-			}
-
-			l.Info("processing domain block side effects")
-			defer func() { l.Info("finished processing domain block side effects") }()
-
-			return p.domainBlockSideEffects(ctx, domainBlock)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
 	apiDomainBlock, errWithCode := p.apiDomainPerm(ctx, domainBlock, false)
 	if errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainBlock, actionID, nil
+	return apiDomainBlock, action.ID, nil
-}
-
-// skipBlockSideEffects checks if side effects of block creation
-// should be skipped for the given domain, taking account of
-// instance federation mode, and existence of any allows
-// which ought to "shield" this domain from being blocked.
-//
-// If the caller should skip, the returned string will be non-zero
-// and will be set to a reason why side effects should be skipped.
-//
-//   - blocklist mode + allow exists: "..." (skip)
-//   - blocklist mode + no allow:     ""    (don't skip)
-//   - allowlist mode + allow exists: ""    (don't skip)
-//   - allowlist mode + no allow:     ""    (don't skip)
-func (p *Processor) skipBlockSideEffects(
-	ctx context.Context,
-	domain string,
-) (string, gtserror.MultiError) {
-	var (
-		skip string // Assume "" (don't skip).
-		errs gtserror.MultiError
-	)
-
-	// Never skip block side effects in allowlist mode.
-	fediMode := config.GetInstanceFederationMode()
-	if fediMode == config.InstanceFederationModeAllowlist {
-		return skip, errs
-	}
-
-	// We know we're in blocklist mode.
-	//
-	// We want to skip domain block side
-	// effects if an allow is already
-	// in place which overrides the block.
-
-	// Check if an explicit allow exists for this domain.
-	domainAllow, err := p.state.DB.GetDomainAllow(ctx, domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("error getting domain allow: %w", err)
-		return skip, errs
-	}
-
-	if domainAllow != nil {
-		skip = "running in blocklist mode, and an explicit allow exists for this domain"
-		return skip, errs
-	}
-
-	return skip, errs
-}
-
-// domainBlockSideEffects processes the side effects of a domain block:
-//
-//  1. Strip most info away from the instance entry for the domain.
-//  2. Pass each account from the domain to the processor for deletion.
-//
-// It should be called asynchronously, since it can take a while when
-// there are many accounts present on the given domain.
-func (p *Processor) domainBlockSideEffects(
-	ctx context.Context,
-	block *gtsmodel.DomainBlock,
-) gtserror.MultiError {
-	var errs gtserror.MultiError
-
-	// If we have an instance entry for this domain,
-	// update it with the new block ID and clear all fields
-	instance, err := p.state.DB.GetInstance(ctx, block.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
-		return errs
-	}
-
-	if instance != nil {
-		// We had an entry for this domain.
-		columns := stubbifyInstance(instance, block.ID)
-		if err := p.state.DB.UpdateInstance(ctx, instance, columns...); err != nil {
-			errs.Appendf("db error updating instance: %w", err)
-			return errs
-		}
-	}
-
-	// For each account that belongs to this domain,
-	// process an account delete message to remove
-	// that account's posts, media, etc.
-	if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
-		if err := p.state.Workers.Client.Process(ctx, &messages.FromClientAPI{
-			APObjectType:   ap.ActorPerson,
-			APActivityType: ap.ActivityDelete,
-			GTSModel:       block,
-			Origin:         account,
-			Target:         account,
-		}); err != nil {
-			errs.Append(err)
-		}
-	}); err != nil {
-		errs.Appendf("db error ranging through accounts: %w", err)
-	}
-
-	return errs
 }
 
 func (p *Processor) deleteDomainBlock(
@@ -247,104 +123,23 @@ func (p *Processor) deleteDomainBlock(
 		return nil, "", gtserror.NewErrorInternalError(err)
 	}
 
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of unblock.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domainBlock.Domain,
+		Type:           gtsmodel.AdminActionUnsuspend,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process domain unblock side
+	if errWithCode := p.state.Actions.Run(
-	// effects asynchronously.
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.Actions.DomainUnblockF(action.ID, domainBlock),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domainBlock.Domain,
-			Type:           gtsmodel.AdminActionUnsuspend,
-			AccountID:      adminAcct.ID,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			// Log start + finish.
-			l := log.WithFields(kv.Fields{
-				{"domain", domainBlock.Domain},
-				{"actionID", actionID},
-			}...).WithContext(ctx)
-
-			l.Info("processing domain unblock side effects")
-			defer func() { l.Info("finished processing domain unblock side effects") }()
-
-			return p.domainUnblockSideEffects(ctx, domainBlock)
-		},
 	); errWithCode != nil {
-		return nil, actionID, errWithCode
+		return nil, action.ID, errWithCode
 	}
 
-	return apiDomainBlock, actionID, nil
+	return apiDomainBlock, action.ID, nil
-}
-
-// domainUnblockSideEffects processes the side effects of undoing a
-// domain block:
-//
-//  1. Mark instance entry as no longer suspended.
-//  2. Mark each account from the domain as no longer suspended, if the
-//     suspension origin corresponds to the ID of the provided domain block.
-//
-// It should be called asynchronously, since it can take a while when
-// there are many accounts present on the given domain.
-func (p *Processor) domainUnblockSideEffects(
-	ctx context.Context,
-	block *gtsmodel.DomainBlock,
-) gtserror.MultiError {
-	var errs gtserror.MultiError
-
-	// Update instance entry for this domain, if we have it.
-	instance, err := p.state.DB.GetInstance(ctx, block.Domain)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("db error getting instance %s: %w", block.Domain, err)
-	}
-
-	if instance != nil {
-		// We had an entry, update it to signal
-		// that it's no longer suspended.
-		instance.SuspendedAt = time.Time{}
-		instance.DomainBlockID = ""
-		if err := p.state.DB.UpdateInstance(
-			ctx,
-			instance,
-			"suspended_at",
-			"domain_block_id",
-		); err != nil {
-			errs.Appendf("db error updating instance: %w", err)
-			return errs
-		}
-	}
-
-	// Unsuspend all accounts whose suspension origin was this domain block.
-	if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
-		if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() {
-			// Account wasn't suspended, nothing to do.
-			return
-		}
-
-		if account.SuspensionOrigin != block.ID {
-			// Account was suspended, but not by
-			// this domain block, leave it alone.
-			return
-		}
-
-		// Account was suspended by this domain
-		// block, mark it as unsuspended.
-		account.SuspendedAt = time.Time{}
-		account.SuspensionOrigin = ""
-
-		if err := p.state.DB.UpdateAccount(
-			ctx,
-			account,
-			"suspended_at",
-			"suspension_origin",
-		); err != nil {
-			errs.Appendf("db error updating account %s: %w", account.Username, err)
-		}
-	}); err != nil {
-		errs.Appendf("db error ranging through accounts: %w", err)
-	}
-
-	return errs
 }

internal/processing/admin/domainkeysexpire.go

@@ -19,7 +19,6 @@
 
 import (
 	"context"
-	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -39,47 +38,23 @@ func (p *Processor) DomainKeysExpire(
 	adminAcct *gtsmodel.Account,
 	domain string,
 ) (string, gtserror.WithCode) {
-	actionID := id.NewULID()
+	// Run admin action to process
+	// side effects of key expiry.
+	action := &gtsmodel.AdminAction{
+		ID:             id.NewULID(),
+		TargetCategory: gtsmodel.AdminActionCategoryDomain,
+		TargetID:       domain,
+		Type:           gtsmodel.AdminActionExpireKeys,
+		AccountID:      adminAcct.ID,
+	}
 
-	// Process key expiration asynchronously.
+	if errWithCode := p.state.Actions.Run(
-	if errWithCode := p.actions.Run(
 		ctx,
-		&gtsmodel.AdminAction{
+		action,
-			ID:             actionID,
+		p.state.Actions.DomainKeysExpireF(domain),
-			TargetCategory: gtsmodel.AdminActionCategoryDomain,
-			TargetID:       domain,
-			Type:           gtsmodel.AdminActionExpireKeys,
-			AccountID:      adminAcct.ID,
-		},
-		func(ctx context.Context) gtserror.MultiError {
-			return p.domainKeysExpireSideEffects(ctx, domain)
-		},
 	); errWithCode != nil {
-		return actionID, errWithCode
+		return action.ID, errWithCode
 	}
 
-	return actionID, nil
+	return action.ID, nil
-}
-
-func (p *Processor) domainKeysExpireSideEffects(ctx context.Context, domain string) gtserror.MultiError {
-	var (
-		expiresAt = time.Now()
-		errs      gtserror.MultiError
-	)
-
-	// For each account on this domain, expire
-	// the public key and update the account.
-	if err := p.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) {
-		account.PublicKeyExpiresAt = expiresAt
-		if err := p.state.DB.UpdateAccount(ctx,
-			account,
-			"public_key_expires_at",
-		); err != nil {
-			errs.Appendf("db error updating account: %w", err)
-		}
-	}); err != nil {
-		errs.Appendf("db error ranging through accounts: %w", err)
-	}
-
-	return errs
 }

internal/processing/admin/domainpermission_test.go

@@ -186,7 +186,7 @@ func (suite *DomainBlockTestSuite) awaitAction(actionID string) {
 	ctx := context.Background()
 
 	if !testrig.WaitFor(func() bool {
-		return suite.adminProcessor.Actions().TotalRunning() == 0
+		return suite.state.Actions.TotalRunning() == 0
 	}) {
 		suite.FailNow("timed out waiting for admin action(s) to finish")
 	}

internal/processing/admin/domainpermissionsubscription.go

@@ -22,6 +22,7 @@
 	"errors"
 	"fmt"
 	"net/url"
+	"slices"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
@@ -283,3 +284,90 @@ func (p *Processor) DomainPermissionSubscriptionRemove(
 
 	return p.apiDomainPermSub(ctx, permSub)
 }
+
+func (p *Processor) DomainPermissionSubscriptionTest(
+	ctx context.Context,
+	acct *gtsmodel.Account,
+	id string,
+) (any, gtserror.WithCode) {
+	permSub, err := p.state.DB.GetDomainPermissionSubscriptionByID(ctx, id)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err := gtserror.Newf("db error getting domain permission subscription %s: %w", id, err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	if permSub == nil {
+		err := fmt.Errorf("domain permission subscription %s not found", id)
+		return nil, gtserror.NewErrorNotFound(err, err.Error())
+	}
+
+	// To process the test/dry-run correctly, we need to get
+	// all domain perm subs of this type with a *higher* priority,
+	// to know whether we ought to create permissions or not.
+	permSubs, err := p.state.DB.GetDomainPermissionSubscriptionsByPriority(
+		ctx,
+		permSub.PermissionType,
+	)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err := gtserror.Newf("db error: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// Find the index of the targeted
+	// subscription in the slice.
+	index := slices.IndexFunc(
+		permSubs,
+		func(ps *gtsmodel.DomainPermissionSubscription) bool {
+			return ps.ID == permSub.ID
+		},
+	)
+
+	// Everything *before* the targeted subscription has a higher priority.
+	getHigherPrios := func() ([]*gtsmodel.DomainPermissionSubscription, error) {
+		return permSubs[:index], nil
+	}
+
+	// Get a transport for calling permSub.URI.
+	tsport, err := p.transport.NewTransportForUsername(ctx, acct.Username)
+	if err != nil {
+		err := gtserror.Newf("error getting transport: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// Call the permSub.URI and parse a list of perms from it.
+	// Any error returned here is a "real" one, not an error
+	// from fetching / parsing the list.
+	createdPerms, err := p.subscriptions.ProcessDomainPermissionSubscription(
+		ctx,
+		permSub,
+		tsport,
+		getHigherPrios,
+		true, // Dry run.
+	)
+	if err != nil {
+		err := gtserror.Newf("error doing dry-run: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	// If permSub has an error set on it now,
+	// we should return it to the caller.
+	if permSub.Error != "" {
+		return map[string]string{
+			"error": permSub.Error,
+		}, nil
+	}
+
+	// No error, so return the list of
+	// perms that would have been created.
+	apiPerms := make([]*apimodel.DomainPermission, 0, len(createdPerms))
+	for _, perm := range createdPerms {
+		apiPerm, errWithCode := p.apiDomainPerm(ctx, perm, false)
+		if errWithCode != nil {
+			return nil, errWithCode
+		}
+
+		apiPerms = append(apiPerms, apiPerm)
+	}
+
+	return apiPerms, nil
+}

internal/processing/admin/util.go

@@ -19,86 +19,12 @@
 
 import (
 	"context"
-	"errors"
-	"time"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-// stubbifyInstance renders the given instance as a stub,
-// removing most information from it and marking it as
-// suspended.
-//
-// For caller's convenience, this function returns the db
-// names of all columns that are updated by it.
-func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string {
-	instance.Title = ""
-	instance.SuspendedAt = time.Now()
-	instance.DomainBlockID = domainBlockID
-	instance.ShortDescription = ""
-	instance.Description = ""
-	instance.Terms = ""
-	instance.ContactEmail = ""
-	instance.ContactAccountUsername = ""
-	instance.ContactAccountID = ""
-	instance.Version = ""
-
-	return []string{
-		"title",
-		"suspended_at",
-		"domain_block_id",
-		"short_description",
-		"description",
-		"terms",
-		"contact_email",
-		"contact_account_username",
-		"contact_account_id",
-		"version",
-	}
-}
-
-// rangeDomainAccounts iterates through all accounts
-// originating from the given domain, and calls the
-// provided range function on each account.
-//
-// If an error is returned while selecting accounts,
-// the loop will stop and return the error.
-func (p *Processor) rangeDomainAccounts(
-	ctx context.Context,
-	domain string,
-	rangeF func(*gtsmodel.Account),
-) error {
-	var (
-		limit = 50   // Limit selection to avoid spiking mem/cpu.
-		maxID string // Start with empty string to select from top.
-	)
-
-	for {
-		// Get (next) page of accounts.
-		accounts, err := p.state.DB.GetInstanceAccounts(ctx, domain, maxID, limit)
-		if err != nil && !errors.Is(err, db.ErrNoEntries) {
-			// Real db error.
-			return gtserror.Newf("db error getting instance accounts: %w", err)
-		}
-
-		if len(accounts) == 0 {
-			// No accounts left, we're done.
-			return nil
-		}
-
-		// Set next max ID for paging down.
-		maxID = accounts[len(accounts)-1].ID
-
-		// Call provided range function.
-		for _, account := range accounts {
-			rangeF(account)
-		}
-	}
-}
-
 // apiDomainPerm is a cheeky shortcut for returning
 // the API version of the given domain permission,
 // or an appropriate error if something goes wrong.

internal/processing/conversations/conversations_test.go

@@ -23,6 +23,7 @@
 	"time"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	dbtest "github.com/superseriousbusiness/gotosocial/internal/db/test"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -103,6 +104,7 @@ func (suite *ConversationsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 	suite.filter = visibility.NewFilter(&suite.state)
 

internal/processing/media/media_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -75,6 +76,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.tc = typeutils.NewConverter(&suite.state)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage

internal/processing/processor.go

@@ -48,6 +48,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing/user"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/workers"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -180,6 +181,7 @@ func (p *Processor) Workers() *workers.Processor {
 // NewProcessor returns a new Processor.
 func NewProcessor(
 	cleaner *cleaner.Cleaner,
+	subscriptions *subscriptions.Subscriptions,
 	converter *typeutils.Converter,
 	federator *federation.Federator,
 	oauthServer oauth.Server,
@@ -210,7 +212,7 @@ func NewProcessor(
 	// Instantiate the rest of the sub
 	// processors + pin them to this struct.
 	processor.account = account.New(&common, state, converter, mediaManager, federator, visFilter, parseMentionFunc)
-	processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender)
+	processor.admin = admin.New(&common, state, cleaner, subscriptions, federator, converter, mediaManager, federator.TransportController(), emailSender)
 	processor.conversations = conversations.New(state, converter, visFilter)
 	processor.fedi = fedi.New(state, &common, converter, federator, visFilter)
 	processor.filtersv1 = filtersv1.New(state, converter, &processor.stream)

internal/processing/processor_test.go

@@ -21,6 +21,7 @@
 	"context"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -34,6 +35,7 @@
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/stream"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/testrig"
@@ -102,6 +104,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
@@ -125,6 +128,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
 
 	suite.processor = processing.NewProcessor(
 		cleaner.New(&suite.state),
+		subscriptions.New(&suite.state, suite.transportController, suite.typeconverter),
 		suite.typeconverter,
 		suite.federator,
 		suite.oauthServer,

internal/processing/status/status_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@@ -84,6 +85,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.typeConverter = typeutils.NewConverter(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 
 	suite.tc = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media"))
 	suite.storage = testrig.NewInMemoryStorage()

internal/processing/stream/stream_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
@@ -50,6 +51,7 @@ func (suite *StreamTestSuite) SetupTest() {
 	suite.testTokens = testrig.NewTestTokens()
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
 	suite.streamProcessor = stream.New(&suite.state, suite.oauthServer)
 

internal/processing/timeline/timeline_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -55,6 +56,7 @@ func (suite *TimelineStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 
 	suite.timeline = timeline.New(
 		&suite.state,

internal/processing/user/user_test.go

@@ -19,6 +19,7 @@
 
 import (
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -49,6 +50,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)

internal/state/state.go

@@ -19,6 +19,7 @@
 
 import (
 	"codeberg.org/gruf/go-mutexes"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/cache"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
@@ -61,9 +62,14 @@ type State struct {
 	// Storage provides access to the storage driver.
 	Storage *storage.Driver
 
-	// Workers provides access to this state's collection of worker pools.
+	// Workers provides access to this
+	// state's collection of worker pools.
 	Workers workers.Workers
 
+	// Struct to manage running admin
+	// actions (and locks thereupon).
+	Actions *actions.Actions
+
 	// prevent pass-by-value.
 	_ nocopy
 }

internal/subscriptions/domainperms.go

@@ -0,0 +1,804 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions
+
+import (
+	"context"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"codeberg.org/gruf/go-kv"
+
+	"github.com/miekg/dns"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// ScheduleJobs schedules domain permission subscription
+// fetching + updating using configured parameters.
+//
+// Returns an error if `MediaCleanupFrom`
+// is not a valid format (hh:mm:ss).
+func (s *Subscriptions) ScheduleJobs() error {
+	const hourMinute = "15:04"
+
+	var (
+		now            = time.Now()
+		processEvery   = config.GetInstanceSubscriptionsProcessEvery()
+		processFromStr = config.GetInstanceSubscriptionsProcessFrom()
+	)
+
+	// Parse processFromStr as hh:mm.
+	// Resulting time will be on 1 Jan year zero.
+	cleanupFrom, err := time.Parse(hourMinute, processFromStr)
+	if err != nil {
+		return gtserror.Newf(
+			"error parsing '%s' in time format 'hh:mm': %w",
+			processFromStr, err,
+		)
+	}
+
+	// Time travel from
+	// year zero, groovy.
+	firstProcessAt := time.Date(
+		now.Year(),
+		now.Month(),
+		now.Day(),
+		cleanupFrom.Hour(),
+		cleanupFrom.Minute(),
+		0,
+		0,
+		now.Location(),
+	)
+
+	// Ensure first processing is in the future.
+	for firstProcessAt.Before(now) {
+		firstProcessAt = firstProcessAt.Add(processEvery)
+	}
+
+	fn := func(ctx context.Context, start time.Time) {
+		log.Info(ctx, "starting instance subscriptions processing")
+
+		// In blocklist (default) mode, process allows
+		// first to provide immunity to block side effects.
+		//
+		// In allowlist mode, process blocks first to
+		// ensure allowlist doesn't override blocks.
+		var order [2]gtsmodel.DomainPermissionType
+		if config.GetInstanceFederationMode() == config.InstanceFederationModeBlocklist {
+			order = [2]gtsmodel.DomainPermissionType{
+				gtsmodel.DomainPermissionAllow,
+				gtsmodel.DomainPermissionBlock,
+			}
+		} else {
+			order = [2]gtsmodel.DomainPermissionType{
+				gtsmodel.DomainPermissionBlock,
+				gtsmodel.DomainPermissionAllow,
+			}
+		}
+
+		// Fetch + process subscribed perms in order.
+		for _, permType := range order {
+			s.ProcessDomainPermissionSubscriptions(ctx, permType)
+		}
+
+		log.Infof(ctx, "finished instance subscriptions processing after %s", time.Since(start))
+	}
+
+	log.Infof(nil,
+		"scheduling instance subscriptions processing to run every %s, starting from %s; next clean will run at %s",
+		processEvery, processFromStr, firstProcessAt,
+	)
+
+	// Schedule processing to execute according to schedule.
+	if !s.state.Workers.Scheduler.AddRecurring(
+		"@subsprocessing",
+		firstProcessAt,
+		processEvery,
+		fn,
+	) {
+		panic("failed to schedule @subsprocessing")
+	}
+
+	return nil
+}
+
+// ProcessDomainPermissionSubscriptions processes all domain permission
+// subscriptions of the given permission type by, in turn, calling the
+// URI of each subscription, parsing the result into a list of domain
+// permissions, and creating (or skipping) each permission as appropriate.
+func (s *Subscriptions) ProcessDomainPermissionSubscriptions(
+	ctx context.Context,
+	permType gtsmodel.DomainPermissionType,
+) {
+	log.Info(ctx, "start")
+
+	// Get permission subscriptions in priority order (highest -> lowest).
+	permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		// Real db error.
+		log.Error(ctx, err)
+		return
+	}
+
+	if len(permSubs) == 0 {
+		// No subscriptions of this
+		// type, so nothing to do.
+		return
+	}
+
+	// Get a transport using the instance account,
+	// we can reuse this for each HTTP call.
+	tsport, err := s.transportController.NewTransportForUsername(ctx, "")
+	if err != nil {
+		log.Error(ctx, err)
+		return
+	}
+
+	for i, permSub := range permSubs {
+		// Higher priority permission subs = everything
+		// above this permission sub in the slice.
+		getHigherPrios := func() ([]*gtsmodel.DomainPermissionSubscription, error) {
+			return permSubs[:i], nil
+		}
+
+		_, err := s.ProcessDomainPermissionSubscription(
+			ctx,
+			permSub,
+			tsport,
+			getHigherPrios,
+			false, // Not dry. Wet, if you will.
+		)
+		if err != nil {
+			// Real db error.
+			log.Error(ctx, err)
+			return
+		}
+
+		// Update this perm sub.
+		err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub)
+		if err != nil {
+			// Real db error.
+			log.Error(ctx, err)
+			return
+		}
+	}
+
+	log.Info(ctx, "finished")
+}
+
+// ProcessDomainPermissionSubscription processes one domain permission
+// subscription by dereferencing the URI, parsing the response into a list
+// of permissions, and for each discovered permission either creating an
+// entry in the database, or ignoring it if it's excluded or already
+// covered by a higher-priority subscription.
+//
+// On success, the slice of discovered DomainPermissions will be returned.
+// In case of parsing error, or error on the remote side, permSub.Error
+// will be updated with the calling/parsing error, and `nil, nil` will be
+// returned. In case of an actual db error, `nil, err` will be returned and
+// the caller should handle it.
+//
+// getHigherPrios should be a function for returning a slice of domain
+// permission subscriptions with a higher priority than the given permSub.
+//
+// If dry == true, then the URI will still be called, and permissions
+// will be parsed, but they will not actually be created.
+//
+// Note that while this function modifies fields on the given permSub,
+// it's up to the caller to update it in the database (if desired).
+func (s *Subscriptions) ProcessDomainPermissionSubscription(
+	ctx context.Context,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	tsport transport.Transport,
+	getHigherPrios func() ([]*gtsmodel.DomainPermissionSubscription, error),
+	dry bool,
+) ([]gtsmodel.DomainPermission, error) {
+	l := log.
+		WithContext(ctx).
+		WithFields(kv.Fields{
+			{"permType", permSub.PermissionType.String()},
+			{"permSubURI", permSub.URI},
+		}...)
+
+	// Set FetchedAt as we're
+	// going to attempt this now.
+	permSub.FetchedAt = time.Now()
+
+	// Call the URI, only force
+	// if we're doing a dry run.
+	resp, err := tsport.DereferenceDomainPermissions(
+		ctx, permSub, !dry,
+	)
+	if err != nil {
+		// Couldn't get this one,
+		// set error + return.
+		errStr := err.Error()
+		l.Warnf("couldn't dereference permSubURI: %+v", err)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	// If the permissions at URI weren't modified
+	// since last time, just update some metadata
+	// to indicate a successful fetch, and return.
+	if resp.Unmodified {
+		l.Debug("received 304 Not Modified from remote")
+		permSub.SuccessfullyFetchedAt = permSub.FetchedAt
+		if permSub.ETag == "" && resp.ETag != "" {
+			// We didn't have an ETag before but
+			// we have one now: probably the remote
+			// added ETag support in the meantime.
+			permSub.ETag = resp.ETag
+		}
+		return nil, nil
+	}
+
+	// At this point we know we got a 200 OK
+	// from the URI, so we've got a live body!
+	// Try to parse the body as a list of wantedPerms
+	// that the subscription wants to create.
+	var wantedPerms []gtsmodel.DomainPermission
+
+	switch permSub.ContentType {
+
+	// text/csv
+	case gtsmodel.DomainPermSubContentTypeCSV:
+		wantedPerms, err = s.permsFromCSV(l, permSub.PermissionType, resp.Body)
+
+	// application/json
+	case gtsmodel.DomainPermSubContentTypeJSON:
+		wantedPerms, err = s.permsFromJSON(l, permSub.PermissionType, resp.Body)
+
+	// text/plain
+	case gtsmodel.DomainPermSubContentTypePlain:
+		wantedPerms, err = s.permsFromPlain(l, permSub.PermissionType, resp.Body)
+	}
+
+	if err != nil {
+		// We retrieved the permissions from remote but
+		// the connection died halfway through transfer,
+		// or we couldn't parse the results, or something.
+		// Just set error and return.
+		errStr := err.Error()
+		l.Warnf("couldn't parse results: %+v", err)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	if len(wantedPerms) == 0 {
+		// Fetch was OK, and parsing was, on the surface at
+		// least, OK, but we didn't get any perms. Consider
+		// this an error as users will probably want to know.
+		const errStr = "fetch successful but parsed zero usable results"
+		l.Warn(errStr)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	// This can now be considered a successful fetch.
+	permSub.SuccessfullyFetchedAt = permSub.FetchedAt
+	permSub.ETag = resp.ETag
+	permSub.Error = ""
+
+	// Need a list of higher priority subscriptions
+	// to ensure we don't create permissions wrongly.
+	higherPrios, err := getHigherPrios()
+	if err != nil {
+		// Proper db error.
+		return nil, err
+	}
+
+	// Keep track of which domain perms are
+	// created (or would be, if dry == true).
+	createdPerms := make([]gtsmodel.DomainPermission, 0, len(wantedPerms))
+
+	// Iterate through wantedPerms and
+	// create (or dry create) each one.
+	for _, wantedPerm := range wantedPerms {
+		l = l.WithField("domain", wantedPerm.GetDomain())
+		created, err := s.processDomainPermission(
+			ctx, l,
+			wantedPerm,
+			permSub,
+			higherPrios,
+			dry,
+		)
+		if err != nil {
+			// Proper db error.
+			return nil, err
+		}
+
+		if !created {
+			continue
+		}
+
+		createdPerms = append(createdPerms, wantedPerm)
+	}
+
+	return createdPerms, nil
+}
+
+// processDomainPermission processes one wanted domain
+// permission discovered via a domain permission sub's URI.
+//
+// Error will only be returned in case of an actual database
+// error, else the error will be logged and nil returned.
+func (s *Subscriptions) processDomainPermission(
+	ctx context.Context,
+	l log.Entry,
+	wantedPerm gtsmodel.DomainPermission,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+	dry bool,
+) (bool, error) {
+	// Set to true if domain permission
+	// actually (would be) created.
+	var created bool
+
+	// If domain is excluded from automatic
+	// permission creation, don't process it.
+	domain := wantedPerm.GetDomain()
+	excluded, err := s.state.DB.IsDomainPermissionExcluded(ctx, domain)
+	if err != nil {
+		// Proper db error.
+		return created, err
+	}
+
+	if excluded {
+		l.Debug("domain is excluded, skipping")
+		return created, nil
+	}
+
+	// Check if a permission already exists for
+	// this domain, and if it's covered already
+	// by a higher-priority subscription.
+	existingPerm, covered, err := s.existingCovered(
+		ctx, permSub.PermissionType, domain, higherPrios,
+	)
+	if err != nil {
+		// Proper db error.
+		return created, err
+	}
+
+	if covered {
+		l.Debug("domain is covered by a higher-priority subscription, skipping")
+		return created, nil
+	}
+
+	// At this point we know we
+	// should create the perm.
+	created = true
+
+	if dry {
+		// Don't do creation or side
+		// effects if we're dry running.
+		return created, nil
+	}
+
+	// Handle perm creation differently depending
+	// on whether or not a perm already existed.
+	existing := !util.IsNil(existingPerm)
+	switch {
+
+	case !existing && *permSub.AsDraft:
+		// No existing perm, create as draft.
+		err = s.state.DB.PutDomainPermissionDraft(
+			ctx,
+			&gtsmodel.DomainPermissionDraft{
+				ID:                 id.NewULID(),
+				PermissionType:     permSub.PermissionType,
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			},
+		)
+
+	case !existing && !*permSub.AsDraft:
+		// No existing perm, create a new one of the
+		// appropriate type, and process side effects.
+		var (
+			insertF func() error
+			action  *gtsmodel.AdminAction
+			actionF actions.AdminActionF
+		)
+
+		if permSub.PermissionType == gtsmodel.DomainPermissionBlock {
+			// Prepare to insert + process a block.
+			domainBlock := &gtsmodel.DomainBlock{
+				ID:                 id.NewULID(),
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			}
+			insertF = func() error { return s.state.DB.CreateDomainBlock(ctx, domainBlock) }
+
+			action = &gtsmodel.AdminAction{
+				ID:             id.NewULID(),
+				TargetCategory: gtsmodel.AdminActionCategoryDomain,
+				TargetID:       domain,
+				Type:           gtsmodel.AdminActionSuspend,
+				AccountID:      permSub.CreatedByAccountID,
+			}
+			actionF = s.state.Actions.DomainBlockF(action.ID, domainBlock)
+
+		} else {
+			// Prepare to insert + process an allow.
+			domainAllow := &gtsmodel.DomainAllow{
+				ID:                 id.NewULID(),
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			}
+			insertF = func() error { return s.state.DB.CreateDomainAllow(ctx, domainAllow) }
+
+			action = &gtsmodel.AdminAction{
+				ID:             id.NewULID(),
+				TargetCategory: gtsmodel.AdminActionCategoryDomain,
+				TargetID:       domain,
+				Type:           gtsmodel.AdminActionUnsuspend,
+				AccountID:      permSub.CreatedByAccountID,
+			}
+			actionF = s.state.Actions.DomainAllowF(action.ID, domainAllow)
+		}
+
+		// Insert the new perm in the db.
+		if err = insertF(); err != nil {
+			// Couldn't insert wanted perm,
+			// don't process side effects.
+			break
+		}
+
+		// Run admin action to process
+		// side effects of permission.
+		err = s.state.Actions.Run(ctx, action, actionF)
+
+	case existingPerm.GetSubscriptionID() != "" || *permSub.AdoptOrphans:
+		// Perm exists but we should adopt/take
+		// it by copying over desired fields.
+		existingPerm.SetCreatedByAccountID(wantedPerm.GetCreatedByAccountID())
+		existingPerm.SetCreatedByAccount(wantedPerm.GetCreatedByAccount())
+		existingPerm.SetSubscriptionID(permSub.ID)
+		existingPerm.SetObfuscate(wantedPerm.GetObfuscate())
+		existingPerm.SetPrivateComment(wantedPerm.GetPrivateComment())
+		existingPerm.SetPublicComment(wantedPerm.GetPublicComment())
+
+		switch p := existingPerm.(type) {
+		case *gtsmodel.DomainBlock:
+			err = s.state.DB.UpdateDomainBlock(ctx, p)
+		case *gtsmodel.DomainAllow:
+			err = s.state.DB.UpdateDomainAllow(ctx, p)
+		}
+
+	default:
+		// Perm exists but we should leave it alone.
+		l.Debug("domain is covered by a higher-priority subscription, skipping")
+	}
+
+	if err != nil && !errors.Is(err, db.ErrAlreadyExists) {
+		// Proper db error.
+		return created, err
+	}
+
+	created = true
+	return created, nil
+}
+
+func (s *Subscriptions) permsFromCSV(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	// Read body into memory as slice of CSV records.
+	records, err := csv.NewReader(body).ReadAll()
+
+	// Whatever happened, we're
+	// done with the body now.
+	body.Close()
+
+	// Check if error reading body.
+	if err != nil {
+		return nil, gtserror.NewfAt(3, "error decoding into csv: %w", err)
+	}
+
+	// Make sure we actually
+	// have some records.
+	if len(records) == 0 {
+		return nil, nil
+	}
+
+	// Validate column headers.
+	columnHeaders := records[0]
+	if !slices.Equal(
+		columnHeaders,
+		[]string{
+			"#domain",
+			"#severity",
+			"#reject_media",
+			"#reject_reports",
+			"#public_comment",
+			"#obfuscate",
+		},
+	) {
+		return nil, gtserror.Newf(
+			"unexpected column headers in csv: %+v",
+			columnHeaders,
+		)
+	}
+
+	// Trim off column headers
+	// now they're validated.
+	records = records[1:]
+
+	// Convert records to permissions slice.
+	perms := make([]gtsmodel.DomainPermission, 0, len(records))
+	for _, record := range records {
+		if len(record) != 6 {
+			l.Warnf("skipping invalid-length record: %+v", record)
+			continue
+		}
+
+		var (
+			domainRaw      = record[0]
+			severity       = record[1]
+			publicComment  = record[4]
+			obfuscate, err = strconv.ParseBool(record[5])
+		)
+
+		if severity != "suspend" {
+			l.Warnf("skipping non-suspend record: %+v", record)
+			continue
+		}
+
+		if err != nil {
+			l.Warnf("couldn't parse obfuscate field of record: %+v", record)
+			continue
+		}
+
+		// Normalize + validate domain.
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		if permType == gtsmodel.DomainPermissionBlock {
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		} else {
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// Set remaining fields.
+		perm.SetPublicComment(publicComment)
+		perm.SetObfuscate(&obfuscate)
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func (s *Subscriptions) permsFromJSON(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	var (
+		dec      = json.NewDecoder(body)
+		apiPerms = make([]*apimodel.DomainPermission, 0)
+	)
+
+	// Read body into memory as
+	// slice of domain permissions.
+	if err := dec.Decode(&apiPerms); err != nil {
+		_ = body.Close() // ensure closed.
+		return nil, gtserror.NewfAt(3, "error decoding into json: %w", err)
+	}
+
+	// Perform a secondary decode just to ensure we drained the
+	// entirety of the data source. Error indicates either extra
+	// trailing garbage, or multiple JSON values (invalid data).
+	if err := dec.Decode(&struct{}{}); err != io.EOF {
+		_ = body.Close() // ensure closed.
+		return nil, gtserror.NewfAt(3, "data remaining after json")
+	}
+
+	// Done with body.
+	_ = body.Close()
+
+	// Convert apimodel perms to barebones internal perms.
+	perms := make([]gtsmodel.DomainPermission, 0, len(apiPerms))
+	for _, apiPerm := range apiPerms {
+
+		// Normalize + validate domain.
+		domainRaw := apiPerm.Domain.Domain
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		if permType == gtsmodel.DomainPermissionBlock {
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		} else {
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// Set remaining fields.
+		perm.SetPublicComment(apiPerm.PublicComment)
+		perm.SetObfuscate(&apiPerm.Obfuscate)
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func (s *Subscriptions) permsFromPlain(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	// Read body into memory as bytes.
+	b, err := io.ReadAll(body)
+
+	// Whatever happened, we're
+	// done with the body now.
+	body.Close()
+
+	// Check if error reading body.
+	if err != nil {
+		return nil, gtserror.NewfAt(3, "error decoding into plain: %w", err)
+	}
+
+	// Coerce to newline-separated list of domains.
+	domains := strings.Split(string(b), "\n")
+
+	// Convert raw domains to permissions.
+	perms := make([]gtsmodel.DomainPermission, 0, len(domains))
+	for _, domainRaw := range domains {
+
+		// Normalize + validate domain.
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		if permType == gtsmodel.DomainPermissionBlock {
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		} else {
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func validateDomain(domain string) (string, error) {
+	// Basic validation.
+	if _, ok := dns.IsDomainName(domain); !ok {
+		err := fmt.Errorf("invalid domain name")
+		return "", err
+	}
+
+	// Convert to punycode.
+	domain, err := util.Punify(domain)
+	if err != nil {
+		err := fmt.Errorf("could not punify domain: %w", err)
+		return "", err
+	}
+
+	// Check for invalid characters
+	// after the punification process.
+	if strings.ContainsAny(domain, "*, \n") {
+		err := fmt.Errorf("invalid char(s) in domain")
+		return "", err
+	}
+
+	return domain, nil
+}
+
+func (s *Subscriptions) existingCovered(
+	ctx context.Context,
+	permType gtsmodel.DomainPermissionType,
+	domain string,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+) (
+	existingPerm gtsmodel.DomainPermission,
+	covered bool,
+	err error,
+) {
+	// Check for existing permission of appropriate type.
+	var dbErr error
+	if permType == gtsmodel.DomainPermissionBlock {
+		existingPerm, dbErr = s.state.DB.GetDomainBlock(ctx, domain)
+	} else {
+		existingPerm, dbErr = s.state.DB.GetDomainAllow(ctx, domain)
+	}
+	if dbErr != nil && !errors.Is(dbErr, db.ErrNoEntries) {
+		// Real db error.
+		err = dbErr
+		return
+	}
+
+	if util.IsNil(existingPerm) {
+		// Can't be covered if
+		// no existing perm.
+		return
+	}
+
+	subscriptionID := existingPerm.GetSubscriptionID()
+	if subscriptionID == "" {
+		// Can't be covered if
+		// no subscription ID.
+		return
+	}
+
+	// Covered if subscription ID is in the slice
+	// of higher-priority permission subscriptions.
+	covered = slices.ContainsFunc(
+		higherPrios,
+		func(permSub *gtsmodel.DomainPermissionSubscription) bool {
+			return permSub.ID == subscriptionID
+		},
+	)
+
+	return
+}

internal/subscriptions/subscriptions.go

@@ -0,0 +1,42 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+type Subscriptions struct {
+	state               *state.State
+	transportController transport.Controller
+	tc                  *typeutils.Converter
+}
+
+func New(
+	state *state.State,
+	transportController transport.Controller,
+	tc *typeutils.Converter,
+) *Subscriptions {
+	return &Subscriptions{
+		state:               state,
+		transportController: transportController,
+		tc:                  tc,
+	}
+}

internal/subscriptions/subscriptions_test.go

@@ -0,0 +1,496 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const (
+	rMediaPath    = "../../testrig/media"
+	rTemplatePath = "../../web/template"
+)
+
+type SubscriptionsTestSuite struct {
+	suite.Suite
+
+	testAccounts map[string]*gtsmodel.Account
+}
+
+func (suite *SubscriptionsTestSuite) SetupSuite() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+	suite.testAccounts = testrig.NewTestAccounts()
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksCSV() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("bigbums6969", permSub.ETag)
+	suite.EqualValues(3, permSub.Count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksJSON() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a JSON list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.json",
+			ContentType:        gtsmodel.DomainPermSubContentTypeJSON,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("don't modify me daddy", permSub.ETag)
+	suite.EqualValues(3, permSub.Count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksPlain() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plain list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.txt",
+			ContentType:        gtsmodel.DomainPermSubContentTypePlain,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("this is a legit etag i swear", permSub.ETag)
+	suite.EqualValues(3, permSub.Count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksCSVETag() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		// Include the ETag so it gets sent with the request.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+			ETag:               "bigbums6969",
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now NOT have blocks for the domains
+	// on the list, as the remote will have returned
+	// 304, indicating we should do nothing.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		_, err := testStructs.State.DB.GetDomainBlock(ctx, domain)
+		if !errors.Is(err, db.ErrNoEntries) {
+			suite.FailNowf("", "domain perm %s created when it shouldn't be")
+		}
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("bigbums6969", permSub.ETag)
+	suite.EqualValues(0, permSub.Count) // Should be 0.
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocks404() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		// The endpoint will return a 404 so we can test erroring.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/does_not_exist.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.EqualValues(0, permSub.Count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`DereferenceDomainPermissions: GET request to https://lists.example.org/does_not_exist.csv failed: status="" body="{"error":"not found"}"`, permSub.Error)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypeCSV() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plaintext list of baddies,
+		// but try to parse as CSV content type (shouldn't work).
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.txt",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(permSub.Count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`permsFromCSV: unexpected column headers in csv: [bumfaces.net]`, permSub.Error)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypePlain() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plaintext list of baddies,
+		// but try to parse as CSV content type (shouldn't work).
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypePlain,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(permSub.Count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`fetch successful but parsed zero usable results`, permSub.Error)
+}
+
+func TestSubscriptionTestSuite(t *testing.T) {
+	suite.Run(t, new(SubscriptionsTestSuite))
+}

internal/transport/derefdomainpermlist.go

@@ -0,0 +1,121 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package transport
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+type DereferenceDomainPermissionsResp struct {
+	// Set only if response was 200 OK.
+	// It's up to the caller to close
+	// this when they're done with it.
+	Body io.ReadCloser
+
+	// True if response
+	// was 304 Not Modified.
+	Unmodified bool
+
+	// May be set
+	// if 200 or 304.
+	ETag string
+}
+
+func (t *transport) DereferenceDomainPermissions(
+	ctx context.Context,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	force bool,
+) (*DereferenceDomainPermissionsResp, error) {
+	// Prepare new HTTP request to endpoint
+	req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set basic auth header if necessary.
+	if permSub.FetchUsername != "" || permSub.FetchPassword != "" {
+		req.SetBasicAuth(permSub.FetchUsername, permSub.FetchPassword)
+	}
+
+	// Set relevant Accept headers.
+	// Allow fallback in case target doesn't
+	// negotiate content type correctly.
+	req.Header.Add("Accept-Charset", "utf-8")
+	req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*")
+
+	// If force is true, we want to skip setting Cache
+	// headers so that we definitely don't get a 304 back.
+	if !force {
+		// If we've successfully fetched this list
+		// before, set If-Modified-Since to last
+		// success to make the request conditional.
+		//
+		// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since
+		if !permSub.SuccessfullyFetchedAt.IsZero() {
+			timeStr := permSub.SuccessfullyFetchedAt.Format(http.TimeFormat)
+			req.Header.Add("If-Modified-Since", timeStr)
+		}
+
+		// If we've got an ETag stored for this list, set
+		// If-None-Match to make the request conditional.
+		// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#caching_of_unchanged_resources.
+		if len(permSub.ETag) != 0 {
+			req.Header.Add("If-None-Match", permSub.ETag)
+		}
+	}
+
+	// Perform the HTTP request
+	rsp, err := t.GET(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// If we have an unexpected / error response,
+	// wrap + return as error. This will also drain
+	// and close the response body for us.
+	if rsp.StatusCode != http.StatusOK &&
+		rsp.StatusCode != http.StatusNotModified {
+		err := gtserror.NewFromResponse(rsp)
+		return nil, err
+	}
+
+	// Check already if we were given an ETag
+	// we can use, as ETag is often returned
+	// even on 304 Not Modified responses.
+	permsResp := &DereferenceDomainPermissionsResp{
+		ETag: rsp.Header.Get("Etag"),
+	}
+
+	if rsp.StatusCode == http.StatusNotModified {
+		// Nothing has changed on the remote side
+		// since we last fetched, so there's nothing
+		// to do and we don't need to read the body.
+		rsp.Body.Close()
+		permsResp.Unmodified = true
+	} else {
+		// Return the live body to the caller.
+		permsResp.Body = rsp.Body
+	}
+
+	return permsResp, nil
+}

internal/transport/transport.go

@@ -78,6 +78,20 @@ type Transport interface {
 	// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
 	DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
 
+	// DereferenceDomainPermissions dereferences the
+	// permissions list present at the given permSub's URI.
+	//
+	// If "force", then If-Modified-Since and If-None-Match
+	// headers will *NOT* be sent with the outgoing request.
+	//
+	// If err == nil and Unmodified == false, then it's up
+	// to the caller to close the returned io.ReadCloser.
+	DereferenceDomainPermissions(
+		ctx context.Context,
+		permSub *gtsmodel.DomainPermissionSubscription,
+		force bool,
+	) (*DereferenceDomainPermissionsResp, error)
+
 	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
 	Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error)
 }

internal/transport/transport_test.go

@@ -21,6 +21,7 @@
 	"context"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -74,6 +75,7 @@ func (suite *TransportTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/typeutils/converter_test.go

@@ -20,6 +20,7 @@
 import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -499,6 +500,7 @@ func (suite *TypeUtilsTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
 	storage := testrig.NewInMemoryStorage()
 	suite.state.Storage = storage
 

internal/typeutils/internaltofrontend.go

@@ -2121,7 +2121,9 @@ func (c *Converter) DomainPermToAPIDomainPerm(
 	domainPerm.PrivateComment = d.GetPrivateComment()
 	domainPerm.SubscriptionID = d.GetSubscriptionID()
 	domainPerm.CreatedBy = d.GetCreatedByAccountID()
-	domainPerm.CreatedAt = util.FormatISO8601(d.GetCreatedAt())
+	if createdAt := d.GetCreatedAt(); !createdAt.IsZero() {
+		domainPerm.CreatedAt = util.FormatISO8601(createdAt)
+	}
 
 	// If this is a draft, also add the permission type.
 	if _, ok := d.(*gtsmodel.DomainPermissionDraft); ok {

testrig/config.go

@@ -99,6 +99,8 @@ func testDefaults() config.Configuration {
 				TagStr: "en-gb",
 			},
 		},
+		InstanceSubscriptionsProcessFrom:  "23:00",        // 11pm,
+		InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
 
 		AccountsRegistrationOpen: true,
 		AccountsReasonRequired:   true,

testrig/processor.go

@@ -26,15 +26,27 @@
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
 // NewTestProcessor returns a Processor suitable for testing purposes.
 // The passed in state will have its worker functions set appropriately,
 // but the state will not be initialized.
-func NewTestProcessor(state *state.State, federator *federation.Federator, emailSender email.Sender, mediaManager *media.Manager) *processing.Processor {
+func NewTestProcessor(
+	state *state.State,
+	federator *federation.Federator,
+	emailSender email.Sender,
+	mediaManager *media.Manager,
+) *processing.Processor {
+
 	return processing.NewProcessor(
 		cleaner.New(state),
+		subscriptions.New(
+			state,
+			federator.TransportController(),
+			typeutils.NewConverter(state),
+		),
 		typeutils.NewConverter(state),
 		federator,
 		NewTestOauthServer(state.DB),

testrig/teststructs.go

@@ -18,6 +18,7 @@
 package testrig
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/actions"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@@ -25,6 +26,8 @@
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/common"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -38,12 +41,13 @@
 // and worker queues, which was causing issues
 // when running all tests at once.
 type TestStructs struct {
-	State         *state.State
+	State               *state.State
-	Common        *common.Processor
+	Common              *common.Processor
-	Processor     *processing.Processor
+	Processor           *processing.Processor
-	HTTPClient    *MockHTTPClient
+	HTTPClient          *MockHTTPClient
-	TypeConverter *typeutils.Converter
+	TypeConverter       *typeutils.Converter
-	EmailSender   email.Sender
+	EmailSender         email.Sender
+	TransportController transport.Controller
 }
 
 func SetupTestStructs(
@@ -56,6 +60,7 @@ func SetupTestStructs(
 
 	db := NewTestDB(&state)
 	state.DB = db
+	state.Actions = actions.New(db, &state.Workers)
 
 	storage := NewInMemoryStorage()
 	state.Storage = storage
@@ -89,6 +94,7 @@ func SetupTestStructs(
 
 	processor := processing.NewProcessor(
 		cleaner.New(&state),
+		subscriptions.New(&state, transportController, typeconverter),
 		typeconverter,
 		federator,
 		oauthServer,
@@ -105,12 +111,13 @@ func SetupTestStructs(
 	StandardStorageSetup(storage, rMediaPath)
 
 	return &TestStructs{
-		State:         &state,
+		State:               &state,
-		Common:        &common,
+		Common:              &common,
-		Processor:     processor,
+		Processor:           processor,
-		HTTPClient:    httpClient,
+		HTTPClient:          httpClient,
-		TypeConverter: typeconverter,
+		TypeConverter:       typeconverter,
-		EmailSender:   emailSender,
+		EmailSender:         emailSender,
+		TransportController: transportController,
 	}
 }
 

testrig/transportcontroller.go

@@ -41,6 +41,8 @@
 const (
 	applicationJSON         = "application/json"
 	applicationActivityJSON = "application/activity+json"
+	textCSV                 = "text/csv"
+	textPlain               = "text/plain"
 )
 
 // NewTestTransportController returns a test transport controller with the given http client.
@@ -101,6 +103,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseBytes         = []byte(`{"error":"404 not found"}`)
 			responseContentType   = applicationJSON
 			responseContentLength = len(responseBytes)
+			extraHeaders          = make(map[string]string, 0)
 			reqURLString          = req.URL.String()
 		)
 
@@ -124,11 +127,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 			responseContentType = applicationJSON
 			responseContentLength = len(responseBytes)
 		} else if strings.Contains(reqURLString, ".well-known/webfinger") {
-			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
 		} else if strings.Contains(reqURLString, ".weird-webfinger-location/webfinger") {
-			responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
 		} else if strings.Contains(reqURLString, ".well-known/host-meta") {
-			responseCode, responseBytes, responseContentType, responseContentLength = HostMetaResponse(req)
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = HostMetaResponse(req)
+		} else if strings.Contains(reqURLString, "lists.example.org") {
+			responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = DomainPermissionSubscriptionResponse(req)
 		} else if note, ok := mockHTTPClient.TestRemoteStatuses[reqURLString]; ok {
 			// the request is for a note that we have stored
 			noteI, err := streams.Serialize(note)
@@ -239,14 +244,23 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
 		}
 
 		log.Debugf(nil, "returning response %s", string(responseBytes))
+
 		reader := bytes.NewReader(responseBytes)
 		readCloser := io.NopCloser(reader)
+
+		header := http.Header{
+			"Content-Type": {responseContentType},
+		}
+		for k, v := range extraHeaders {
+			header.Add(k, v)
+		}
+
 		return &http.Response{
 			Request:       req,
 			StatusCode:    responseCode,
 			Body:          readCloser,
 			ContentLength: int64(responseContentLength),
-			Header:        http.Header{"Content-Type": {responseContentType}},
+			Header:        header,
 		}, nil
 	}
 
@@ -261,7 +275,13 @@ func (m *MockHTTPClient) DoSigned(req *http.Request, sign httpclient.SignFunc) (
 	return m.do(req)
 }
 
-func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
+func HostMetaResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
 	var hm *apimodel.HostMeta
 
 	if req.URL.String() == "https://misconfigured-instance.com/.well-known/host-meta" {
@@ -297,7 +317,13 @@ func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte
 	return
 }
 
-func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
+func WebfingerResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
 	var wfr *apimodel.WellKnownResponse
 
 	switch req.URL.String() {
@@ -410,3 +436,89 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt
 	responseContentLength = len(wfrJSON)
 	return
 }
+
+func DomainPermissionSubscriptionResponse(req *http.Request) (
+	responseCode int,
+	responseBytes []byte,
+	responseContentType string,
+	responseContentLength int,
+	extraHeaders map[string]string,
+) {
+
+	const (
+		csvResp = `#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate
+bumfaces.net,suspend,false,false,big jerks,false
+peepee.poopoo,suspend,false,false,harassment,false
+nothanks.com,suspend,false,false,,false`
+		csvRespETag = "bigbums6969"
+
+		textResp = `bumfaces.net
+peepee.poopoo
+nothanks.com`
+		textRespETag = "this is a legit etag i swear"
+
+		jsonResp = `[
+  {
+    "domain": "bumfaces.net",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "big jerks"
+  },
+    {
+    "domain": "peepee.poopoo",
+    "suspended_at": "2020-05-13T13:29:12.000Z",
+    "public_comment": "harassment"
+  },
+    {
+    "domain": "nothanks.com",
+    "suspended_at": "2020-05-13T13:29:12.000Z"
+  }
+]`
+		jsonRespETag = "don't modify me daddy"
+	)
+
+	switch req.URL.String() {
+	case "https://lists.example.org/baddies.csv":
+		extraHeaders = map[string]string{"ETag": csvRespETag}
+		if req.Header.Get("If-None-Match") == csvRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(csvResp)
+			responseContentType = textCSV
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	case "https://lists.example.org/baddies.txt":
+		extraHeaders = map[string]string{"ETag": textRespETag}
+		if req.Header.Get("If-None-Match") == textRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(textResp)
+			responseContentType = textPlain
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	case "https://lists.example.org/baddies.json":
+		extraHeaders = map[string]string{"ETag": jsonRespETag}
+		if req.Header.Get("If-None-Match") == jsonRespETag {
+			// Cached.
+			responseCode = http.StatusNotModified
+		} else {
+			responseBytes = []byte(jsonResp)
+			responseContentType = applicationJSON
+			responseCode = http.StatusOK
+		}
+		responseContentLength = len(responseBytes)
+
+	default:
+		responseCode = http.StatusNotFound
+		responseBytes = []byte(`{"error":"not found"}`)
+		responseContentType = applicationJSON
+		responseContentLength = len(responseBytes)
+	}
+
+	return
+}