diff --git a/cmd/gotosocial/action/admin/account/account.go b/cmd/gotosocial/action/admin/account/account.go index 57d0d3805..7dfb6b1d4 100644 --- a/cmd/gotosocial/action/admin/account/account.go +++ b/cmd/gotosocial/action/admin/account/account.go @@ -40,7 +40,8 @@ func initState(ctx context.Context) (*state.State, error) { state.Caches.Init() state.Caches.Start() - // Set the state DB connection + // Only set state DB connection. + // Don't need Actions or Workers for this (yet). dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { return nil, fmt.Errorf("error creating dbConn: %w", err) diff --git a/cmd/gotosocial/action/admin/media/list.go b/cmd/gotosocial/action/admin/media/list.go index 547954d4c..a017539ed 100644 --- a/cmd/gotosocial/action/admin/media/list.go +++ b/cmd/gotosocial/action/admin/media/list.go @@ -127,6 +127,8 @@ func setupList(ctx context.Context) (*list, error) { state.Caches.Init() state.Caches.Start() + // Only set state DB connection. + // Don't need Actions or Workers for this. dbService, err := bundb.NewBunDBService(ctx, &state) if err != nil { return nil, fmt.Errorf("error creating dbservice: %w", err) diff --git a/cmd/gotosocial/action/admin/media/prune/common.go b/cmd/gotosocial/action/admin/media/prune/common.go index 5b42a6687..d73676f5b 100644 --- a/cmd/gotosocial/action/admin/media/prune/common.go +++ b/cmd/gotosocial/action/admin/media/prune/common.go @@ -45,10 +45,12 @@ func setupPrune(ctx context.Context) (*prune, error) { state.Caches.Start() // Scheduler is required for the - // claner, but no other workers + // cleaner, but no other workers // are needed for this CLI action. state.Workers.StartScheduler() + // Set state DB connection. + // Don't need Actions for this. dbService, err := bundb.NewBunDBService(ctx, &state) if err != nil { return nil, fmt.Errorf("error creating dbservice: %w", err) diff --git a/cmd/gotosocial/action/admin/trans/export.go b/cmd/gotosocial/action/admin/trans/export.go index f76982a1b..dae2db7db 100644 --- a/cmd/gotosocial/action/admin/trans/export.go +++ b/cmd/gotosocial/action/admin/trans/export.go @@ -33,12 +33,12 @@ var Export action.GTSAction = func(ctx context.Context) error { var state state.State + // Only set state DB connection. + // Don't need Actions or Workers for this. dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { return fmt.Errorf("error creating dbservice: %s", err) } - - // Set the state DB connection state.DB = dbConn exporter := trans.NewExporter(dbConn) diff --git a/cmd/gotosocial/action/admin/trans/import.go b/cmd/gotosocial/action/admin/trans/import.go index 1ebf587ff..d34c816bb 100644 --- a/cmd/gotosocial/action/admin/trans/import.go +++ b/cmd/gotosocial/action/admin/trans/import.go @@ -33,12 +33,12 @@ var Import action.GTSAction = func(ctx context.Context) error { var state state.State + // Only set state DB connection. + // Don't need Actions or Workers for this. dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { return fmt.Errorf("error creating dbservice: %s", err) } - - // Set the state DB connection state.DB = dbConn importer := trans.NewImporter(dbConn) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 376ade13d..d5ac50934 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -32,6 +32,7 @@ "github.com/KimMachineGun/automemlimit/memlimit" "github.com/gin-gonic/gin" "github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api" apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" "github.com/superseriousbusiness/gotosocial/internal/cleaner" @@ -44,6 +45,7 @@ "github.com/superseriousbusiness/gotosocial/internal/metrics" "github.com/superseriousbusiness/gotosocial/internal/middleware" tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/tracing" "go.uber.org/automaxprocs/maxprocs" @@ -164,6 +166,10 @@ // Set DB on state. state.DB = dbService + // Set Actions on state, providing workers to + // Actions as well for triggering side effects. + state.Actions = actions.New(dbService, &state.Workers) + // Ensure necessary database instance prerequisites exist. if err := dbService.CreateInstanceAccount(ctx); err != nil { return fmt.Errorf("error creating instance account: %s", err) @@ -283,15 +289,18 @@ func(context.Context, time.Time) { // Create background cleaner. cleaner := cleaner.New(state) - // Now schedule background cleaning tasks. - if err := cleaner.ScheduleJobs(); err != nil { - return fmt.Errorf("error scheduling cleaner jobs: %w", err) - } + // Create subscriptions fetcher. + subscriptions := subscriptions.New( + state, + transportController, + typeConverter, + ) // Create the processor using all the // other services we've created so far. process = processing.NewProcessor( cleaner, + subscriptions, typeConverter, federator, oauthServer, @@ -302,6 +311,16 @@ func(context.Context, time.Time) { intFilter, ) + // Schedule background cleaning tasks. + if err := cleaner.ScheduleJobs(); err != nil { + return fmt.Errorf("error scheduling cleaner jobs: %w", err) + } + + // Schedule background subscriptions updating. + if err := subscriptions.ScheduleJobs(); err != nil { + return fmt.Errorf("error scheduling subscriptions jobs: %w", err) + } + // Initialize the specialized workers pools. state.Workers.Client.Init(messages.ClientMsgIndices()) state.Workers.Federator.Init(messages.FederatorMsgIndices()) diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go index 19588c70a..f6721d8b6 100644 --- a/cmd/gotosocial/action/testrig/testrig.go +++ b/cmd/gotosocial/action/testrig/testrig.go @@ -47,6 +47,7 @@ "github.com/superseriousbusiness/gotosocial/internal/router" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/tracing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" @@ -314,11 +315,23 @@ // Create background cleaner. cleaner := cleaner.New(state) - // Now schedule background cleaning tasks. + // Schedule background cleaning tasks. if err := cleaner.ScheduleJobs(); err != nil { return fmt.Errorf("error scheduling cleaner jobs: %w", err) } + // Create subscriptions fetcher. + subscriptions := subscriptions.New( + state, + transportController, + typeConverter, + ) + + // Schedule background subscriptions updating. + if err := subscriptions.ScheduleJobs(); err != nil { + return fmt.Errorf("error scheduling subscriptions jobs: %w", err) + } + // Finally start the main http server! if err := route.Start(); err != nil { return fmt.Errorf("error starting router: %w", err) diff --git a/internal/processing/admin/actions.go b/internal/actions/actions.go similarity index 74% rename from internal/processing/admin/actions.go rename to internal/actions/actions.go index 968e45baa..b872a1ffd 100644 --- a/internal/processing/admin/actions.go +++ b/internal/actions/actions.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package admin +package actions import ( "context" @@ -23,11 +23,12 @@ "sync" "time" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/workers" ) func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode { @@ -42,15 +43,34 @@ func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode { } type Actions struct { - r map[string]*gtsmodel.AdminAction - state *state.State + // Map of running actions. + running map[string]*gtsmodel.AdminAction - // Not embedded struct, - // to shield from access - // by outside packages. + // Lock for running admin actions. + // + // Not embedded struct, to shield + // from access by outside packages. m sync.Mutex + + // DB for storing, updating, + // deleting admin actions etc. + db db.DB + + // Workers for queuing + // admin action side effects. + workers *workers.Workers } +func New(db db.DB, workers *workers.Workers) *Actions { + return &Actions{ + running: make(map[string]*gtsmodel.AdminAction), + db: db, + workers: workers, + } +} + +type AdminActionF func(context.Context) gtserror.MultiError + // Run runs the given admin action by executing the supplied function. // // Run handles locking, action insertion and updating, so you don't have to! @@ -62,10 +82,10 @@ type Actions struct { // will be updated on the provided admin action in the database. func (a *Actions) Run( ctx context.Context, - action *gtsmodel.AdminAction, - f func(context.Context) gtserror.MultiError, + adminAction *gtsmodel.AdminAction, + f AdminActionF, ) gtserror.WithCode { - actionKey := action.Key() + actionKey := adminAction.Key() // LOCK THE MAP HERE, since we're // going to do some operations on it. @@ -73,7 +93,7 @@ func (a *Actions) Run( // Bail if an action with // this key is already running. - running, ok := a.r[actionKey] + running, ok := a.running[actionKey] if ok { a.m.Unlock() return errActionConflict(running) @@ -81,7 +101,7 @@ func (a *Actions) Run( // Action with this key not // yet running, create it. - if err := a.state.DB.PutAdminAction(ctx, action); err != nil { + if err := a.db.PutAdminAction(ctx, adminAction); err != nil { err = gtserror.Newf("db error putting admin action %s: %w", actionKey, err) // Don't store in map @@ -92,7 +112,7 @@ func (a *Actions) Run( // Action was inserted, // store in map. - a.r[actionKey] = action + a.running[actionKey] = adminAction // UNLOCK THE MAP HERE, since // we're done modifying it for now. @@ -104,22 +124,22 @@ func (a *Actions) Run( // Run the thing and collect errors. if errs := f(ctx); errs != nil { - action.Errors = make([]string, 0, len(errs)) + adminAction.Errors = make([]string, 0, len(errs)) for _, err := range errs { - action.Errors = append(action.Errors, err.Error()) + adminAction.Errors = append(adminAction.Errors, err.Error()) } } // Action is no longer running: // remove from running map. a.m.Lock() - delete(a.r, actionKey) + delete(a.running, actionKey) a.m.Unlock() // Mark as completed in the db, // storing errors for later review. - action.CompletedAt = time.Now() - if err := a.state.DB.UpdateAdminAction(ctx, action, "completed_at", "errors"); err != nil { + adminAction.CompletedAt = time.Now() + if err := a.db.UpdateAdminAction(ctx, adminAction, "completed_at", "errors"); err != nil { log.Errorf(ctx, "db error marking action %s as completed: %q", actionKey, err) } }() @@ -135,8 +155,8 @@ func (a *Actions) GetRunning() []*gtsmodel.AdminAction { defer a.m.Unlock() // Assemble all currently running actions. - running := make([]*gtsmodel.AdminAction, 0, len(a.r)) - for _, action := range a.r { + running := make([]*gtsmodel.AdminAction, 0, len(a.running)) + for _, action := range a.running { running = append(running, action) } @@ -166,5 +186,5 @@ func (a *Actions) TotalRunning() int { a.m.Lock() defer a.m.Unlock() - return len(a.r) + return len(a.running) } diff --git a/internal/processing/admin/actions_test.go b/internal/actions/actions_test.go similarity index 80% rename from internal/processing/admin/actions_test.go rename to internal/actions/actions_test.go index 9d12ae84d..37ca06d01 100644 --- a/internal/processing/admin/actions_test.go +++ b/internal/actions/actions_test.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package admin_test +package actions_test import ( "context" @@ -32,12 +32,26 @@ "github.com/superseriousbusiness/gotosocial/testrig" ) +const ( + rMediaPath = "../../testrig/media" + rTemplatePath = "../../web/template" +) + type ActionsTestSuite struct { - AdminStandardTestSuite + suite.Suite +} + +func (suite *ActionsTestSuite) SetupSuite() { + testrig.InitTestConfig() + testrig.InitTestLog() } func (suite *ActionsTestSuite) TestActionOverlap() { - ctx := context.Background() + var ( + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + ctx = context.Background() + ) + defer testrig.TearDownTestStructs(testStructs) // Suspend account. action1 := >smodel.AdminAction{ @@ -61,7 +75,7 @@ func (suite *ActionsTestSuite) TestActionOverlap() { key2 := action2.Key() suite.Equal("account/01H90S1CXQ97J9625C5YBXZWGT", key2) - errWithCode := suite.adminProcessor.Actions().Run( + errWithCode := testStructs.State.Actions.Run( ctx, action1, func(ctx context.Context) gtserror.MultiError { @@ -74,7 +88,7 @@ func(ctx context.Context) gtserror.MultiError { // While first action is sleeping, try to // process another with the same key. - errWithCode = suite.adminProcessor.Actions().Run( + errWithCode = testStructs.State.Actions.Run( ctx, action2, func(ctx context.Context) gtserror.MultiError { @@ -90,13 +104,13 @@ func(ctx context.Context) gtserror.MultiError { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return testStructs.State.Actions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } // Try again. - errWithCode = suite.adminProcessor.Actions().Run( + errWithCode = testStructs.State.Actions.Run( ctx, action2, func(ctx context.Context) gtserror.MultiError { @@ -107,14 +121,18 @@ func(ctx context.Context) gtserror.MultiError { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return testStructs.State.Actions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } } func (suite *ActionsTestSuite) TestActionWithErrors() { - ctx := context.Background() + var ( + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + ctx = context.Background() + ) + defer testrig.TearDownTestStructs(testStructs) // Suspend a domain. action := >smodel.AdminAction{ @@ -125,7 +143,7 @@ func (suite *ActionsTestSuite) TestActionWithErrors() { AccountID: "01H90S1ZZXP4N74H4A9RVW1MRP", } - errWithCode := suite.adminProcessor.Actions().Run( + errWithCode := testStructs.State.Actions.Run( ctx, action, func(ctx context.Context) gtserror.MultiError { @@ -140,13 +158,13 @@ func(ctx context.Context) gtserror.MultiError { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return testStructs.State.Actions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } // Get action from the db. - dbAction, err := suite.db.GetAdminAction(ctx, action.ID) + dbAction, err := testStructs.State.DB.GetAdminAction(ctx, action.ID) if err != nil { suite.FailNow(err.Error()) } diff --git a/internal/actions/domainkeys.go b/internal/actions/domainkeys.go new file mode 100644 index 000000000..9f803cb93 --- /dev/null +++ b/internal/actions/domainkeys.go @@ -0,0 +1,51 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package actions + +import ( + "context" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (a *Actions) DomainKeysExpireF(domain string) AdminActionF { + return func(ctx context.Context) gtserror.MultiError { + var ( + expiresAt = time.Now() + errs gtserror.MultiError + ) + + // For each account on this domain, expire + // the public key and update the account. + if err := a.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) { + account.PublicKeyExpiresAt = expiresAt + if err := a.db.UpdateAccount(ctx, + account, + "public_key_expires_at", + ); err != nil { + errs.Appendf("db error updating account: %w", err) + } + }); err != nil { + errs.Appendf("db error ranging through accounts: %w", err) + } + + return errs + } +} diff --git a/internal/actions/domainperms.go b/internal/actions/domainperms.go new file mode 100644 index 000000000..44321dd90 --- /dev/null +++ b/internal/actions/domainperms.go @@ -0,0 +1,387 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package actions + +import ( + "context" + "errors" + "time" + + "codeberg.org/gruf/go-kv" + + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/messages" +) + +// Returns an AdminActionF for +// domain allow side effects. +func (a *Actions) DomainAllowF( + actionID string, + domainAllow *gtsmodel.DomainAllow, +) AdminActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "allow"}, + {"actionID", actionID}, + {"domain", domainAllow.Domain}, + }...) + + // Log start + finish. + l.Info("processing side effects") + errs := a.domainAllowSideEffects(ctx, domainAllow) + l.Info("finished processing side effects") + + return errs + } +} + +func (a *Actions) domainAllowSideEffects( + ctx context.Context, + allow *gtsmodel.DomainAllow, +) gtserror.MultiError { + if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { + // We're running in allowlist mode, + // so there are no side effects to + // process here. + return nil + } + + // We're running in blocklist mode or + // some similar mode which necessitates + // domain allow side effects if a block + // was in place when the allow was created. + // + // So, check if there's a block. + block, err := a.db.GetDomainBlock(ctx, allow.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs := gtserror.NewMultiError(1) + errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) + return errs + } + + if block == nil { + // No block? + // No problem! + return nil + } + + // There was a block, over which the new + // allow ought to take precedence. To account + // for this, just run side effects as though + // the domain was being unblocked, while + // leaving the existing block in place. + // + // Any accounts that were suspended by + // the block will be unsuspended and be + // able to interact with the instance again. + return a.domainUnblockSideEffects(ctx, block) +} + +// Returns an AdminActionF for +// domain unallow side effects. +func (a *Actions) DomainUnallowF( + actionID string, + domainAllow *gtsmodel.DomainAllow, +) AdminActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "unallow"}, + {"actionID", actionID}, + {"domain", domainAllow.Domain}, + }...) + + // Log start + finish. + l.Info("processing side effects") + errs := a.domainUnallowSideEffects(ctx, domainAllow) + l.Info("finished processing side effects") + + return errs + } +} + +func (a *Actions) domainUnallowSideEffects( + ctx context.Context, + allow *gtsmodel.DomainAllow, +) gtserror.MultiError { + if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { + // We're running in allowlist mode, + // so there are no side effects to + // process here. + return nil + } + + // We're running in blocklist mode or + // some similar mode which necessitates + // domain allow side effects if a block + // was in place when the allow was removed. + // + // So, check if there's a block. + block, err := a.db.GetDomainBlock(ctx, allow.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs := gtserror.NewMultiError(1) + errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) + return errs + } + + if block == nil { + // No block? + // No problem! + return nil + } + + // There was a block, over which the previous + // allow was taking precedence. Now that the + // allow has been removed, we should put the + // side effects of the block back in place. + // + // To do this, process the block side effects + // again as though the block were freshly + // created. This will mark all accounts from + // the blocked domain as suspended, and clean + // up their follows/following, media, etc. + return a.domainBlockSideEffects(ctx, block) +} + +func (a *Actions) DomainBlockF( + actionID string, + domainBlock *gtsmodel.DomainBlock, +) AdminActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "block"}, + {"actionID", actionID}, + {"domain", domainBlock.Domain}, + }...) + + skip, err := a.skipBlockSideEffects(ctx, domainBlock.Domain) + if err != nil { + return err + } + + if skip != "" { + l.Infof("skipping side effects: %s", skip) + return nil + } + + l.Info("processing side effects") + errs := a.domainBlockSideEffects(ctx, domainBlock) + l.Info("finished processing side effects") + + return errs + } +} + +// domainBlockSideEffects processes the side effects of a domain block: +// +// 1. Strip most info away from the instance entry for the domain. +// 2. Pass each account from the domain to the processor for deletion. +// +// It should be called asynchronously, since it can take a while when +// there are many accounts present on the given domain. +func (a *Actions) domainBlockSideEffects( + ctx context.Context, + block *gtsmodel.DomainBlock, +) gtserror.MultiError { + var errs gtserror.MultiError + + // If we have an instance entry for this domain, + // update it with the new block ID and clear all fields + instance, err := a.db.GetInstance(ctx, block.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs.Appendf("db error getting instance %s: %w", block.Domain, err) + return errs + } + + if instance != nil { + // We had an entry for this domain. + columns := stubbifyInstance(instance, block.ID) + if err := a.db.UpdateInstance(ctx, instance, columns...); err != nil { + errs.Appendf("db error updating instance: %w", err) + return errs + } + } + + // For each account that belongs to this domain, + // process an account delete message to remove + // that account's posts, media, etc. + if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { + if err := a.workers.Client.Process(ctx, &messages.FromClientAPI{ + APObjectType: ap.ActorPerson, + APActivityType: ap.ActivityDelete, + GTSModel: block, + Origin: account, + Target: account, + }); err != nil { + errs.Append(err) + } + }); err != nil { + errs.Appendf("db error ranging through accounts: %w", err) + } + + return errs +} + +func (a *Actions) DomainUnblockF( + actionID string, + domainBlock *gtsmodel.DomainBlock, +) AdminActionF { + return func(ctx context.Context) gtserror.MultiError { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"action", "unblock"}, + {"actionID", actionID}, + {"domain", domainBlock.Domain}, + }...) + + l.Info("processing side effects") + errs := a.domainUnblockSideEffects(ctx, domainBlock) + l.Info("finished processing side effects") + + return errs + } +} + +// domainUnblockSideEffects processes the side effects of undoing a +// domain block: +// +// 1. Mark instance entry as no longer suspended. +// 2. Mark each account from the domain as no longer suspended, if the +// suspension origin corresponds to the ID of the provided domain block. +// +// It should be called asynchronously, since it can take a while when +// there are many accounts present on the given domain. +func (a *Actions) domainUnblockSideEffects( + ctx context.Context, + block *gtsmodel.DomainBlock, +) gtserror.MultiError { + var errs gtserror.MultiError + + // Update instance entry for this domain, if we have it. + instance, err := a.db.GetInstance(ctx, block.Domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs.Appendf("db error getting instance %s: %w", block.Domain, err) + } + + if instance != nil { + // We had an entry, update it to signal + // that it's no longer suspended. + instance.SuspendedAt = time.Time{} + instance.DomainBlockID = "" + if err := a.db.UpdateInstance( + ctx, + instance, + "suspended_at", + "domain_block_id", + ); err != nil { + errs.Appendf("db error updating instance: %w", err) + return errs + } + } + + // Unsuspend all accounts whose suspension origin was this domain block. + if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { + if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() { + // Account wasn't suspended, nothing to do. + return + } + + if account.SuspensionOrigin != block.ID { + // Account was suspended, but not by + // this domain block, leave it alone. + return + } + + // Account was suspended by this domain + // block, mark it as unsuspended. + account.SuspendedAt = time.Time{} + account.SuspensionOrigin = "" + + if err := a.db.UpdateAccount( + ctx, + account, + "suspended_at", + "suspension_origin", + ); err != nil { + errs.Appendf("db error updating account %s: %w", account.Username, err) + } + }); err != nil { + errs.Appendf("db error ranging through accounts: %w", err) + } + + return errs +} + +// skipBlockSideEffects checks if side effects of block creation +// should be skipped for the given domain, taking account of +// instance federation mode, and existence of any allows +// which ought to "shield" this domain from being blocked. +// +// If the caller should skip, the returned string will be non-zero +// and will be set to a reason why side effects should be skipped. +// +// - blocklist mode + allow exists: "..." (skip) +// - blocklist mode + no allow: "" (don't skip) +// - allowlist mode + allow exists: "" (don't skip) +// - allowlist mode + no allow: "" (don't skip) +func (a *Actions) skipBlockSideEffects( + ctx context.Context, + domain string, +) (string, gtserror.MultiError) { + var ( + skip string // Assume "" (don't skip). + errs gtserror.MultiError + ) + + // Never skip block side effects in allowlist mode. + fediMode := config.GetInstanceFederationMode() + if fediMode == config.InstanceFederationModeAllowlist { + return skip, errs + } + + // We know we're in blocklist mode. + // + // We want to skip domain block side + // effects if an allow is already + // in place which overrides the block. + + // Check if an explicit allow exists for this domain. + domainAllow, err := a.db.GetDomainAllow(ctx, domain) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + errs.Appendf("error getting domain allow: %w", err) + return skip, errs + } + + if domainAllow != nil { + skip = "running in blocklist mode, and an explicit allow exists for this domain" + return skip, errs + } + + return skip, errs +} diff --git a/internal/actions/util.go b/internal/actions/util.go new file mode 100644 index 000000000..9c64adb94 --- /dev/null +++ b/internal/actions/util.go @@ -0,0 +1,99 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package actions + +import ( + "context" + "errors" + "time" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// stubbifyInstance renders the given instance as a stub, +// removing most information from it and marking it as +// suspended. +// +// For caller's convenience, this function returns the db +// names of all columns that are updated by it. +func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string { + instance.Title = "" + instance.SuspendedAt = time.Now() + instance.DomainBlockID = domainBlockID + instance.ShortDescription = "" + instance.Description = "" + instance.Terms = "" + instance.ContactEmail = "" + instance.ContactAccountUsername = "" + instance.ContactAccountID = "" + instance.Version = "" + + return []string{ + "title", + "suspended_at", + "domain_block_id", + "short_description", + "description", + "terms", + "contact_email", + "contact_account_username", + "contact_account_id", + "version", + } +} + +// rangeDomainAccounts iterates through all accounts +// originating from the given domain, and calls the +// provided range function on each account. +// +// If an error is returned while selecting accounts, +// the loop will stop and return the error. +func (a *Actions) rangeDomainAccounts( + ctx context.Context, + domain string, + rangeF func(*gtsmodel.Account), +) error { + var ( + limit = 50 // Limit selection to avoid spiking mem/cpu. + maxID string // Start with empty string to select from top. + ) + + for { + // Get (next) page of accounts. + accounts, err := a.db.GetInstanceAccounts(ctx, domain, maxID, limit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real db error. + return gtserror.Newf("db error getting instance accounts: %w", err) + } + + if len(accounts) == 0 { + // No accounts left, we're done. + return nil + } + + // Set next max ID for paging down. + maxID = accounts[len(accounts)-1].ID + + // Call provided range function. + for _, account := range accounts { + rangeF(account) + } + } +} diff --git a/internal/api/activitypub/emoji/emojiget_test.go b/internal/api/activitypub/emoji/emojiget_test.go index 4d687a049..f3e742cb5 100644 --- a/internal/api/activitypub/emoji/emojiget_test.go +++ b/internal/api/activitypub/emoji/emojiget_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/activitypub/emoji" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -73,6 +74,7 @@ func (suite *EmojiGetTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage suite.tc = typeutils.NewConverter(&suite.state) diff --git a/internal/api/activitypub/users/user_test.go b/internal/api/activitypub/users/user_test.go index 4d55aad3d..e7d3d774b 100644 --- a/internal/api/activitypub/users/user_test.go +++ b/internal/api/activitypub/users/user_test.go @@ -20,6 +20,7 @@ import ( "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -84,6 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( diff --git a/internal/api/auth/auth_test.go b/internal/api/auth/auth_test.go index d77b1a3d4..13c1ae5e1 100644 --- a/internal/api/auth/auth_test.go +++ b/internal/api/auth/auth_test.go @@ -26,6 +26,7 @@ "github.com/gin-contrib/sessions/memstore" "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/auth" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -84,6 +85,7 @@ func (suite *AuthStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage suite.mediaManager = testrig.NewTestMediaManager(&suite.state) diff --git a/internal/api/client/accounts/account_test.go b/internal/api/client/accounts/account_test.go index 2f8664756..b0abf402c 100644 --- a/internal/api/client/accounts/account_test.go +++ b/internal/api/client/accounts/account_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/accounts" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -85,6 +86,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/admin/admin.go b/internal/api/client/admin/admin.go index 68a088b4d..a5a16f35f 100644 --- a/internal/api/client/admin/admin.go +++ b/internal/api/client/admin/admin.go @@ -46,6 +46,7 @@ DomainPermissionSubscriptionsPathWithID = DomainPermissionSubscriptionsPath + "/:" + apiutil.IDKey DomainPermissionSubscriptionsPreviewPath = DomainPermissionSubscriptionsPath + "/preview" DomainPermissionSubscriptionRemovePath = DomainPermissionSubscriptionsPathWithID + "/remove" + DomainPermissionSubscriptionTestPath = DomainPermissionSubscriptionsPathWithID + "/test" DomainKeysExpirePath = BasePath + "/domain_keys_expire" HeaderAllowsPath = BasePath + "/header_allows" HeaderAllowsPathWithID = HeaderAllowsPath + "/:" + apiutil.IDKey @@ -129,6 +130,7 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H attachHandler(http.MethodGet, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionGETHandler) attachHandler(http.MethodPatch, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionPATCHHandler) attachHandler(http.MethodPost, DomainPermissionSubscriptionRemovePath, m.DomainPermissionSubscriptionRemovePOSTHandler) + attachHandler(http.MethodPost, DomainPermissionSubscriptionTestPath, m.DomainPermissionSubscriptionTestPOSTHandler) // header filtering administration routes attachHandler(http.MethodGet, HeaderAllowsPathWithID, m.HeaderFilterAllowGET) diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go index 962ec3872..c7e7260cd 100644 --- a/internal/api/client/admin/admin_test.go +++ b/internal/api/client/admin/admin_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -91,6 +92,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/admin/domainpermissionsubscriptiontest.go b/internal/api/client/admin/domainpermissionsubscriptiontest.go new file mode 100644 index 000000000..395a1a69c --- /dev/null +++ b/internal/api/client/admin/domainpermissionsubscriptiontest.go @@ -0,0 +1,118 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// DomainPermissionSubscriptionTestPOSTHandler swagger:operation POST /api/v1/admin/domain_permission_subscriptions/{id}/test domainPermissionSubscriptionTest +// +// Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*. +// +// The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message. +// +// This is useful in cases where you want to check that your instance can actually fetch + parse a list. +// +// --- +// tags: +// - admin +// +// produces: +// - application/json +// +// parameters: +// - +// name: id +// required: true +// in: path +// description: ID of the domain permission draft. +// type: string +// +// security: +// - OAuth2 Bearer: +// - admin +// +// responses: +// '200': +// description: >- +// Either an array of domain permissions, OR an error message of the form +// `{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched. +// schema: +// type: array +// items: +// "$ref": "#/definitions/domain" +// '400': +// description: bad request +// '401': +// description: unauthorized +// '403': +// description: forbidden +// '406': +// description: not acceptable +// '409': +// description: conflict +// '500': +// description: internal server error +func (m *Module) DomainPermissionSubscriptionTestPOSTHandler(c *gin.Context) { + authed, err := oauth.Authed(c, true, true, true, true) + if err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1) + return + } + + if !*authed.User.Admin { + err := fmt.Errorf("user %s not an admin", authed.User.ID) + apiutil.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGetV1) + return + } + + if authed.Account.IsMoving() { + apiutil.ForbiddenAfterMove(c) + return + } + + if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil { + apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1) + return + } + + id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey)) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + resp, errWithCode := m.processor.Admin().DomainPermissionSubscriptionTest( + c.Request.Context(), + authed.Account, + id, + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + apiutil.JSON(c, http.StatusOK, resp) +} diff --git a/internal/api/client/admin/domainpermissionsubscruptiontest_test.go b/internal/api/client/admin/domainpermissionsubscruptiontest_test.go new file mode 100644 index 000000000..46861aba1 --- /dev/null +++ b/internal/api/client/admin/domainpermissionsubscruptiontest_test.go @@ -0,0 +1,125 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/client/admin" + apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +type DomainPermissionSubscriptionTestTestSuite struct { + AdminStandardTestSuite +} + +func (suite *DomainPermissionSubscriptionTestTestSuite) TestDomainPermissionSubscriptionTest() { + var ( + ctx = context.Background() + testAccount = suite.testAccounts["admin_account"] + permSub = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + + // Create a subscription for a CSV list of baddies. + err := suite.state.DB.PutDomainPermissionSubscription(ctx, permSub) + if err != nil { + suite.FailNow(err.Error()) + } + + // Prepare the request to the /test endpoint. + subPath := strings.ReplaceAll( + admin.DomainPermissionSubscriptionTestPath, + ":id", permSub.ID, + ) + path := "/api" + subPath + recorder := httptest.NewRecorder() + ginCtx := suite.newContext(recorder, http.MethodPost, nil, path, "application/json") + ginCtx.Params = gin.Params{ + gin.Param{ + Key: apiutil.IDKey, + Value: permSub.ID, + }, + } + + // Trigger the handler. + suite.adminModule.DomainPermissionSubscriptionTestPOSTHandler(ginCtx) + suite.Equal(http.StatusOK, recorder.Code) + + // Read the body back. + b, err := io.ReadAll(recorder.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + dst := new(bytes.Buffer) + if err := json.Indent(dst, b, "", " "); err != nil { + suite.FailNow(err.Error()) + } + + // Ensure expected. + suite.Equal(`[ + { + "domain": "bumfaces.net", + "public_comment": "big jerks" + }, + { + "domain": "peepee.poopoo", + "public_comment": "harassment" + }, + { + "domain": "nothanks.com" + } +]`, dst.String()) + + // No permissions should be created + // since this is a dry run / test. + blocked, err := suite.state.DB.AreDomainsBlocked( + ctx, + []string{"bumfaces.net", "peepee.poopoo", "nothanks.com"}, + ) + if err != nil { + suite.FailNow(err.Error()) + } + suite.False(blocked) +} + +func TestDomainPermissionSubscriptionTestTestSuite(t *testing.T) { + suite.Run(t, &DomainPermissionSubscriptionTestTestSuite{}) +} diff --git a/internal/api/client/bookmarks/bookmarks_test.go b/internal/api/client/bookmarks/bookmarks_test.go index cb796e9e8..30602ad3b 100644 --- a/internal/api/client/bookmarks/bookmarks_test.go +++ b/internal/api/client/bookmarks/bookmarks_test.go @@ -28,6 +28,7 @@ "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/bookmarks" "github.com/superseriousbusiness/gotosocial/internal/api/client/statuses" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" @@ -95,6 +96,7 @@ func (suite *BookmarkTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/favourites/favourites_test.go b/internal/api/client/favourites/favourites_test.go index bd0ebce2e..571f647fb 100644 --- a/internal/api/client/favourites/favourites_test.go +++ b/internal/api/client/favourites/favourites_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/favourites" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -79,6 +80,7 @@ func (suite *FavouritesStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/filters/v1/filter_test.go b/internal/api/client/filters/v1/filter_test.go index 7553008d3..9a8258f12 100644 --- a/internal/api/client/filters/v1/filter_test.go +++ b/internal/api/client/filters/v1/filter_test.go @@ -23,6 +23,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" filtersV1 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v1" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/filters/v2/filter_test.go b/internal/api/client/filters/v2/filter_test.go index 8249546fb..0c13c33f7 100644 --- a/internal/api/client/filters/v2/filter_test.go +++ b/internal/api/client/filters/v2/filter_test.go @@ -23,6 +23,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" filtersV2 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v2" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/followedtags/followedtags_test.go b/internal/api/client/followedtags/followedtags_test.go index 883ab033b..5d46c0fa0 100644 --- a/internal/api/client/followedtags/followedtags_test.go +++ b/internal/api/client/followedtags/followedtags_test.go @@ -21,6 +21,7 @@ "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/followedtags" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -79,6 +80,7 @@ func (suite *FollowedTagsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/followrequests/followrequest_test.go b/internal/api/client/followrequests/followrequest_test.go index fc9843b4a..46dde691c 100644 --- a/internal/api/client/followrequests/followrequest_test.go +++ b/internal/api/client/followrequests/followrequest_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/followrequests" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -82,6 +83,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go index 8bfe444e5..8de29b4e1 100644 --- a/internal/api/client/instance/instance_test.go +++ b/internal/api/client/instance/instance_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/instance" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -84,6 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/lists/lists_test.go b/internal/api/client/lists/lists_test.go index 844d54cbb..f2ebfd29e 100644 --- a/internal/api/client/lists/lists_test.go +++ b/internal/api/client/lists/lists_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/lists" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -85,6 +86,7 @@ func (suite *ListsStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/mutes/mutes_test.go b/internal/api/client/mutes/mutes_test.go index 5d450e32c..35f6fa4e5 100644 --- a/internal/api/client/mutes/mutes_test.go +++ b/internal/api/client/mutes/mutes_test.go @@ -25,6 +25,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/mutes" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -81,6 +82,7 @@ func (suite *MutesTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/notifications/notifications_test.go b/internal/api/client/notifications/notifications_test.go index 23af65cb4..693be3d8f 100644 --- a/internal/api/client/notifications/notifications_test.go +++ b/internal/api/client/notifications/notifications_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/notifications" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -81,6 +82,7 @@ func (suite *NotificationsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/polls/polls_test.go b/internal/api/client/polls/polls_test.go index 5a3c83580..28cc190ba 100644 --- a/internal/api/client/polls/polls_test.go +++ b/internal/api/client/polls/polls_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/polls" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -76,6 +77,7 @@ func (suite *PollsStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/reports/reports_test.go b/internal/api/client/reports/reports_test.go index b36017d69..af084671e 100644 --- a/internal/api/client/reports/reports_test.go +++ b/internal/api/client/reports/reports_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/reports" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -76,6 +77,7 @@ func (suite *ReportsStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/search/search_test.go b/internal/api/client/search/search_test.go index 5ba198062..ce2f34fc7 100644 --- a/internal/api/client/search/search_test.go +++ b/internal/api/client/search/search_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/search" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -80,6 +81,7 @@ func (suite *SearchStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/statuses/status_test.go b/internal/api/client/statuses/status_test.go index 1a92276a1..a56963c45 100644 --- a/internal/api/client/statuses/status_test.go +++ b/internal/api/client/statuses/status_test.go @@ -25,6 +25,7 @@ "strings" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/statuses" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -192,6 +193,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/streaming/streaming_test.go b/internal/api/client/streaming/streaming_test.go index acdcafd8a..2080f2b69 100644 --- a/internal/api/client/streaming/streaming_test.go +++ b/internal/api/client/streaming/streaming_test.go @@ -31,6 +31,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/streaming" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -92,6 +93,7 @@ func (suite *StreamingTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/tags/tags_test.go b/internal/api/client/tags/tags_test.go index 79c708b10..f86d71053 100644 --- a/internal/api/client/tags/tags_test.go +++ b/internal/api/client/tags/tags_test.go @@ -26,6 +26,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/tags" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" @@ -87,6 +88,7 @@ func (suite *TagsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go index 808daf1a3..b9542f8be 100644 --- a/internal/api/client/user/user_test.go +++ b/internal/api/client/user/user_test.go @@ -24,6 +24,7 @@ "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/client/user" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -72,6 +73,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/api/fileserver/fileserver_test.go b/internal/api/fileserver/fileserver_test.go index e5f684d0c..34912bae3 100644 --- a/internal/api/fileserver/fileserver_test.go +++ b/internal/api/fileserver/fileserver_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/fileserver" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -98,6 +99,7 @@ func (suite *FileserverTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") diff --git a/internal/api/wellknown/webfinger/webfinger_test.go b/internal/api/wellknown/webfinger/webfinger_test.go index 67ac5a64e..68c671a13 100644 --- a/internal/api/wellknown/webfinger/webfinger_test.go +++ b/internal/api/wellknown/webfinger/webfinger_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/api/wellknown/webfinger" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -79,6 +80,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( diff --git a/internal/api/wellknown/webfinger/webfingerget_test.go b/internal/api/wellknown/webfinger/webfingerget_test.go index ce9bc0ccf..b3aec57fe 100644 --- a/internal/api/wellknown/webfinger/webfingerget_test.go +++ b/internal/api/wellknown/webfinger/webfingerget_test.go @@ -39,6 +39,7 @@ "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -90,6 +91,7 @@ func (suite *WebfingerGetTestSuite) funkifyAccountDomain(host string, accountDom suite.processor = processing.NewProcessor( cleaner.New(&suite.state), + subscriptions.New(&suite.state, suite.federator.TransportController(), suite.tc), suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), diff --git a/internal/cleaner/media_test.go b/internal/cleaner/media_test.go index 6e653c07c..74da2827b 100644 --- a/internal/cleaner/media_test.go +++ b/internal/cleaner/media_test.go @@ -26,6 +26,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" @@ -67,6 +68,7 @@ func (suite *MediaTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.storage = testrig.NewInMemoryStorage() suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.state.Storage = suite.storage testrig.StandardStorageSetup(suite.storage, "../../testrig/media") diff --git a/internal/config/config.go b/internal/config/config.go index 2bf2a77ad..72154b3f2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -78,15 +78,17 @@ type Configuration struct { WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."` WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"` - InstanceFederationMode string `name:"instance-federation-mode" usage:"Set instance federation mode."` - InstanceFederationSpamFilter bool `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"` - InstanceExposePeers bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"` - InstanceExposeSuspended bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"` - InstanceExposeSuspendedWeb bool `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"` - InstanceExposePublicTimeline bool `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"` - InstanceDeliverToSharedInboxes bool `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."` - InstanceInjectMastodonVersion bool `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"` - InstanceLanguages language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."` + InstanceFederationMode string `name:"instance-federation-mode" usage:"Set instance federation mode."` + InstanceFederationSpamFilter bool `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"` + InstanceExposePeers bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"` + InstanceExposeSuspended bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"` + InstanceExposeSuspendedWeb bool `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"` + InstanceExposePublicTimeline bool `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"` + InstanceDeliverToSharedInboxes bool `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."` + InstanceInjectMastodonVersion bool `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"` + InstanceLanguages language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."` + InstanceSubscriptionsProcessFrom string `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."` + InstanceSubscriptionsProcessEvery time.Duration `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."` AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."` AccountsReasonRequired bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 97d96d1ba..8c2ae90de 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -58,13 +58,15 @@ WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/", - InstanceFederationMode: InstanceFederationModeDefault, - InstanceFederationSpamFilter: false, - InstanceExposePeers: false, - InstanceExposeSuspended: false, - InstanceExposeSuspendedWeb: false, - InstanceDeliverToSharedInboxes: true, - InstanceLanguages: make(language.Languages, 0), + InstanceFederationMode: InstanceFederationModeDefault, + InstanceFederationSpamFilter: false, + InstanceExposePeers: false, + InstanceExposeSuspended: false, + InstanceExposeSuspendedWeb: false, + InstanceDeliverToSharedInboxes: true, + InstanceLanguages: make(language.Languages, 0), + InstanceSubscriptionsProcessFrom: "23:00", // 11pm, + InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day. AccountsRegistrationOpen: false, AccountsReasonRequired: true, diff --git a/internal/config/flags.go b/internal/config/flags.go index f96709e70..6f0957c36 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -90,6 +90,8 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) { cmd.Flags().Bool(InstanceExposeSuspendedWebFlag(), cfg.InstanceExposeSuspendedWeb, fieldtag("InstanceExposeSuspendedWeb", "usage")) cmd.Flags().Bool(InstanceDeliverToSharedInboxesFlag(), cfg.InstanceDeliverToSharedInboxes, fieldtag("InstanceDeliverToSharedInboxes", "usage")) cmd.Flags().StringSlice(InstanceLanguagesFlag(), cfg.InstanceLanguages.TagStrs(), fieldtag("InstanceLanguages", "usage")) + cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage")) + cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage")) // Accounts cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage")) diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 625c4ea78..e1c41638c 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1000,6 +1000,62 @@ func GetInstanceLanguages() language.Languages { return global.GetInstanceLangua // SetInstanceLanguages safely sets the value for global configuration 'InstanceLanguages' field func SetInstanceLanguages(v language.Languages) { global.SetInstanceLanguages(v) } +// GetInstanceSubscriptionsProcessFrom safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field +func (st *ConfigState) GetInstanceSubscriptionsProcessFrom() (v string) { + st.mutex.RLock() + v = st.config.InstanceSubscriptionsProcessFrom + st.mutex.RUnlock() + return +} + +// SetInstanceSubscriptionsProcessFrom safely sets the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field +func (st *ConfigState) SetInstanceSubscriptionsProcessFrom(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.InstanceSubscriptionsProcessFrom = v + st.reloadToViper() +} + +// InstanceSubscriptionsProcessFromFlag returns the flag name for the 'InstanceSubscriptionsProcessFrom' field +func InstanceSubscriptionsProcessFromFlag() string { return "instance-subscriptions-process-from" } + +// GetInstanceSubscriptionsProcessFrom safely fetches the value for global configuration 'InstanceSubscriptionsProcessFrom' field +func GetInstanceSubscriptionsProcessFrom() string { + return global.GetInstanceSubscriptionsProcessFrom() +} + +// SetInstanceSubscriptionsProcessFrom safely sets the value for global configuration 'InstanceSubscriptionsProcessFrom' field +func SetInstanceSubscriptionsProcessFrom(v string) { global.SetInstanceSubscriptionsProcessFrom(v) } + +// GetInstanceSubscriptionsProcessEvery safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field +func (st *ConfigState) GetInstanceSubscriptionsProcessEvery() (v time.Duration) { + st.mutex.RLock() + v = st.config.InstanceSubscriptionsProcessEvery + st.mutex.RUnlock() + return +} + +// SetInstanceSubscriptionsProcessEvery safely sets the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field +func (st *ConfigState) SetInstanceSubscriptionsProcessEvery(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.InstanceSubscriptionsProcessEvery = v + st.reloadToViper() +} + +// InstanceSubscriptionsProcessEveryFlag returns the flag name for the 'InstanceSubscriptionsProcessEvery' field +func InstanceSubscriptionsProcessEveryFlag() string { return "instance-subscriptions-process-every" } + +// GetInstanceSubscriptionsProcessEvery safely fetches the value for global configuration 'InstanceSubscriptionsProcessEvery' field +func GetInstanceSubscriptionsProcessEvery() time.Duration { + return global.GetInstanceSubscriptionsProcessEvery() +} + +// SetInstanceSubscriptionsProcessEvery safely sets the value for global configuration 'InstanceSubscriptionsProcessEvery' field +func SetInstanceSubscriptionsProcessEvery(v time.Duration) { + global.SetInstanceSubscriptionsProcessEvery(v) +} + // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) { st.mutex.RLock() diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go index f00e876ae..1690a68e2 100644 --- a/internal/federation/dereferencing/dereferencer_test.go +++ b/internal/federation/dereferencing/dereferencer_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" @@ -77,6 +78,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.client = testrig.NewMockHTTPClient(nil, "../../../testrig/media") suite.storage = testrig.NewInMemoryStorage() suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.state.Storage = suite.storage visFilter := visibility.NewFilter(&suite.state) diff --git a/internal/federation/federatingdb/federatingdb_test.go b/internal/federation/federatingdb/federatingdb_test.go index 360094887..f07d828c7 100644 --- a/internal/federation/federatingdb/federatingdb_test.go +++ b/internal/federation/federatingdb/federatingdb_test.go @@ -22,6 +22,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" @@ -91,6 +92,7 @@ func (suite *FederatingDBTestSuite) SetupTest() { testrig.StandardDBSetup(suite.db, suite.testAccounts) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) } func (suite *FederatingDBTestSuite) TearDownTest() { diff --git a/internal/media/media_test.go b/internal/media/media_test.go index 0980bf295..daefc910d 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -53,6 +54,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.storage = testrig.NewInMemoryStorage() suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.state.Storage = suite.storage testrig.StandardStorageSetup(suite.storage, "../../testrig/media") diff --git a/internal/oauth/clientstore_test.go b/internal/oauth/clientstore_test.go index fc3dabc3a..5b01f1548 100644 --- a/internal/oauth/clientstore_test.go +++ b/internal/oauth/clientstore_test.go @@ -22,6 +22,7 @@ "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/state" @@ -54,6 +55,7 @@ func (suite *PgClientStoreTestSuite) SetupTest() { testrig.InitTestConfig() suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) testrig.StandardDBSetup(suite.db, nil) } diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go index 8eec1f9dd..0ff66f76d 100644 --- a/internal/processing/account/account_test.go +++ b/internal/processing/account/account_test.go @@ -22,6 +22,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -93,6 +94,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( diff --git a/internal/processing/admin/account_test.go b/internal/processing/admin/account_test.go index 59b8afc77..7665cf4e3 100644 --- a/internal/processing/admin/account_test.go +++ b/internal/processing/admin/account_test.go @@ -53,7 +53,7 @@ func (suite *AccountTestSuite) TestAccountActionSuspend() { // Wait for action to finish. if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return suite.state.Actions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } diff --git a/internal/processing/admin/accountaction.go b/internal/processing/admin/accountaction.go index 59d4b420e..3072c3d51 100644 --- a/internal/processing/admin/accountaction.go +++ b/internal/processing/admin/accountaction.go @@ -68,7 +68,7 @@ func (p *Processor) accountActionSuspend( ) (string, gtserror.WithCode) { actionID := id.NewULID() - errWithCode := p.actions.Run( + errWithCode := p.state.Actions.Run( ctx, >smodel.AdminAction{ ID: actionID, diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go index 170298ca5..08e6bf0d5 100644 --- a/internal/processing/admin/admin.go +++ b/internal/processing/admin/admin.go @@ -21,10 +21,10 @@ "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/processing/common" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -33,21 +33,14 @@ type Processor struct { // common processor logic c *common.Processor - state *state.State - cleaner *cleaner.Cleaner - converter *typeutils.Converter - federator *federation.Federator - media *media.Manager - transport transport.Controller - email email.Sender - - // admin Actions currently - // undergoing processing - actions *Actions -} - -func (p *Processor) Actions() *Actions { - return p.actions + state *state.State + cleaner *cleaner.Cleaner + subscriptions *subscriptions.Subscriptions + converter *typeutils.Converter + federator *federation.Federator + media *media.Manager + transport transport.Controller + email email.Sender } // New returns a new admin processor. @@ -55,6 +48,7 @@ func New( common *common.Processor, state *state.State, cleaner *cleaner.Cleaner, + subscriptions *subscriptions.Subscriptions, federator *federation.Federator, converter *typeutils.Converter, mediaManager *media.Manager, @@ -62,17 +56,14 @@ func New( emailSender email.Sender, ) Processor { return Processor{ - c: common, - state: state, - cleaner: cleaner, - converter: converter, - federator: federator, - media: mediaManager, - transport: transportController, - email: emailSender, - actions: &Actions{ - r: make(map[string]*gtsmodel.AdminAction), - state: state, - }, + c: common, + state: state, + cleaner: cleaner, + subscriptions: subscriptions, + converter: converter, + federator: federator, + media: mediaManager, + transport: transportController, + email: emailSender, } } diff --git a/internal/processing/admin/admin_test.go b/internal/processing/admin/admin_test.go index 3251264b6..cdb6af2b0 100644 --- a/internal/processing/admin/admin_test.go +++ b/internal/processing/admin/admin_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -33,6 +34,7 @@ "github.com/superseriousbusiness/gotosocial/internal/processing/admin" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -89,6 +91,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( @@ -109,6 +112,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { suite.processor = processing.NewProcessor( cleaner.New(&suite.state), + subscriptions.New(&suite.state, suite.transportController, suite.tc), suite.tc, suite.federator, suite.oauthServer, diff --git a/internal/processing/admin/domainallow.go b/internal/processing/admin/domainallow.go index bab54e308..e21538429 100644 --- a/internal/processing/admin/domainallow.go +++ b/internal/processing/admin/domainallow.go @@ -22,14 +22,11 @@ "errors" "fmt" - "codeberg.org/gruf/go-kv" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/text" ) @@ -69,84 +66,30 @@ func (p *Processor) createDomainAllow( } } - actionID := id.NewULID() + // Run admin action to process + // side effects of allow. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domainAllow.Domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: adminAcct.ID, + } - // Process domain allow side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.Actions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domain, - Type: gtsmodel.AdminActionSuspend, - AccountID: adminAcct.ID, - Text: domainAllow.PrivateComment, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - l.Info("processing domain allow side effects") - defer func() { l.Info("finished processing domain allow side effects") }() - - return p.domainAllowSideEffects(ctx, domainAllow) - }, + action, + p.state.Actions.DomainAllowF(action.ID, domainAllow), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } apiDomainAllow, errWithCode := p.apiDomainPerm(ctx, domainAllow, false) if errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainAllow, actionID, nil -} - -func (p *Processor) domainAllowSideEffects( - ctx context.Context, - allow *gtsmodel.DomainAllow, -) gtserror.MultiError { - if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { - // We're running in allowlist mode, - // so there are no side effects to - // process here. - return nil - } - - // We're running in blocklist mode or - // some similar mode which necessitates - // domain allow side effects if a block - // was in place when the allow was created. - // - // So, check if there's a block. - block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs := gtserror.NewMultiError(1) - errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) - return errs - } - - if block == nil { - // No block? - // No problem! - return nil - } - - // There was a block, over which the new - // allow ought to take precedence. To account - // for this, just run side effects as though - // the domain was being unblocked, while - // leaving the existing block in place. - // - // Any accounts that were suspended by - // the block will be unsuspended and be - // able to interact with the instance again. - return p.domainUnblockSideEffects(ctx, block) + return apiDomainAllow, action.ID, nil } func (p *Processor) deleteDomainAllow( @@ -179,77 +122,23 @@ func (p *Processor) deleteDomainAllow( return nil, "", gtserror.NewErrorInternalError(err) } - actionID := id.NewULID() + // Run admin action to process + // side effects of unallow. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domainAllow.Domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: adminAcct.ID, + } - // Process domain unallow side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.Actions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domainAllow.Domain, - Type: gtsmodel.AdminActionUnsuspend, - AccountID: adminAcct.ID, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domainAllow.Domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - l.Info("processing domain unallow side effects") - defer func() { l.Info("finished processing domain unallow side effects") }() - - return p.domainUnallowSideEffects(ctx, domainAllow) - }, + action, + p.state.Actions.DomainUnallowF(action.ID, domainAllow), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainAllow, actionID, nil -} - -func (p *Processor) domainUnallowSideEffects( - ctx context.Context, - allow *gtsmodel.DomainAllow, -) gtserror.MultiError { - if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist { - // We're running in allowlist mode, - // so there are no side effects to - // process here. - return nil - } - - // We're running in blocklist mode or - // some similar mode which necessitates - // domain allow side effects if a block - // was in place when the allow was removed. - // - // So, check if there's a block. - block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs := gtserror.NewMultiError(1) - errs.Appendf("db error getting domain block %s: %w", allow.Domain, err) - return errs - } - - if block == nil { - // No block? - // No problem! - return nil - } - - // There was a block, over which the previous - // allow was taking precedence. Now that the - // allow has been removed, we should put the - // side effects of the block back in place. - // - // To do this, process the block side effects - // again as though the block were freshly - // created. This will mark all accounts from - // the blocked domain as suspended, and clean - // up their follows/following, media, etc. - return p.domainBlockSideEffects(ctx, block) + return apiDomainAllow, action.ID, nil } diff --git a/internal/processing/admin/domainblock.go b/internal/processing/admin/domainblock.go index 2fe10c97b..940c0dfce 100644 --- a/internal/processing/admin/domainblock.go +++ b/internal/processing/admin/domainblock.go @@ -21,18 +21,12 @@ "context" "errors" "fmt" - "time" - "codeberg.org/gruf/go-kv" - "github.com/superseriousbusiness/gotosocial/internal/ap" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/text" ) @@ -72,149 +66,31 @@ func (p *Processor) createDomainBlock( } } - actionID := id.NewULID() + // Run admin action to process + // side effects of block. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionSuspend, + AccountID: adminAcct.ID, + Text: domainBlock.PrivateComment, + } - // Process domain block side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.Actions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domain, - Type: gtsmodel.AdminActionSuspend, - AccountID: adminAcct.ID, - Text: domainBlock.PrivateComment, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - skip, err := p.skipBlockSideEffects(ctx, domain) - if err != nil { - return err - } - if skip != "" { - l.Infof("skipping domain block side effects: %s", skip) - return nil - } - - l.Info("processing domain block side effects") - defer func() { l.Info("finished processing domain block side effects") }() - - return p.domainBlockSideEffects(ctx, domainBlock) - }, + action, + p.state.Actions.DomainBlockF(action.ID, domainBlock), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } apiDomainBlock, errWithCode := p.apiDomainPerm(ctx, domainBlock, false) if errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainBlock, actionID, nil -} - -// skipBlockSideEffects checks if side effects of block creation -// should be skipped for the given domain, taking account of -// instance federation mode, and existence of any allows -// which ought to "shield" this domain from being blocked. -// -// If the caller should skip, the returned string will be non-zero -// and will be set to a reason why side effects should be skipped. -// -// - blocklist mode + allow exists: "..." (skip) -// - blocklist mode + no allow: "" (don't skip) -// - allowlist mode + allow exists: "" (don't skip) -// - allowlist mode + no allow: "" (don't skip) -func (p *Processor) skipBlockSideEffects( - ctx context.Context, - domain string, -) (string, gtserror.MultiError) { - var ( - skip string // Assume "" (don't skip). - errs gtserror.MultiError - ) - - // Never skip block side effects in allowlist mode. - fediMode := config.GetInstanceFederationMode() - if fediMode == config.InstanceFederationModeAllowlist { - return skip, errs - } - - // We know we're in blocklist mode. - // - // We want to skip domain block side - // effects if an allow is already - // in place which overrides the block. - - // Check if an explicit allow exists for this domain. - domainAllow, err := p.state.DB.GetDomainAllow(ctx, domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs.Appendf("error getting domain allow: %w", err) - return skip, errs - } - - if domainAllow != nil { - skip = "running in blocklist mode, and an explicit allow exists for this domain" - return skip, errs - } - - return skip, errs -} - -// domainBlockSideEffects processes the side effects of a domain block: -// -// 1. Strip most info away from the instance entry for the domain. -// 2. Pass each account from the domain to the processor for deletion. -// -// It should be called asynchronously, since it can take a while when -// there are many accounts present on the given domain. -func (p *Processor) domainBlockSideEffects( - ctx context.Context, - block *gtsmodel.DomainBlock, -) gtserror.MultiError { - var errs gtserror.MultiError - - // If we have an instance entry for this domain, - // update it with the new block ID and clear all fields - instance, err := p.state.DB.GetInstance(ctx, block.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs.Appendf("db error getting instance %s: %w", block.Domain, err) - return errs - } - - if instance != nil { - // We had an entry for this domain. - columns := stubbifyInstance(instance, block.ID) - if err := p.state.DB.UpdateInstance(ctx, instance, columns...); err != nil { - errs.Appendf("db error updating instance: %w", err) - return errs - } - } - - // For each account that belongs to this domain, - // process an account delete message to remove - // that account's posts, media, etc. - if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { - if err := p.state.Workers.Client.Process(ctx, &messages.FromClientAPI{ - APObjectType: ap.ActorPerson, - APActivityType: ap.ActivityDelete, - GTSModel: block, - Origin: account, - Target: account, - }); err != nil { - errs.Append(err) - } - }); err != nil { - errs.Appendf("db error ranging through accounts: %w", err) - } - - return errs + return apiDomainBlock, action.ID, nil } func (p *Processor) deleteDomainBlock( @@ -247,104 +123,23 @@ func (p *Processor) deleteDomainBlock( return nil, "", gtserror.NewErrorInternalError(err) } - actionID := id.NewULID() + // Run admin action to process + // side effects of unblock. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domainBlock.Domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: adminAcct.ID, + } - // Process domain unblock side - // effects asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.Actions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domainBlock.Domain, - Type: gtsmodel.AdminActionUnsuspend, - AccountID: adminAcct.ID, - }, - func(ctx context.Context) gtserror.MultiError { - // Log start + finish. - l := log.WithFields(kv.Fields{ - {"domain", domainBlock.Domain}, - {"actionID", actionID}, - }...).WithContext(ctx) - - l.Info("processing domain unblock side effects") - defer func() { l.Info("finished processing domain unblock side effects") }() - - return p.domainUnblockSideEffects(ctx, domainBlock) - }, + action, + p.state.Actions.DomainUnblockF(action.ID, domainBlock), ); errWithCode != nil { - return nil, actionID, errWithCode + return nil, action.ID, errWithCode } - return apiDomainBlock, actionID, nil -} - -// domainUnblockSideEffects processes the side effects of undoing a -// domain block: -// -// 1. Mark instance entry as no longer suspended. -// 2. Mark each account from the domain as no longer suspended, if the -// suspension origin corresponds to the ID of the provided domain block. -// -// It should be called asynchronously, since it can take a while when -// there are many accounts present on the given domain. -func (p *Processor) domainUnblockSideEffects( - ctx context.Context, - block *gtsmodel.DomainBlock, -) gtserror.MultiError { - var errs gtserror.MultiError - - // Update instance entry for this domain, if we have it. - instance, err := p.state.DB.GetInstance(ctx, block.Domain) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - errs.Appendf("db error getting instance %s: %w", block.Domain, err) - } - - if instance != nil { - // We had an entry, update it to signal - // that it's no longer suspended. - instance.SuspendedAt = time.Time{} - instance.DomainBlockID = "" - if err := p.state.DB.UpdateInstance( - ctx, - instance, - "suspended_at", - "domain_block_id", - ); err != nil { - errs.Appendf("db error updating instance: %w", err) - return errs - } - } - - // Unsuspend all accounts whose suspension origin was this domain block. - if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) { - if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() { - // Account wasn't suspended, nothing to do. - return - } - - if account.SuspensionOrigin != block.ID { - // Account was suspended, but not by - // this domain block, leave it alone. - return - } - - // Account was suspended by this domain - // block, mark it as unsuspended. - account.SuspendedAt = time.Time{} - account.SuspensionOrigin = "" - - if err := p.state.DB.UpdateAccount( - ctx, - account, - "suspended_at", - "suspension_origin", - ); err != nil { - errs.Appendf("db error updating account %s: %w", account.Username, err) - } - }); err != nil { - errs.Appendf("db error ranging through accounts: %w", err) - } - - return errs + return apiDomainBlock, action.ID, nil } diff --git a/internal/processing/admin/domainkeysexpire.go b/internal/processing/admin/domainkeysexpire.go index 9853becbd..76d3ad90f 100644 --- a/internal/processing/admin/domainkeysexpire.go +++ b/internal/processing/admin/domainkeysexpire.go @@ -19,7 +19,6 @@ import ( "context" - "time" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -39,47 +38,23 @@ func (p *Processor) DomainKeysExpire( adminAcct *gtsmodel.Account, domain string, ) (string, gtserror.WithCode) { - actionID := id.NewULID() + // Run admin action to process + // side effects of key expiry. + action := >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionExpireKeys, + AccountID: adminAcct.ID, + } - // Process key expiration asynchronously. - if errWithCode := p.actions.Run( + if errWithCode := p.state.Actions.Run( ctx, - >smodel.AdminAction{ - ID: actionID, - TargetCategory: gtsmodel.AdminActionCategoryDomain, - TargetID: domain, - Type: gtsmodel.AdminActionExpireKeys, - AccountID: adminAcct.ID, - }, - func(ctx context.Context) gtserror.MultiError { - return p.domainKeysExpireSideEffects(ctx, domain) - }, + action, + p.state.Actions.DomainKeysExpireF(domain), ); errWithCode != nil { - return actionID, errWithCode + return action.ID, errWithCode } - return actionID, nil -} - -func (p *Processor) domainKeysExpireSideEffects(ctx context.Context, domain string) gtserror.MultiError { - var ( - expiresAt = time.Now() - errs gtserror.MultiError - ) - - // For each account on this domain, expire - // the public key and update the account. - if err := p.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) { - account.PublicKeyExpiresAt = expiresAt - if err := p.state.DB.UpdateAccount(ctx, - account, - "public_key_expires_at", - ); err != nil { - errs.Appendf("db error updating account: %w", err) - } - }); err != nil { - errs.Appendf("db error ranging through accounts: %w", err) - } - - return errs + return action.ID, nil } diff --git a/internal/processing/admin/domainpermission_test.go b/internal/processing/admin/domainpermission_test.go index 5a73693db..577ec69b4 100644 --- a/internal/processing/admin/domainpermission_test.go +++ b/internal/processing/admin/domainpermission_test.go @@ -186,7 +186,7 @@ func (suite *DomainBlockTestSuite) awaitAction(actionID string) { ctx := context.Background() if !testrig.WaitFor(func() bool { - return suite.adminProcessor.Actions().TotalRunning() == 0 + return suite.state.Actions.TotalRunning() == 0 }) { suite.FailNow("timed out waiting for admin action(s) to finish") } diff --git a/internal/processing/admin/domainpermissionsubscription.go b/internal/processing/admin/domainpermissionsubscription.go index 3d2f63d56..31be40a80 100644 --- a/internal/processing/admin/domainpermissionsubscription.go +++ b/internal/processing/admin/domainpermissionsubscription.go @@ -22,6 +22,7 @@ "errors" "fmt" "net/url" + "slices" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util" @@ -283,3 +284,90 @@ func (p *Processor) DomainPermissionSubscriptionRemove( return p.apiDomainPermSub(ctx, permSub) } + +func (p *Processor) DomainPermissionSubscriptionTest( + ctx context.Context, + acct *gtsmodel.Account, + id string, +) (any, gtserror.WithCode) { + permSub, err := p.state.DB.GetDomainPermissionSubscriptionByID(ctx, id) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("db error getting domain permission subscription %s: %w", id, err) + return nil, gtserror.NewErrorInternalError(err) + } + + if permSub == nil { + err := fmt.Errorf("domain permission subscription %s not found", id) + return nil, gtserror.NewErrorNotFound(err, err.Error()) + } + + // To process the test/dry-run correctly, we need to get + // all domain perm subs of this type with a *higher* priority, + // to know whether we ought to create permissions or not. + permSubs, err := p.state.DB.GetDomainPermissionSubscriptionsByPriority( + ctx, + permSub.PermissionType, + ) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("db error: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Find the index of the targeted + // subscription in the slice. + index := slices.IndexFunc( + permSubs, + func(ps *gtsmodel.DomainPermissionSubscription) bool { + return ps.ID == permSub.ID + }, + ) + + // Everything *before* the targeted subscription has a higher priority. + getHigherPrios := func() ([]*gtsmodel.DomainPermissionSubscription, error) { + return permSubs[:index], nil + } + + // Get a transport for calling permSub.URI. + tsport, err := p.transport.NewTransportForUsername(ctx, acct.Username) + if err != nil { + err := gtserror.Newf("error getting transport: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // Call the permSub.URI and parse a list of perms from it. + // Any error returned here is a "real" one, not an error + // from fetching / parsing the list. + createdPerms, err := p.subscriptions.ProcessDomainPermissionSubscription( + ctx, + permSub, + tsport, + getHigherPrios, + true, // Dry run. + ) + if err != nil { + err := gtserror.Newf("error doing dry-run: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } + + // If permSub has an error set on it now, + // we should return it to the caller. + if permSub.Error != "" { + return map[string]string{ + "error": permSub.Error, + }, nil + } + + // No error, so return the list of + // perms that would have been created. + apiPerms := make([]*apimodel.DomainPermission, 0, len(createdPerms)) + for _, perm := range createdPerms { + apiPerm, errWithCode := p.apiDomainPerm(ctx, perm, false) + if errWithCode != nil { + return nil, errWithCode + } + + apiPerms = append(apiPerms, apiPerm) + } + + return apiPerms, nil +} diff --git a/internal/processing/admin/util.go b/internal/processing/admin/util.go index aef435856..f04b3654b 100644 --- a/internal/processing/admin/util.go +++ b/internal/processing/admin/util.go @@ -19,86 +19,12 @@ import ( "context" - "errors" - "time" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -// stubbifyInstance renders the given instance as a stub, -// removing most information from it and marking it as -// suspended. -// -// For caller's convenience, this function returns the db -// names of all columns that are updated by it. -func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string { - instance.Title = "" - instance.SuspendedAt = time.Now() - instance.DomainBlockID = domainBlockID - instance.ShortDescription = "" - instance.Description = "" - instance.Terms = "" - instance.ContactEmail = "" - instance.ContactAccountUsername = "" - instance.ContactAccountID = "" - instance.Version = "" - - return []string{ - "title", - "suspended_at", - "domain_block_id", - "short_description", - "description", - "terms", - "contact_email", - "contact_account_username", - "contact_account_id", - "version", - } -} - -// rangeDomainAccounts iterates through all accounts -// originating from the given domain, and calls the -// provided range function on each account. -// -// If an error is returned while selecting accounts, -// the loop will stop and return the error. -func (p *Processor) rangeDomainAccounts( - ctx context.Context, - domain string, - rangeF func(*gtsmodel.Account), -) error { - var ( - limit = 50 // Limit selection to avoid spiking mem/cpu. - maxID string // Start with empty string to select from top. - ) - - for { - // Get (next) page of accounts. - accounts, err := p.state.DB.GetInstanceAccounts(ctx, domain, maxID, limit) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - // Real db error. - return gtserror.Newf("db error getting instance accounts: %w", err) - } - - if len(accounts) == 0 { - // No accounts left, we're done. - return nil - } - - // Set next max ID for paging down. - maxID = accounts[len(accounts)-1].ID - - // Call provided range function. - for _, account := range accounts { - rangeF(account) - } - } -} - // apiDomainPerm is a cheeky shortcut for returning // the API version of the given domain permission, // or an appropriate error if something goes wrong. diff --git a/internal/processing/conversations/conversations_test.go b/internal/processing/conversations/conversations_test.go index cc7ec617e..1d37e24be 100644 --- a/internal/processing/conversations/conversations_test.go +++ b/internal/processing/conversations/conversations_test.go @@ -23,6 +23,7 @@ "time" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" dbtest "github.com/superseriousbusiness/gotosocial/internal/db/test" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -103,6 +104,7 @@ func (suite *ConversationsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) suite.filter = visibility.NewFilter(&suite.state) diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go index 80f1a7be7..28e8222a7 100644 --- a/internal/processing/media/media_test.go +++ b/internal/processing/media/media_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -75,6 +76,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.tc = typeutils.NewConverter(&suite.state) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/processing/processor.go b/internal/processing/processor.go index ce0f1cfb8..8dabfba96 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -48,6 +48,7 @@ "github.com/superseriousbusiness/gotosocial/internal/processing/user" "github.com/superseriousbusiness/gotosocial/internal/processing/workers" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -180,6 +181,7 @@ func (p *Processor) Workers() *workers.Processor { // NewProcessor returns a new Processor. func NewProcessor( cleaner *cleaner.Cleaner, + subscriptions *subscriptions.Subscriptions, converter *typeutils.Converter, federator *federation.Federator, oauthServer oauth.Server, @@ -210,7 +212,7 @@ func NewProcessor( // Instantiate the rest of the sub // processors + pin them to this struct. processor.account = account.New(&common, state, converter, mediaManager, federator, visFilter, parseMentionFunc) - processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender) + processor.admin = admin.New(&common, state, cleaner, subscriptions, federator, converter, mediaManager, federator.TransportController(), emailSender) processor.conversations = conversations.New(state, converter, visFilter) processor.fedi = fedi.New(state, &common, converter, federator, visFilter) processor.filtersv1 = filtersv1.New(state, converter, &processor.stream) diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go index d0898a98d..dc4507ba0 100644 --- a/internal/processing/processor_test.go +++ b/internal/processing/processor_test.go @@ -21,6 +21,7 @@ "context" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -34,6 +35,7 @@ "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/stream" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" @@ -102,6 +104,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.testActivities = testrig.NewTestActivities(suite.testAccounts) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage @@ -125,6 +128,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.processor = processing.NewProcessor( cleaner.New(&suite.state), + subscriptions.New(&suite.state, suite.transportController, suite.typeconverter), suite.typeconverter, suite.federator, suite.oauthServer, diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go index b3c446d14..604df095f 100644 --- a/internal/processing/status/status_test.go +++ b/internal/processing/status/status_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" @@ -84,6 +85,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.typeConverter = typeutils.NewConverter(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.tc = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media")) suite.storage = testrig.NewInMemoryStorage() diff --git a/internal/processing/stream/stream_test.go b/internal/processing/stream/stream_test.go index 2569ac701..98f44e999 100644 --- a/internal/processing/stream/stream_test.go +++ b/internal/processing/stream/stream_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/oauth" @@ -50,6 +51,7 @@ func (suite *StreamTestSuite) SetupTest() { suite.testTokens = testrig.NewTestTokens() suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.streamProcessor = stream.New(&suite.state, suite.oauthServer) diff --git a/internal/processing/timeline/timeline_test.go b/internal/processing/timeline/timeline_test.go index 593bfb8f3..a41572ab0 100644 --- a/internal/processing/timeline/timeline_test.go +++ b/internal/processing/timeline/timeline_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -55,6 +56,7 @@ func (suite *TimelineStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.timeline = timeline.New( &suite.state, diff --git a/internal/processing/user/user_test.go b/internal/processing/user/user_test.go index e473c5bb0..75ef985fb 100644 --- a/internal/processing/user/user_test.go +++ b/internal/processing/user/user_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -49,6 +50,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) diff --git a/internal/state/state.go b/internal/state/state.go index 90683acd4..8e962f10e 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -19,6 +19,7 @@ import ( "codeberg.org/gruf/go-mutexes" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/cache" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/storage" @@ -61,9 +62,14 @@ type State struct { // Storage provides access to the storage driver. Storage *storage.Driver - // Workers provides access to this state's collection of worker pools. + // Workers provides access to this + // state's collection of worker pools. Workers workers.Workers + // Struct to manage running admin + // actions (and locks thereupon). + Actions *actions.Actions + // prevent pass-by-value. _ nocopy } diff --git a/internal/subscriptions/domainperms.go b/internal/subscriptions/domainperms.go new file mode 100644 index 000000000..121647732 --- /dev/null +++ b/internal/subscriptions/domainperms.go @@ -0,0 +1,804 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subscriptions + +import ( + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strconv" + "strings" + "time" + + "codeberg.org/gruf/go-kv" + + "github.com/miekg/dns" + "github.com/superseriousbusiness/gotosocial/internal/actions" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// ScheduleJobs schedules domain permission subscription +// fetching + updating using configured parameters. +// +// Returns an error if `MediaCleanupFrom` +// is not a valid format (hh:mm:ss). +func (s *Subscriptions) ScheduleJobs() error { + const hourMinute = "15:04" + + var ( + now = time.Now() + processEvery = config.GetInstanceSubscriptionsProcessEvery() + processFromStr = config.GetInstanceSubscriptionsProcessFrom() + ) + + // Parse processFromStr as hh:mm. + // Resulting time will be on 1 Jan year zero. + cleanupFrom, err := time.Parse(hourMinute, processFromStr) + if err != nil { + return gtserror.Newf( + "error parsing '%s' in time format 'hh:mm': %w", + processFromStr, err, + ) + } + + // Time travel from + // year zero, groovy. + firstProcessAt := time.Date( + now.Year(), + now.Month(), + now.Day(), + cleanupFrom.Hour(), + cleanupFrom.Minute(), + 0, + 0, + now.Location(), + ) + + // Ensure first processing is in the future. + for firstProcessAt.Before(now) { + firstProcessAt = firstProcessAt.Add(processEvery) + } + + fn := func(ctx context.Context, start time.Time) { + log.Info(ctx, "starting instance subscriptions processing") + + // In blocklist (default) mode, process allows + // first to provide immunity to block side effects. + // + // In allowlist mode, process blocks first to + // ensure allowlist doesn't override blocks. + var order [2]gtsmodel.DomainPermissionType + if config.GetInstanceFederationMode() == config.InstanceFederationModeBlocklist { + order = [2]gtsmodel.DomainPermissionType{ + gtsmodel.DomainPermissionAllow, + gtsmodel.DomainPermissionBlock, + } + } else { + order = [2]gtsmodel.DomainPermissionType{ + gtsmodel.DomainPermissionBlock, + gtsmodel.DomainPermissionAllow, + } + } + + // Fetch + process subscribed perms in order. + for _, permType := range order { + s.ProcessDomainPermissionSubscriptions(ctx, permType) + } + + log.Infof(ctx, "finished instance subscriptions processing after %s", time.Since(start)) + } + + log.Infof(nil, + "scheduling instance subscriptions processing to run every %s, starting from %s; next clean will run at %s", + processEvery, processFromStr, firstProcessAt, + ) + + // Schedule processing to execute according to schedule. + if !s.state.Workers.Scheduler.AddRecurring( + "@subsprocessing", + firstProcessAt, + processEvery, + fn, + ) { + panic("failed to schedule @subsprocessing") + } + + return nil +} + +// ProcessDomainPermissionSubscriptions processes all domain permission +// subscriptions of the given permission type by, in turn, calling the +// URI of each subscription, parsing the result into a list of domain +// permissions, and creating (or skipping) each permission as appropriate. +func (s *Subscriptions) ProcessDomainPermissionSubscriptions( + ctx context.Context, + permType gtsmodel.DomainPermissionType, +) { + log.Info(ctx, "start") + + // Get permission subscriptions in priority order (highest -> lowest). + permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real db error. + log.Error(ctx, err) + return + } + + if len(permSubs) == 0 { + // No subscriptions of this + // type, so nothing to do. + return + } + + // Get a transport using the instance account, + // we can reuse this for each HTTP call. + tsport, err := s.transportController.NewTransportForUsername(ctx, "") + if err != nil { + log.Error(ctx, err) + return + } + + for i, permSub := range permSubs { + // Higher priority permission subs = everything + // above this permission sub in the slice. + getHigherPrios := func() ([]*gtsmodel.DomainPermissionSubscription, error) { + return permSubs[:i], nil + } + + _, err := s.ProcessDomainPermissionSubscription( + ctx, + permSub, + tsport, + getHigherPrios, + false, // Not dry. Wet, if you will. + ) + if err != nil { + // Real db error. + log.Error(ctx, err) + return + } + + // Update this perm sub. + err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub) + if err != nil { + // Real db error. + log.Error(ctx, err) + return + } + } + + log.Info(ctx, "finished") +} + +// ProcessDomainPermissionSubscription processes one domain permission +// subscription by dereferencing the URI, parsing the response into a list +// of permissions, and for each discovered permission either creating an +// entry in the database, or ignoring it if it's excluded or already +// covered by a higher-priority subscription. +// +// On success, the slice of discovered DomainPermissions will be returned. +// In case of parsing error, or error on the remote side, permSub.Error +// will be updated with the calling/parsing error, and `nil, nil` will be +// returned. In case of an actual db error, `nil, err` will be returned and +// the caller should handle it. +// +// getHigherPrios should be a function for returning a slice of domain +// permission subscriptions with a higher priority than the given permSub. +// +// If dry == true, then the URI will still be called, and permissions +// will be parsed, but they will not actually be created. +// +// Note that while this function modifies fields on the given permSub, +// it's up to the caller to update it in the database (if desired). +func (s *Subscriptions) ProcessDomainPermissionSubscription( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + tsport transport.Transport, + getHigherPrios func() ([]*gtsmodel.DomainPermissionSubscription, error), + dry bool, +) ([]gtsmodel.DomainPermission, error) { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"permType", permSub.PermissionType.String()}, + {"permSubURI", permSub.URI}, + }...) + + // Set FetchedAt as we're + // going to attempt this now. + permSub.FetchedAt = time.Now() + + // Call the URI, only force + // if we're doing a dry run. + resp, err := tsport.DereferenceDomainPermissions( + ctx, permSub, !dry, + ) + if err != nil { + // Couldn't get this one, + // set error + return. + errStr := err.Error() + l.Warnf("couldn't dereference permSubURI: %+v", err) + permSub.Error = errStr + return nil, nil + } + + // If the permissions at URI weren't modified + // since last time, just update some metadata + // to indicate a successful fetch, and return. + if resp.Unmodified { + l.Debug("received 304 Not Modified from remote") + permSub.SuccessfullyFetchedAt = permSub.FetchedAt + if permSub.ETag == "" && resp.ETag != "" { + // We didn't have an ETag before but + // we have one now: probably the remote + // added ETag support in the meantime. + permSub.ETag = resp.ETag + } + return nil, nil + } + + // At this point we know we got a 200 OK + // from the URI, so we've got a live body! + // Try to parse the body as a list of wantedPerms + // that the subscription wants to create. + var wantedPerms []gtsmodel.DomainPermission + + switch permSub.ContentType { + + // text/csv + case gtsmodel.DomainPermSubContentTypeCSV: + wantedPerms, err = s.permsFromCSV(l, permSub.PermissionType, resp.Body) + + // application/json + case gtsmodel.DomainPermSubContentTypeJSON: + wantedPerms, err = s.permsFromJSON(l, permSub.PermissionType, resp.Body) + + // text/plain + case gtsmodel.DomainPermSubContentTypePlain: + wantedPerms, err = s.permsFromPlain(l, permSub.PermissionType, resp.Body) + } + + if err != nil { + // We retrieved the permissions from remote but + // the connection died halfway through transfer, + // or we couldn't parse the results, or something. + // Just set error and return. + errStr := err.Error() + l.Warnf("couldn't parse results: %+v", err) + permSub.Error = errStr + return nil, nil + } + + if len(wantedPerms) == 0 { + // Fetch was OK, and parsing was, on the surface at + // least, OK, but we didn't get any perms. Consider + // this an error as users will probably want to know. + const errStr = "fetch successful but parsed zero usable results" + l.Warn(errStr) + permSub.Error = errStr + return nil, nil + } + + // This can now be considered a successful fetch. + permSub.SuccessfullyFetchedAt = permSub.FetchedAt + permSub.ETag = resp.ETag + permSub.Error = "" + + // Need a list of higher priority subscriptions + // to ensure we don't create permissions wrongly. + higherPrios, err := getHigherPrios() + if err != nil { + // Proper db error. + return nil, err + } + + // Keep track of which domain perms are + // created (or would be, if dry == true). + createdPerms := make([]gtsmodel.DomainPermission, 0, len(wantedPerms)) + + // Iterate through wantedPerms and + // create (or dry create) each one. + for _, wantedPerm := range wantedPerms { + l = l.WithField("domain", wantedPerm.GetDomain()) + created, err := s.processDomainPermission( + ctx, l, + wantedPerm, + permSub, + higherPrios, + dry, + ) + if err != nil { + // Proper db error. + return nil, err + } + + if !created { + continue + } + + createdPerms = append(createdPerms, wantedPerm) + } + + return createdPerms, nil +} + +// processDomainPermission processes one wanted domain +// permission discovered via a domain permission sub's URI. +// +// Error will only be returned in case of an actual database +// error, else the error will be logged and nil returned. +func (s *Subscriptions) processDomainPermission( + ctx context.Context, + l log.Entry, + wantedPerm gtsmodel.DomainPermission, + permSub *gtsmodel.DomainPermissionSubscription, + higherPrios []*gtsmodel.DomainPermissionSubscription, + dry bool, +) (bool, error) { + // Set to true if domain permission + // actually (would be) created. + var created bool + + // If domain is excluded from automatic + // permission creation, don't process it. + domain := wantedPerm.GetDomain() + excluded, err := s.state.DB.IsDomainPermissionExcluded(ctx, domain) + if err != nil { + // Proper db error. + return created, err + } + + if excluded { + l.Debug("domain is excluded, skipping") + return created, nil + } + + // Check if a permission already exists for + // this domain, and if it's covered already + // by a higher-priority subscription. + existingPerm, covered, err := s.existingCovered( + ctx, permSub.PermissionType, domain, higherPrios, + ) + if err != nil { + // Proper db error. + return created, err + } + + if covered { + l.Debug("domain is covered by a higher-priority subscription, skipping") + return created, nil + } + + // At this point we know we + // should create the perm. + created = true + + if dry { + // Don't do creation or side + // effects if we're dry running. + return created, nil + } + + // Handle perm creation differently depending + // on whether or not a perm already existed. + existing := !util.IsNil(existingPerm) + switch { + + case !existing && *permSub.AsDraft: + // No existing perm, create as draft. + err = s.state.DB.PutDomainPermissionDraft( + ctx, + >smodel.DomainPermissionDraft{ + ID: id.NewULID(), + PermissionType: permSub.PermissionType, + Domain: domain, + CreatedByAccountID: permSub.CreatedByAccount.ID, + CreatedByAccount: permSub.CreatedByAccount, + PrivateComment: permSub.URI, + PublicComment: wantedPerm.GetPublicComment(), + Obfuscate: wantedPerm.GetObfuscate(), + SubscriptionID: permSub.ID, + }, + ) + + case !existing && !*permSub.AsDraft: + // No existing perm, create a new one of the + // appropriate type, and process side effects. + var ( + insertF func() error + action *gtsmodel.AdminAction + actionF actions.AdminActionF + ) + + if permSub.PermissionType == gtsmodel.DomainPermissionBlock { + // Prepare to insert + process a block. + domainBlock := >smodel.DomainBlock{ + ID: id.NewULID(), + Domain: domain, + CreatedByAccountID: permSub.CreatedByAccount.ID, + CreatedByAccount: permSub.CreatedByAccount, + PrivateComment: permSub.URI, + PublicComment: wantedPerm.GetPublicComment(), + Obfuscate: wantedPerm.GetObfuscate(), + SubscriptionID: permSub.ID, + } + insertF = func() error { return s.state.DB.CreateDomainBlock(ctx, domainBlock) } + + action = >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionSuspend, + AccountID: permSub.CreatedByAccountID, + } + actionF = s.state.Actions.DomainBlockF(action.ID, domainBlock) + + } else { + // Prepare to insert + process an allow. + domainAllow := >smodel.DomainAllow{ + ID: id.NewULID(), + Domain: domain, + CreatedByAccountID: permSub.CreatedByAccount.ID, + CreatedByAccount: permSub.CreatedByAccount, + PrivateComment: permSub.URI, + PublicComment: wantedPerm.GetPublicComment(), + Obfuscate: wantedPerm.GetObfuscate(), + SubscriptionID: permSub.ID, + } + insertF = func() error { return s.state.DB.CreateDomainAllow(ctx, domainAllow) } + + action = >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: permSub.CreatedByAccountID, + } + actionF = s.state.Actions.DomainAllowF(action.ID, domainAllow) + } + + // Insert the new perm in the db. + if err = insertF(); err != nil { + // Couldn't insert wanted perm, + // don't process side effects. + break + } + + // Run admin action to process + // side effects of permission. + err = s.state.Actions.Run(ctx, action, actionF) + + case existingPerm.GetSubscriptionID() != "" || *permSub.AdoptOrphans: + // Perm exists but we should adopt/take + // it by copying over desired fields. + existingPerm.SetCreatedByAccountID(wantedPerm.GetCreatedByAccountID()) + existingPerm.SetCreatedByAccount(wantedPerm.GetCreatedByAccount()) + existingPerm.SetSubscriptionID(permSub.ID) + existingPerm.SetObfuscate(wantedPerm.GetObfuscate()) + existingPerm.SetPrivateComment(wantedPerm.GetPrivateComment()) + existingPerm.SetPublicComment(wantedPerm.GetPublicComment()) + + switch p := existingPerm.(type) { + case *gtsmodel.DomainBlock: + err = s.state.DB.UpdateDomainBlock(ctx, p) + case *gtsmodel.DomainAllow: + err = s.state.DB.UpdateDomainAllow(ctx, p) + } + + default: + // Perm exists but we should leave it alone. + l.Debug("domain is covered by a higher-priority subscription, skipping") + } + + if err != nil && !errors.Is(err, db.ErrAlreadyExists) { + // Proper db error. + return created, err + } + + created = true + return created, nil +} + +func (s *Subscriptions) permsFromCSV( + l log.Entry, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + // Read body into memory as slice of CSV records. + records, err := csv.NewReader(body).ReadAll() + + // Whatever happened, we're + // done with the body now. + body.Close() + + // Check if error reading body. + if err != nil { + return nil, gtserror.NewfAt(3, "error decoding into csv: %w", err) + } + + // Make sure we actually + // have some records. + if len(records) == 0 { + return nil, nil + } + + // Validate column headers. + columnHeaders := records[0] + if !slices.Equal( + columnHeaders, + []string{ + "#domain", + "#severity", + "#reject_media", + "#reject_reports", + "#public_comment", + "#obfuscate", + }, + ) { + return nil, gtserror.Newf( + "unexpected column headers in csv: %+v", + columnHeaders, + ) + } + + // Trim off column headers + // now they're validated. + records = records[1:] + + // Convert records to permissions slice. + perms := make([]gtsmodel.DomainPermission, 0, len(records)) + for _, record := range records { + if len(record) != 6 { + l.Warnf("skipping invalid-length record: %+v", record) + continue + } + + var ( + domainRaw = record[0] + severity = record[1] + publicComment = record[4] + obfuscate, err = strconv.ParseBool(record[5]) + ) + + if severity != "suspend" { + l.Warnf("skipping non-suspend record: %+v", record) + continue + } + + if err != nil { + l.Warnf("couldn't parse obfuscate field of record: %+v", record) + continue + } + + // Normalize + validate domain. + domain, err := validateDomain(domainRaw) + if err != nil { + l.Warnf("skipping invalid domain %s: %+v", domainRaw, err) + continue + } + + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + if permType == gtsmodel.DomainPermissionBlock { + perm = >smodel.DomainBlock{Domain: domain} + } else { + perm = >smodel.DomainAllow{Domain: domain} + } + + // Set remaining fields. + perm.SetPublicComment(publicComment) + perm.SetObfuscate(&obfuscate) + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +} + +func (s *Subscriptions) permsFromJSON( + l log.Entry, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + var ( + dec = json.NewDecoder(body) + apiPerms = make([]*apimodel.DomainPermission, 0) + ) + + // Read body into memory as + // slice of domain permissions. + if err := dec.Decode(&apiPerms); err != nil { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "error decoding into json: %w", err) + } + + // Perform a secondary decode just to ensure we drained the + // entirety of the data source. Error indicates either extra + // trailing garbage, or multiple JSON values (invalid data). + if err := dec.Decode(&struct{}{}); err != io.EOF { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "data remaining after json") + } + + // Done with body. + _ = body.Close() + + // Convert apimodel perms to barebones internal perms. + perms := make([]gtsmodel.DomainPermission, 0, len(apiPerms)) + for _, apiPerm := range apiPerms { + + // Normalize + validate domain. + domainRaw := apiPerm.Domain.Domain + domain, err := validateDomain(domainRaw) + if err != nil { + l.Warnf("skipping invalid domain %s: %+v", domainRaw, err) + continue + } + + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + if permType == gtsmodel.DomainPermissionBlock { + perm = >smodel.DomainBlock{Domain: domain} + } else { + perm = >smodel.DomainAllow{Domain: domain} + } + + // Set remaining fields. + perm.SetPublicComment(apiPerm.PublicComment) + perm.SetObfuscate(&apiPerm.Obfuscate) + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +} + +func (s *Subscriptions) permsFromPlain( + l log.Entry, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + // Read body into memory as bytes. + b, err := io.ReadAll(body) + + // Whatever happened, we're + // done with the body now. + body.Close() + + // Check if error reading body. + if err != nil { + return nil, gtserror.NewfAt(3, "error decoding into plain: %w", err) + } + + // Coerce to newline-separated list of domains. + domains := strings.Split(string(b), "\n") + + // Convert raw domains to permissions. + perms := make([]gtsmodel.DomainPermission, 0, len(domains)) + for _, domainRaw := range domains { + + // Normalize + validate domain. + domain, err := validateDomain(domainRaw) + if err != nil { + l.Warnf("skipping invalid domain %s: %+v", domainRaw, err) + continue + } + + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + if permType == gtsmodel.DomainPermissionBlock { + perm = >smodel.DomainBlock{Domain: domain} + } else { + perm = >smodel.DomainAllow{Domain: domain} + } + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +} + +func validateDomain(domain string) (string, error) { + // Basic validation. + if _, ok := dns.IsDomainName(domain); !ok { + err := fmt.Errorf("invalid domain name") + return "", err + } + + // Convert to punycode. + domain, err := util.Punify(domain) + if err != nil { + err := fmt.Errorf("could not punify domain: %w", err) + return "", err + } + + // Check for invalid characters + // after the punification process. + if strings.ContainsAny(domain, "*, \n") { + err := fmt.Errorf("invalid char(s) in domain") + return "", err + } + + return domain, nil +} + +func (s *Subscriptions) existingCovered( + ctx context.Context, + permType gtsmodel.DomainPermissionType, + domain string, + higherPrios []*gtsmodel.DomainPermissionSubscription, +) ( + existingPerm gtsmodel.DomainPermission, + covered bool, + err error, +) { + // Check for existing permission of appropriate type. + var dbErr error + if permType == gtsmodel.DomainPermissionBlock { + existingPerm, dbErr = s.state.DB.GetDomainBlock(ctx, domain) + } else { + existingPerm, dbErr = s.state.DB.GetDomainAllow(ctx, domain) + } + if dbErr != nil && !errors.Is(dbErr, db.ErrNoEntries) { + // Real db error. + err = dbErr + return + } + + if util.IsNil(existingPerm) { + // Can't be covered if + // no existing perm. + return + } + + subscriptionID := existingPerm.GetSubscriptionID() + if subscriptionID == "" { + // Can't be covered if + // no subscription ID. + return + } + + // Covered if subscription ID is in the slice + // of higher-priority permission subscriptions. + covered = slices.ContainsFunc( + higherPrios, + func(permSub *gtsmodel.DomainPermissionSubscription) bool { + return permSub.ID == subscriptionID + }, + ) + + return +} diff --git a/internal/subscriptions/subscriptions.go b/internal/subscriptions/subscriptions.go new file mode 100644 index 000000000..3826cf185 --- /dev/null +++ b/internal/subscriptions/subscriptions.go @@ -0,0 +1,42 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subscriptions + +import ( + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" +) + +type Subscriptions struct { + state *state.State + transportController transport.Controller + tc *typeutils.Converter +} + +func New( + state *state.State, + transportController transport.Controller, + tc *typeutils.Converter, +) *Subscriptions { + return &Subscriptions{ + state: state, + transportController: transportController, + tc: tc, + } +} diff --git a/internal/subscriptions/subscriptions_test.go b/internal/subscriptions/subscriptions_test.go new file mode 100644 index 000000000..643124b15 --- /dev/null +++ b/internal/subscriptions/subscriptions_test.go @@ -0,0 +1,496 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subscriptions_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" + "github.com/superseriousbusiness/gotosocial/internal/util" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +const ( + rMediaPath = "../../testrig/media" + rTemplatePath = "../../web/template" +) + +type SubscriptionsTestSuite struct { + suite.Suite + + testAccounts map[string]*gtsmodel.Account +} + +func (suite *SubscriptionsTestSuite) SetupSuite() { + testrig.InitTestConfig() + testrig.InitTestLog() + suite.testAccounts = testrig.NewTestAccounts() +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksCSV() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a CSV list of baddies. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now have blocks for + // each domain on the subscribed list. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + var ( + perm gtsmodel.DomainPermission + err error + ) + if !testrig.WaitFor(func() bool { + perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain) + return err == nil + }) { + suite.FailNowf("", "timed out waiting for domain %s", domain) + } + + suite.Equal(testSubscription.ID, perm.GetSubscriptionID()) + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("bigbums6969", permSub.ETag) + suite.EqualValues(3, permSub.Count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksJSON() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a JSON list of baddies. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.json", + ContentType: gtsmodel.DomainPermSubContentTypeJSON, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now have blocks for + // each domain on the subscribed list. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + var ( + perm gtsmodel.DomainPermission + err error + ) + if !testrig.WaitFor(func() bool { + perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain) + return err == nil + }) { + suite.FailNowf("", "timed out waiting for domain %s", domain) + } + + suite.Equal(testSubscription.ID, perm.GetSubscriptionID()) + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("don't modify me daddy", permSub.ETag) + suite.EqualValues(3, permSub.Count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksPlain() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a plain list of baddies. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.txt", + ContentType: gtsmodel.DomainPermSubContentTypePlain, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now have blocks for + // each domain on the subscribed list. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + var ( + perm gtsmodel.DomainPermission + err error + ) + if !testrig.WaitFor(func() bool { + perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain) + return err == nil + }) { + suite.FailNowf("", "timed out waiting for domain %s", domain) + } + + suite.Equal(testSubscription.ID, perm.GetSubscriptionID()) + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("this is a legit etag i swear", permSub.ETag) + suite.EqualValues(3, permSub.Count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksCSVETag() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a CSV list of baddies. + // Include the ETag so it gets sent with the request. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + ETag: "bigbums6969", + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // We should now NOT have blocks for the domains + // on the list, as the remote will have returned + // 304, indicating we should do nothing. + for _, domain := range []string{ + "bumfaces.net", + "peepee.poopoo", + "nothanks.com", + } { + _, err := testStructs.State.DB.GetDomainBlock(ctx, domain) + if !errors.Is(err, db.ErrNoEntries) { + suite.FailNowf("", "domain perm %s created when it shouldn't be") + } + } + + // The just-fetched perm sub should + // have ETag and count etc set now. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Equal("bigbums6969", permSub.ETag) + suite.EqualValues(0, permSub.Count) // Should be 0. + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocks404() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a CSV list of baddies. + // The endpoint will return a 404 so we can test erroring. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/does_not_exist.csv", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // The just-fetched perm sub should have an error set on it. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.EqualValues(0, permSub.Count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.Zero(permSub.SuccessfullyFetchedAt) + suite.Equal(`DereferenceDomainPermissions: GET request to https://lists.example.org/does_not_exist.csv failed: status="" body="{"error":"not found"}"`, permSub.Error) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypeCSV() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a plaintext list of baddies, + // but try to parse as CSV content type (shouldn't work). + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.txt", + ContentType: gtsmodel.DomainPermSubContentTypeCSV, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // The just-fetched perm sub should have an error set on it. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Zero(permSub.Count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.Zero(permSub.SuccessfullyFetchedAt) + suite.Equal(`permsFromCSV: unexpected column headers in csv: [bumfaces.net]`, permSub.Error) +} + +func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypePlain() { + var ( + ctx = context.Background() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // Create a subscription for a plaintext list of baddies, + // but try to parse as CSV content type (shouldn't work). + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(true), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.csv", + ContentType: gtsmodel.DomainPermSubContentTypePlain, + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process all subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // The just-fetched perm sub should have an error set on it. + permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID( + ctx, testSubscription.ID, + ) + if err != nil { + suite.FailNow(err.Error()) + } + + suite.Zero(permSub.Count) + suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute) + suite.Zero(permSub.SuccessfullyFetchedAt) + suite.Equal(`fetch successful but parsed zero usable results`, permSub.Error) +} + +func TestSubscriptionTestSuite(t *testing.T) { + suite.Run(t, new(SubscriptionsTestSuite)) +} diff --git a/internal/transport/derefdomainpermlist.go b/internal/transport/derefdomainpermlist.go new file mode 100644 index 000000000..e4881c2da --- /dev/null +++ b/internal/transport/derefdomainpermlist.go @@ -0,0 +1,121 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transport + +import ( + "context" + "io" + "net/http" + + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type DereferenceDomainPermissionsResp struct { + // Set only if response was 200 OK. + // It's up to the caller to close + // this when they're done with it. + Body io.ReadCloser + + // True if response + // was 304 Not Modified. + Unmodified bool + + // May be set + // if 200 or 304. + ETag string +} + +func (t *transport) DereferenceDomainPermissions( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + force bool, +) (*DereferenceDomainPermissionsResp, error) { + // Prepare new HTTP request to endpoint + req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil) + if err != nil { + return nil, err + } + + // Set basic auth header if necessary. + if permSub.FetchUsername != "" || permSub.FetchPassword != "" { + req.SetBasicAuth(permSub.FetchUsername, permSub.FetchPassword) + } + + // Set relevant Accept headers. + // Allow fallback in case target doesn't + // negotiate content type correctly. + req.Header.Add("Accept-Charset", "utf-8") + req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*") + + // If force is true, we want to skip setting Cache + // headers so that we definitely don't get a 304 back. + if !force { + // If we've successfully fetched this list + // before, set If-Modified-Since to last + // success to make the request conditional. + // + // See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since + if !permSub.SuccessfullyFetchedAt.IsZero() { + timeStr := permSub.SuccessfullyFetchedAt.Format(http.TimeFormat) + req.Header.Add("If-Modified-Since", timeStr) + } + + // If we've got an ETag stored for this list, set + // If-None-Match to make the request conditional. + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#caching_of_unchanged_resources. + if len(permSub.ETag) != 0 { + req.Header.Add("If-None-Match", permSub.ETag) + } + } + + // Perform the HTTP request + rsp, err := t.GET(req) + if err != nil { + return nil, err + } + + // If we have an unexpected / error response, + // wrap + return as error. This will also drain + // and close the response body for us. + if rsp.StatusCode != http.StatusOK && + rsp.StatusCode != http.StatusNotModified { + err := gtserror.NewFromResponse(rsp) + return nil, err + } + + // Check already if we were given an ETag + // we can use, as ETag is often returned + // even on 304 Not Modified responses. + permsResp := &DereferenceDomainPermissionsResp{ + ETag: rsp.Header.Get("Etag"), + } + + if rsp.StatusCode == http.StatusNotModified { + // Nothing has changed on the remote side + // since we last fetched, so there's nothing + // to do and we don't need to read the body. + rsp.Body.Close() + permsResp.Unmodified = true + } else { + // Return the live body to the caller. + permsResp.Body = rsp.Body + } + + return permsResp, nil +} diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 7f7e985fc..45d43ff18 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -78,6 +78,20 @@ type Transport interface { // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) + // DereferenceDomainPermissions dereferences the + // permissions list present at the given permSub's URI. + // + // If "force", then If-Modified-Since and If-None-Match + // headers will *NOT* be sent with the outgoing request. + // + // If err == nil and Unmodified == false, then it's up + // to the caller to close the returned io.ReadCloser. + DereferenceDomainPermissions( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + force bool, + ) (*DereferenceDomainPermissionsResp, error) + // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error) } diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go index 3a884d53f..b43425eaf 100644 --- a/internal/transport/transport_test.go +++ b/internal/transport/transport_test.go @@ -21,6 +21,7 @@ "context" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -74,6 +75,7 @@ func (suite *TransportTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go index 0676bea1b..904b96f40 100644 --- a/internal/typeutils/converter_test.go +++ b/internal/typeutils/converter_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -499,6 +500,7 @@ func (suite *TypeUtilsTestSuite) SetupTest() { suite.db = testrig.NewTestDB(&suite.state) suite.state.DB = suite.db + suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers) storage := testrig.NewInMemoryStorage() suite.state.Storage = storage diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 6f3bed11b..4aa57318e 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -2121,7 +2121,9 @@ func (c *Converter) DomainPermToAPIDomainPerm( domainPerm.PrivateComment = d.GetPrivateComment() domainPerm.SubscriptionID = d.GetSubscriptionID() domainPerm.CreatedBy = d.GetCreatedByAccountID() - domainPerm.CreatedAt = util.FormatISO8601(d.GetCreatedAt()) + if createdAt := d.GetCreatedAt(); !createdAt.IsZero() { + domainPerm.CreatedAt = util.FormatISO8601(createdAt) + } // If this is a draft, also add the permission type. if _, ok := d.(*gtsmodel.DomainPermissionDraft); ok { diff --git a/testrig/config.go b/testrig/config.go index 673ed46b6..0a957a831 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -99,6 +99,8 @@ func testDefaults() config.Configuration { TagStr: "en-gb", }, }, + InstanceSubscriptionsProcessFrom: "23:00", // 11pm, + InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day. AccountsRegistrationOpen: true, AccountsReasonRequired: true, diff --git a/testrig/processor.go b/testrig/processor.go index e098de33a..bbb8d9d1d 100644 --- a/testrig/processor.go +++ b/testrig/processor.go @@ -26,15 +26,27 @@ "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) // NewTestProcessor returns a Processor suitable for testing purposes. // The passed in state will have its worker functions set appropriately, // but the state will not be initialized. -func NewTestProcessor(state *state.State, federator *federation.Federator, emailSender email.Sender, mediaManager *media.Manager) *processing.Processor { +func NewTestProcessor( + state *state.State, + federator *federation.Federator, + emailSender email.Sender, + mediaManager *media.Manager, +) *processing.Processor { + return processing.NewProcessor( cleaner.New(state), + subscriptions.New( + state, + federator.TransportController(), + typeutils.NewConverter(state), + ), typeutils.NewConverter(state), federator, NewTestOauthServer(state.DB), diff --git a/testrig/teststructs.go b/testrig/teststructs.go index b88e37d55..9677ad219 100644 --- a/testrig/teststructs.go +++ b/testrig/teststructs.go @@ -18,6 +18,7 @@ package testrig import ( + "github.com/superseriousbusiness/gotosocial/internal/actions" "github.com/superseriousbusiness/gotosocial/internal/cleaner" "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" @@ -25,6 +26,8 @@ "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/processing/common" "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/subscriptions" + "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -38,12 +41,13 @@ // and worker queues, which was causing issues // when running all tests at once. type TestStructs struct { - State *state.State - Common *common.Processor - Processor *processing.Processor - HTTPClient *MockHTTPClient - TypeConverter *typeutils.Converter - EmailSender email.Sender + State *state.State + Common *common.Processor + Processor *processing.Processor + HTTPClient *MockHTTPClient + TypeConverter *typeutils.Converter + EmailSender email.Sender + TransportController transport.Controller } func SetupTestStructs( @@ -56,6 +60,7 @@ func SetupTestStructs( db := NewTestDB(&state) state.DB = db + state.Actions = actions.New(db, &state.Workers) storage := NewInMemoryStorage() state.Storage = storage @@ -89,6 +94,7 @@ func SetupTestStructs( processor := processing.NewProcessor( cleaner.New(&state), + subscriptions.New(&state, transportController, typeconverter), typeconverter, federator, oauthServer, @@ -105,12 +111,13 @@ func SetupTestStructs( StandardStorageSetup(storage, rMediaPath) return &TestStructs{ - State: &state, - Common: &common, - Processor: processor, - HTTPClient: httpClient, - TypeConverter: typeconverter, - EmailSender: emailSender, + State: &state, + Common: &common, + Processor: processor, + HTTPClient: httpClient, + TypeConverter: typeconverter, + EmailSender: emailSender, + TransportController: transportController, } } diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go index 385c620db..8faed93ad 100644 --- a/testrig/transportcontroller.go +++ b/testrig/transportcontroller.go @@ -41,6 +41,8 @@ const ( applicationJSON = "application/json" applicationActivityJSON = "application/activity+json" + textCSV = "text/csv" + textPlain = "text/plain" ) // NewTestTransportController returns a test transport controller with the given http client. @@ -101,6 +103,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat responseBytes = []byte(`{"error":"404 not found"}`) responseContentType = applicationJSON responseContentLength = len(responseBytes) + extraHeaders = make(map[string]string, 0) reqURLString = req.URL.String() ) @@ -124,11 +127,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat responseContentType = applicationJSON responseContentLength = len(responseBytes) } else if strings.Contains(reqURLString, ".well-known/webfinger") { - responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req) + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req) } else if strings.Contains(reqURLString, ".weird-webfinger-location/webfinger") { - responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req) + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req) } else if strings.Contains(reqURLString, ".well-known/host-meta") { - responseCode, responseBytes, responseContentType, responseContentLength = HostMetaResponse(req) + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = HostMetaResponse(req) + } else if strings.Contains(reqURLString, "lists.example.org") { + responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = DomainPermissionSubscriptionResponse(req) } else if note, ok := mockHTTPClient.TestRemoteStatuses[reqURLString]; ok { // the request is for a note that we have stored noteI, err := streams.Serialize(note) @@ -239,14 +244,23 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat } log.Debugf(nil, "returning response %s", string(responseBytes)) + reader := bytes.NewReader(responseBytes) readCloser := io.NopCloser(reader) + + header := http.Header{ + "Content-Type": {responseContentType}, + } + for k, v := range extraHeaders { + header.Add(k, v) + } + return &http.Response{ Request: req, StatusCode: responseCode, Body: readCloser, ContentLength: int64(responseContentLength), - Header: http.Header{"Content-Type": {responseContentType}}, + Header: header, }, nil } @@ -261,7 +275,13 @@ func (m *MockHTTPClient) DoSigned(req *http.Request, sign httpclient.SignFunc) ( return m.do(req) } -func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) { +func HostMetaResponse(req *http.Request) ( + responseCode int, + responseBytes []byte, + responseContentType string, + responseContentLength int, + extraHeaders map[string]string, +) { var hm *apimodel.HostMeta if req.URL.String() == "https://misconfigured-instance.com/.well-known/host-meta" { @@ -297,7 +317,13 @@ func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte return } -func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) { +func WebfingerResponse(req *http.Request) ( + responseCode int, + responseBytes []byte, + responseContentType string, + responseContentLength int, + extraHeaders map[string]string, +) { var wfr *apimodel.WellKnownResponse switch req.URL.String() { @@ -410,3 +436,89 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt responseContentLength = len(wfrJSON) return } + +func DomainPermissionSubscriptionResponse(req *http.Request) ( + responseCode int, + responseBytes []byte, + responseContentType string, + responseContentLength int, + extraHeaders map[string]string, +) { + + const ( + csvResp = `#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate +bumfaces.net,suspend,false,false,big jerks,false +peepee.poopoo,suspend,false,false,harassment,false +nothanks.com,suspend,false,false,,false` + csvRespETag = "bigbums6969" + + textResp = `bumfaces.net +peepee.poopoo +nothanks.com` + textRespETag = "this is a legit etag i swear" + + jsonResp = `[ + { + "domain": "bumfaces.net", + "suspended_at": "2020-05-13T13:29:12.000Z", + "public_comment": "big jerks" + }, + { + "domain": "peepee.poopoo", + "suspended_at": "2020-05-13T13:29:12.000Z", + "public_comment": "harassment" + }, + { + "domain": "nothanks.com", + "suspended_at": "2020-05-13T13:29:12.000Z" + } +]` + jsonRespETag = "don't modify me daddy" + ) + + switch req.URL.String() { + case "https://lists.example.org/baddies.csv": + extraHeaders = map[string]string{"ETag": csvRespETag} + if req.Header.Get("If-None-Match") == csvRespETag { + // Cached. + responseCode = http.StatusNotModified + } else { + responseBytes = []byte(csvResp) + responseContentType = textCSV + responseCode = http.StatusOK + } + responseContentLength = len(responseBytes) + + case "https://lists.example.org/baddies.txt": + extraHeaders = map[string]string{"ETag": textRespETag} + if req.Header.Get("If-None-Match") == textRespETag { + // Cached. + responseCode = http.StatusNotModified + } else { + responseBytes = []byte(textResp) + responseContentType = textPlain + responseCode = http.StatusOK + } + responseContentLength = len(responseBytes) + + case "https://lists.example.org/baddies.json": + extraHeaders = map[string]string{"ETag": jsonRespETag} + if req.Header.Get("If-None-Match") == jsonRespETag { + // Cached. + responseCode = http.StatusNotModified + } else { + responseBytes = []byte(jsonResp) + responseContentType = applicationJSON + responseCode = http.StatusOK + } + responseContentLength = len(responseBytes) + + default: + responseCode = http.StatusNotFound + responseBytes = []byte(`{"error":"not found"}`) + responseContentType = applicationJSON + responseContentLength = len(responseBytes) + } + + return +}