extern/gotosocial
1
1
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-01-06 22:28:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

cock

Browse Source
This commit is contained in:
tobi 2025-01-04 15:21:59 +01:00
parent 10945e7809
commit d1f135f0bf
81 changed files with 2890 additions and 601 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/gotosocial/action/admin/account/account.go
View File

@ -40,7 +40,8 @@ func initState(ctx context.Context) (*state.State, error) {
state.Caches.Init()
state.Caches.Start()
// Set the state DB connection
// Only set state DB connection.
// Don't need Actions or Workers for this (yet).
dbConn, err := bundb.NewBunDBService(ctx, &state)
if err != nil {
return nil, fmt.Errorf("error creating dbConn: %w", err)

2
cmd/gotosocial/action/admin/media/list.go
View File

@ -127,6 +127,8 @@ func setupList(ctx context.Context) (*list, error) {
state.Caches.Init()
state.Caches.Start()
// Only set state DB connection.
// Don't need Actions or Workers for this.
dbService, err := bundb.NewBunDBService(ctx, &state)
if err != nil {
return nil, fmt.Errorf("error creating dbservice: %w", err)

4
cmd/gotosocial/action/admin/media/prune/common.go
View File

@ -45,10 +45,12 @@ func setupPrune(ctx context.Context) (*prune, error) {
state.Caches.Start()
// Scheduler is required for the
// claner, but no other workers
// cleaner, but no other workers
// are needed for this CLI action.
state.Workers.StartScheduler()
// Set state DB connection.
// Don't need Actions for this.
dbService, err := bundb.NewBunDBService(ctx, &state)
if err != nil {
return nil, fmt.Errorf("error creating dbservice: %w", err)

4
cmd/gotosocial/action/admin/trans/export.go
View File

@ -33,12 +33,12 @@
var Export action.GTSAction = func(ctx context.Context) error {
var state state.State
// Only set state DB connection.
// Don't need Actions or Workers for this.
dbConn, err := bundb.NewBunDBService(ctx, &state)
if err != nil {
return fmt.Errorf("error creating dbservice: %s", err)
}
// Set the state DB connection
state.DB = dbConn
exporter := trans.NewExporter(dbConn)

4
cmd/gotosocial/action/admin/trans/import.go
View File

@ -33,12 +33,12 @@
var Import action.GTSAction = func(ctx context.Context) error {
var state state.State
// Only set state DB connection.
// Don't need Actions or Workers for this.
dbConn, err := bundb.NewBunDBService(ctx, &state)
if err != nil {
return fmt.Errorf("error creating dbservice: %s", err)
}
// Set the state DB connection
state.DB = dbConn
importer := trans.NewImporter(dbConn)

27
cmd/gotosocial/action/server/server.go
View File

@ -32,6 +32,7 @@
"github.com/KimMachineGun/automemlimit/memlimit"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
@ -44,6 +45,7 @@
"github.com/superseriousbusiness/gotosocial/internal/metrics"
"github.com/superseriousbusiness/gotosocial/internal/middleware"
tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/tracing"
"go.uber.org/automaxprocs/maxprocs"
@ -164,6 +166,10 @@
// Set DB on state.
state.DB = dbService
// Set Actions on state, providing workers to
// Actions as well for triggering side effects.
state.Actions = actions.New(dbService, &state.Workers)
// Ensure necessary database instance prerequisites exist.
if err := dbService.CreateInstanceAccount(ctx); err != nil {
return fmt.Errorf("error creating instance account: %s", err)
@ -283,15 +289,18 @@ func(context.Context, time.Time) {
// Create background cleaner.
cleaner := cleaner.New(state)
// Now schedule background cleaning tasks.
if err := cleaner.ScheduleJobs(); err != nil {
return fmt.Errorf("error scheduling cleaner jobs: %w", err)
}
// Create subscriptions fetcher.
subscriptions := subscriptions.New(
state,
transportController,
typeConverter,
)
// Create the processor using all the
// other services we've created so far.
process = processing.NewProcessor(
cleaner,
subscriptions,
typeConverter,
federator,
oauthServer,
@ -302,6 +311,16 @@ func(context.Context, time.Time) {
intFilter,
)
// Schedule background cleaning tasks.
if err := cleaner.ScheduleJobs(); err != nil {
return fmt.Errorf("error scheduling cleaner jobs: %w", err)
}
// Schedule background subscriptions updating.
if err := subscriptions.ScheduleJobs(); err != nil {
return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
}
// Initialize the specialized workers pools.
state.Workers.Client.Init(messages.ClientMsgIndices())
state.Workers.Federator.Init(messages.FederatorMsgIndices())

15
cmd/gotosocial/action/testrig/testrig.go
View File

@ -47,6 +47,7 @@
"github.com/superseriousbusiness/gotosocial/internal/router"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/tracing"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@ -314,11 +315,23 @@
// Create background cleaner.
cleaner := cleaner.New(state)
// Now schedule background cleaning tasks.
// Schedule background cleaning tasks.
if err := cleaner.ScheduleJobs(); err != nil {
return fmt.Errorf("error scheduling cleaner jobs: %w", err)
}
// Create subscriptions fetcher.
subscriptions := subscriptions.New(
state,
transportController,
typeConverter,
)
// Schedule background subscriptions updating.
if err := subscriptions.ScheduleJobs(); err != nil {
return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
}
// Finally start the main http server!
if err := route.Start(); err != nil {
return fmt.Errorf("error starting router: %w", err)

62
internal/processing/admin/actions.go → internal/actions/actions.go
View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package admin
package actions
import (
"context"
@ -23,11 +23,12 @@
"sync"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/workers"
)
func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode {
@ -42,15 +43,34 @@ func errActionConflict(action *gtsmodel.AdminAction) gtserror.WithCode {
}
type Actions struct {
r map[string]*gtsmodel.AdminAction
state *state.State
// Map of running actions.
running map[string]*gtsmodel.AdminAction
// Not embedded struct,
// to shield from access
// by outside packages.
// Lock for running admin actions.
//
// Not embedded struct, to shield
// from access by outside packages.
m sync.Mutex
// DB for storing, updating,
// deleting admin actions etc.
db db.DB
// Workers for queuing
// admin action side effects.
workers *workers.Workers
}
func New(db db.DB, workers *workers.Workers) *Actions {
return &Actions{
running: make(map[string]*gtsmodel.AdminAction),
db: db,
workers: workers,
}
}
type AdminActionF func(context.Context) gtserror.MultiError
// Run runs the given admin action by executing the supplied function.
//
// Run handles locking, action insertion and updating, so you don't have to!
@ -62,10 +82,10 @@ type Actions struct {
// will be updated on the provided admin action in the database.
func (a *Actions) Run(
ctx context.Context,
action *gtsmodel.AdminAction,
f func(context.Context) gtserror.MultiError,
adminAction *gtsmodel.AdminAction,
f AdminActionF,
) gtserror.WithCode {
actionKey := action.Key()
actionKey := adminAction.Key()
// LOCK THE MAP HERE, since we're
// going to do some operations on it.
@ -73,7 +93,7 @@ func (a *Actions) Run(
// Bail if an action with
// this key is already running.
running, ok := a.r[actionKey]
running, ok := a.running[actionKey]
if ok {
a.m.Unlock()
return errActionConflict(running)
@ -81,7 +101,7 @@ func (a *Actions) Run(
// Action with this key not
// yet running, create it.
if err := a.state.DB.PutAdminAction(ctx, action); err != nil {
if err := a.db.PutAdminAction(ctx, adminAction); err != nil {
err = gtserror.Newf("db error putting admin action %s: %w", actionKey, err)
// Don't store in map
@ -92,7 +112,7 @@ func (a *Actions) Run(
// Action was inserted,
// store in map.
a.r[actionKey] = action
a.running[actionKey] = adminAction
// UNLOCK THE MAP HERE, since
// we're done modifying it for now.
@ -104,22 +124,22 @@ func (a *Actions) Run(
// Run the thing and collect errors.
if errs := f(ctx); errs != nil {
action.Errors = make([]string, 0, len(errs))
adminAction.Errors = make([]string, 0, len(errs))
for _, err := range errs {
action.Errors = append(action.Errors, err.Error())
adminAction.Errors = append(adminAction.Errors, err.Error())
}
}
// Action is no longer running:
// remove from running map.
a.m.Lock()
delete(a.r, actionKey)
delete(a.running, actionKey)
a.m.Unlock()
// Mark as completed in the db,
// storing errors for later review.
action.CompletedAt = time.Now()
if err := a.state.DB.UpdateAdminAction(ctx, action, "completed_at", "errors"); err != nil {
adminAction.CompletedAt = time.Now()
if err := a.db.UpdateAdminAction(ctx, adminAction, "completed_at", "errors"); err != nil {
log.Errorf(ctx, "db error marking action %s as completed: %q", actionKey, err)
}
}()
@ -135,8 +155,8 @@ func (a *Actions) GetRunning() []*gtsmodel.AdminAction {
defer a.m.Unlock()
// Assemble all currently running actions.
running := make([]*gtsmodel.AdminAction, 0, len(a.r))
for _, action := range a.r {
running := make([]*gtsmodel.AdminAction, 0, len(a.running))
for _, action := range a.running {
running = append(running, action)
}
@ -166,5 +186,5 @@ func (a *Actions) TotalRunning() int {
a.m.Lock()
defer a.m.Unlock()
return len(a.r)
return len(a.running)
}

42
internal/processing/admin/actions_test.go → internal/actions/actions_test.go
View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package admin_test
package actions_test
import (
"context"
@ -32,12 +32,26 @@
"github.com/superseriousbusiness/gotosocial/testrig"
)
const (
rMediaPath = "../../testrig/media"
rTemplatePath = "../../web/template"
)
type ActionsTestSuite struct {
AdminStandardTestSuite
suite.Suite
}
func (suite *ActionsTestSuite) SetupSuite() {
testrig.InitTestConfig()
testrig.InitTestLog()
}
func (suite *ActionsTestSuite) TestActionOverlap() {
ctx := context.Background()
var (
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
ctx = context.Background()
)
defer testrig.TearDownTestStructs(testStructs)
// Suspend account.
action1 := &gtsmodel.AdminAction{
@ -61,7 +75,7 @@ func (suite *ActionsTestSuite) TestActionOverlap() {
key2 := action2.Key()
suite.Equal("account/01H90S1CXQ97J9625C5YBXZWGT", key2)
errWithCode := suite.adminProcessor.Actions().Run(
errWithCode := testStructs.State.Actions.Run(
ctx,
action1,
func(ctx context.Context) gtserror.MultiError {
@ -74,7 +88,7 @@ func(ctx context.Context) gtserror.MultiError {
// While first action is sleeping, try to
// process another with the same key.
errWithCode = suite.adminProcessor.Actions().Run(
errWithCode = testStructs.State.Actions.Run(
ctx,
action2,
func(ctx context.Context) gtserror.MultiError {
@ -90,13 +104,13 @@ func(ctx context.Context) gtserror.MultiError {
// Wait for action to finish.
if !testrig.WaitFor(func() bool {
return suite.adminProcessor.Actions().TotalRunning() == 0
return testStructs.State.Actions.TotalRunning() == 0
}) {
suite.FailNow("timed out waiting for admin action(s) to finish")
}
// Try again.
errWithCode = suite.adminProcessor.Actions().Run(
errWithCode = testStructs.State.Actions.Run(
ctx,
action2,
func(ctx context.Context) gtserror.MultiError {
@ -107,14 +121,18 @@ func(ctx context.Context) gtserror.MultiError {
// Wait for action to finish.
if !testrig.WaitFor(func() bool {
return suite.adminProcessor.Actions().TotalRunning() == 0
return testStructs.State.Actions.TotalRunning() == 0
}) {
suite.FailNow("timed out waiting for admin action(s) to finish")
}
}
func (suite *ActionsTestSuite) TestActionWithErrors() {
ctx := context.Background()
var (
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
ctx = context.Background()
)
defer testrig.TearDownTestStructs(testStructs)
// Suspend a domain.
action := &gtsmodel.AdminAction{
@ -125,7 +143,7 @@ func (suite *ActionsTestSuite) TestActionWithErrors() {
AccountID: "01H90S1ZZXP4N74H4A9RVW1MRP",
}
errWithCode := suite.adminProcessor.Actions().Run(
errWithCode := testStructs.State.Actions.Run(
ctx,
action,
func(ctx context.Context) gtserror.MultiError {
@ -140,13 +158,13 @@ func(ctx context.Context) gtserror.MultiError {
// Wait for action to finish.
if !testrig.WaitFor(func() bool {
return suite.adminProcessor.Actions().TotalRunning() == 0
return testStructs.State.Actions.TotalRunning() == 0
}) {
suite.FailNow("timed out waiting for admin action(s) to finish")
}
// Get action from the db.
dbAction, err := suite.db.GetAdminAction(ctx, action.ID)
dbAction, err := testStructs.State.DB.GetAdminAction(ctx, action.ID)
if err != nil {
suite.FailNow(err.Error())
}

51
internal/actions/domainkeys.go Normal file
View File

@ -0,0 +1,51 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package actions
import (
"context"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (a *Actions) DomainKeysExpireF(domain string) AdminActionF {
return func(ctx context.Context) gtserror.MultiError {
var (
expiresAt = time.Now()
errs gtserror.MultiError
)
// For each account on this domain, expire
// the public key and update the account.
if err := a.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) {
account.PublicKeyExpiresAt = expiresAt
if err := a.db.UpdateAccount(ctx,
account,
"public_key_expires_at",
); err != nil {
errs.Appendf("db error updating account: %w", err)
}
}); err != nil {
errs.Appendf("db error ranging through accounts: %w", err)
}
return errs
}
}

387
internal/actions/domainperms.go Normal file
View File

@ -0,0 +1,387 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package actions
import (
"context"
"errors"
"time"
"codeberg.org/gruf/go-kv"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/messages"
)
// Returns an AdminActionF for
// domain allow side effects.
func (a *Actions) DomainAllowF(
actionID string,
domainAllow *gtsmodel.DomainAllow,
) AdminActionF {
return func(ctx context.Context) gtserror.MultiError {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"action", "allow"},
{"actionID", actionID},
{"domain", domainAllow.Domain},
}...)
// Log start + finish.
l.Info("processing side effects")
errs := a.domainAllowSideEffects(ctx, domainAllow)
l.Info("finished processing side effects")
return errs
}
}
func (a *Actions) domainAllowSideEffects(
ctx context.Context,
allow *gtsmodel.DomainAllow,
) gtserror.MultiError {
if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
// We're running in allowlist mode,
// so there are no side effects to
// process here.
return nil
}
// We're running in blocklist mode or
// some similar mode which necessitates
// domain allow side effects if a block
// was in place when the allow was created.
//
// So, check if there's a block.
block, err := a.db.GetDomainBlock(ctx, allow.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs := gtserror.NewMultiError(1)
errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
return errs
}
if block == nil {
// No block?
// No problem!
return nil
}
// There was a block, over which the new
// allow ought to take precedence. To account
// for this, just run side effects as though
// the domain was being unblocked, while
// leaving the existing block in place.
//
// Any accounts that were suspended by
// the block will be unsuspended and be
// able to interact with the instance again.
return a.domainUnblockSideEffects(ctx, block)
}
// Returns an AdminActionF for
// domain unallow side effects.
func (a *Actions) DomainUnallowF(
actionID string,
domainAllow *gtsmodel.DomainAllow,
) AdminActionF {
return func(ctx context.Context) gtserror.MultiError {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"action", "unallow"},
{"actionID", actionID},
{"domain", domainAllow.Domain},
}...)
// Log start + finish.
l.Info("processing side effects")
errs := a.domainUnallowSideEffects(ctx, domainAllow)
l.Info("finished processing side effects")
return errs
}
}
func (a *Actions) domainUnallowSideEffects(
ctx context.Context,
allow *gtsmodel.DomainAllow,
) gtserror.MultiError {
if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
// We're running in allowlist mode,
// so there are no side effects to
// process here.
return nil
}
// We're running in blocklist mode or
// some similar mode which necessitates
// domain allow side effects if a block
// was in place when the allow was removed.
//
// So, check if there's a block.
block, err := a.db.GetDomainBlock(ctx, allow.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs := gtserror.NewMultiError(1)
errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
return errs
}
if block == nil {
// No block?
// No problem!
return nil
}
// There was a block, over which the previous
// allow was taking precedence. Now that the
// allow has been removed, we should put the
// side effects of the block back in place.
//
// To do this, process the block side effects
// again as though the block were freshly
// created. This will mark all accounts from
// the blocked domain as suspended, and clean
// up their follows/following, media, etc.
return a.domainBlockSideEffects(ctx, block)
}
func (a *Actions) DomainBlockF(
actionID string,
domainBlock *gtsmodel.DomainBlock,
) AdminActionF {
return func(ctx context.Context) gtserror.MultiError {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"action", "block"},
{"actionID", actionID},
{"domain", domainBlock.Domain},
}...)
skip, err := a.skipBlockSideEffects(ctx, domainBlock.Domain)
if err != nil {
return err
}
if skip != "" {
l.Infof("skipping side effects: %s", skip)
return nil
}
l.Info("processing side effects")
errs := a.domainBlockSideEffects(ctx, domainBlock)
l.Info("finished processing side effects")
return errs
}
}
// domainBlockSideEffects processes the side effects of a domain block:
//
// 1. Strip most info away from the instance entry for the domain.
// 2. Pass each account from the domain to the processor for deletion.
//
// It should be called asynchronously, since it can take a while when
// there are many accounts present on the given domain.
func (a *Actions) domainBlockSideEffects(
ctx context.Context,
block *gtsmodel.DomainBlock,
) gtserror.MultiError {
var errs gtserror.MultiError
// If we have an instance entry for this domain,
// update it with the new block ID and clear all fields
instance, err := a.db.GetInstance(ctx, block.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs.Appendf("db error getting instance %s: %w", block.Domain, err)
return errs
}
if instance != nil {
// We had an entry for this domain.
columns := stubbifyInstance(instance, block.ID)
if err := a.db.UpdateInstance(ctx, instance, columns...); err != nil {
errs.Appendf("db error updating instance: %w", err)
return errs
}
}
// For each account that belongs to this domain,
// process an account delete message to remove
// that account's posts, media, etc.
if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
if err := a.workers.Client.Process(ctx, &messages.FromClientAPI{
APObjectType: ap.ActorPerson,
APActivityType: ap.ActivityDelete,
GTSModel: block,
Origin: account,
Target: account,
}); err != nil {
errs.Append(err)
}
}); err != nil {
errs.Appendf("db error ranging through accounts: %w", err)
}
return errs
}
func (a *Actions) DomainUnblockF(
actionID string,
domainBlock *gtsmodel.DomainBlock,
) AdminActionF {
return func(ctx context.Context) gtserror.MultiError {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"action", "unblock"},
{"actionID", actionID},
{"domain", domainBlock.Domain},
}...)
l.Info("processing side effects")
errs := a.domainUnblockSideEffects(ctx, domainBlock)
l.Info("finished processing side effects")
return errs
}
}
// domainUnblockSideEffects processes the side effects of undoing a
// domain block:
//
// 1. Mark instance entry as no longer suspended.
// 2. Mark each account from the domain as no longer suspended, if the
// suspension origin corresponds to the ID of the provided domain block.
//
// It should be called asynchronously, since it can take a while when
// there are many accounts present on the given domain.
func (a *Actions) domainUnblockSideEffects(
ctx context.Context,
block *gtsmodel.DomainBlock,
) gtserror.MultiError {
var errs gtserror.MultiError
// Update instance entry for this domain, if we have it.
instance, err := a.db.GetInstance(ctx, block.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs.Appendf("db error getting instance %s: %w", block.Domain, err)
}
if instance != nil {
// We had an entry, update it to signal
// that it's no longer suspended.
instance.SuspendedAt = time.Time{}
instance.DomainBlockID = ""
if err := a.db.UpdateInstance(
ctx,
instance,
"suspended_at",
"domain_block_id",
); err != nil {
errs.Appendf("db error updating instance: %w", err)
return errs
}
}
// Unsuspend all accounts whose suspension origin was this domain block.
if err := a.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() {
// Account wasn't suspended, nothing to do.
return
}
if account.SuspensionOrigin != block.ID {
// Account was suspended, but not by
// this domain block, leave it alone.
return
}
// Account was suspended by this domain
// block, mark it as unsuspended.
account.SuspendedAt = time.Time{}
account.SuspensionOrigin = ""
if err := a.db.UpdateAccount(
ctx,
account,
"suspended_at",
"suspension_origin",
); err != nil {
errs.Appendf("db error updating account %s: %w", account.Username, err)
}
}); err != nil {
errs.Appendf("db error ranging through accounts: %w", err)
}
return errs
}
// skipBlockSideEffects checks if side effects of block creation
// should be skipped for the given domain, taking account of
// instance federation mode, and existence of any allows
// which ought to "shield" this domain from being blocked.
//
// If the caller should skip, the returned string will be non-zero
// and will be set to a reason why side effects should be skipped.
//
// - blocklist mode + allow exists: "..." (skip)
// - blocklist mode + no allow: "" (don't skip)
// - allowlist mode + allow exists: "" (don't skip)
// - allowlist mode + no allow: "" (don't skip)
func (a *Actions) skipBlockSideEffects(
ctx context.Context,
domain string,
) (string, gtserror.MultiError) {
var (
skip string // Assume "" (don't skip).
errs gtserror.MultiError
)
// Never skip block side effects in allowlist mode.
fediMode := config.GetInstanceFederationMode()
if fediMode == config.InstanceFederationModeAllowlist {
return skip, errs
}
// We know we're in blocklist mode.
//
// We want to skip domain block side
// effects if an allow is already
// in place which overrides the block.
// Check if an explicit allow exists for this domain.
domainAllow, err := a.db.GetDomainAllow(ctx, domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs.Appendf("error getting domain allow: %w", err)
return skip, errs
}
if domainAllow != nil {
skip = "running in blocklist mode, and an explicit allow exists for this domain"
return skip, errs
}
return skip, errs
}

99
internal/actions/util.go Normal file
View File

@ -0,0 +1,99 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package actions
import (
"context"
"errors"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// stubbifyInstance renders the given instance as a stub,
// removing most information from it and marking it as
// suspended.
//
// For caller's convenience, this function returns the db
// names of all columns that are updated by it.
func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string {
instance.Title = ""
instance.SuspendedAt = time.Now()
instance.DomainBlockID = domainBlockID
instance.ShortDescription = ""
instance.Description = ""
instance.Terms = ""
instance.ContactEmail = ""
instance.ContactAccountUsername = ""
instance.ContactAccountID = ""
instance.Version = ""
return []string{
"title",
"suspended_at",
"domain_block_id",
"short_description",
"description",
"terms",
"contact_email",
"contact_account_username",
"contact_account_id",
"version",
}
}
// rangeDomainAccounts iterates through all accounts
// originating from the given domain, and calls the
// provided range function on each account.
//
// If an error is returned while selecting accounts,
// the loop will stop and return the error.
func (a *Actions) rangeDomainAccounts(
ctx context.Context,
domain string,
rangeF func(*gtsmodel.Account),
) error {
var (
limit = 50 // Limit selection to avoid spiking mem/cpu.
maxID string // Start with empty string to select from top.
)
for {
// Get (next) page of accounts.
accounts, err := a.db.GetInstanceAccounts(ctx, domain, maxID, limit)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// Real db error.
return gtserror.Newf("db error getting instance accounts: %w", err)
}
if len(accounts) == 0 {
// No accounts left, we're done.
return nil
}
// Set next max ID for paging down.
maxID = accounts[len(accounts)-1].ID
// Call provided range function.
for _, account := range accounts {
rangeF(account)
}
}
}

2
internal/api/activitypub/emoji/emojiget_test.go
View File

@ -25,6 +25,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/emoji"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -73,6 +74,7 @@ func (suite *EmojiGetTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage
suite.tc = typeutils.NewConverter(&suite.state)

2
internal/api/activitypub/users/user_test.go
View File

@ -20,6 +20,7 @@
import (
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/activitypub/users"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -84,6 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.tc = typeutils.NewConverter(&suite.state)
testrig.StartTimelines(

2
internal/api/auth/auth_test.go
View File

@ -26,6 +26,7 @@
"github.com/gin-contrib/sessions/memstore"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/auth"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -84,6 +85,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage
suite.mediaManager = testrig.NewTestMediaManager(&suite.state)

2
internal/api/client/accounts/account_test.go
View File

@ -25,6 +25,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/accounts"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -85,6 +86,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/admin/admin.go
View File

@ -46,6 +46,7 @@
DomainPermissionSubscriptionsPathWithID = DomainPermissionSubscriptionsPath + "/:" + apiutil.IDKey
DomainPermissionSubscriptionsPreviewPath = DomainPermissionSubscriptionsPath + "/preview"
DomainPermissionSubscriptionRemovePath = DomainPermissionSubscriptionsPathWithID + "/remove"
DomainPermissionSubscriptionTestPath = DomainPermissionSubscriptionsPathWithID + "/test"
DomainKeysExpirePath = BasePath + "/domain_keys_expire"
HeaderAllowsPath = BasePath + "/header_allows"
HeaderAllowsPathWithID = HeaderAllowsPath + "/:" + apiutil.IDKey
@ -129,6 +130,7 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
attachHandler(http.MethodGet, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionGETHandler)
attachHandler(http.MethodPatch, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionPATCHHandler)
attachHandler(http.MethodPost, DomainPermissionSubscriptionRemovePath, m.DomainPermissionSubscriptionRemovePOSTHandler)
attachHandler(http.MethodPost, DomainPermissionSubscriptionTestPath, m.DomainPermissionSubscriptionTestPOSTHandler)
// header filtering administration routes
attachHandler(http.MethodGet, HeaderAllowsPathWithID, m.HeaderFilterAllowGET)

2
internal/api/client/admin/admin_test.go
View File

@ -25,6 +25,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -91,6 +92,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

118
internal/api/client/admin/domainpermissionsubscriptiontest.go Normal file
View File

@ -0,0 +1,118 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package admin
import (
"fmt"
"net/http"
"github.com/gin-gonic/gin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
// DomainPermissionSubscriptionTestPOSTHandler swagger:operation POST /api/v1/admin/domain_permission_subscriptions/{id}/test domainPermissionSubscriptionTest
//
// Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*.
//
// The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message.
//
// This is useful in cases where you want to check that your instance can actually fetch + parse a list.
//
// ---
// tags:
// - admin
//
// produces:
// - application/json
//
// parameters:
// -
// name: id
// required: true
// in: path
// description: ID of the domain permission draft.
// type: string
//
// security:
// - OAuth2 Bearer:
// - admin
//
// responses:
// '200':
// description: >-
// Either an array of domain permissions, OR an error message of the form
// `{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched.
// schema:
// type: array
// items:
// "$ref": "#/definitions/domain"
// '400':
// description: bad request
// '401':
// description: unauthorized
// '403':
// description: forbidden
// '406':
// description: not acceptable
// '409':
// description: conflict
// '500':
// description: internal server error
func (m *Module) DomainPermissionSubscriptionTestPOSTHandler(c *gin.Context) {
authed, err := oauth.Authed(c, true, true, true, true)
if err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
return
}
if !*authed.User.Admin {
err := fmt.Errorf("user %s not an admin", authed.User.ID)
apiutil.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGetV1)
return
}
if authed.Account.IsMoving() {
apiutil.ForbiddenAfterMove(c)
return
}
if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
return
}
id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey))
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
resp, errWithCode := m.processor.Admin().DomainPermissionSubscriptionTest(
c.Request.Context(),
authed.Account,
id,
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
apiutil.JSON(c, http.StatusOK, resp)
}

125
internal/api/client/admin/domainpermissionsubscruptiontest_test.go Normal file
View File

@ -0,0 +1,125 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package admin_test
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
type DomainPermissionSubscriptionTestTestSuite struct {
AdminStandardTestSuite
}
func (suite *DomainPermissionSubscriptionTestTestSuite) TestDomainPermissionSubscriptionTest() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["admin_account"]
permSub = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/baddies.csv",
ContentType: gtsmodel.DomainPermSubContentTypeCSV,
}
)
// Create a subscription for a CSV list of baddies.
err := suite.state.DB.PutDomainPermissionSubscription(ctx, permSub)
if err != nil {
suite.FailNow(err.Error())
}
// Prepare the request to the /test endpoint.
subPath := strings.ReplaceAll(
admin.DomainPermissionSubscriptionTestPath,
":id", permSub.ID,
)
path := "/api" + subPath
recorder := httptest.NewRecorder()
ginCtx := suite.newContext(recorder, http.MethodPost, nil, path, "application/json")
ginCtx.Params = gin.Params{
gin.Param{
Key: apiutil.IDKey,
Value: permSub.ID,
},
}
// Trigger the handler.
suite.adminModule.DomainPermissionSubscriptionTestPOSTHandler(ginCtx)
suite.Equal(http.StatusOK, recorder.Code)
// Read the body back.
b, err := io.ReadAll(recorder.Body)
if err != nil {
suite.FailNow(err.Error())
}
dst := new(bytes.Buffer)
if err := json.Indent(dst, b, "", " "); err != nil {
suite.FailNow(err.Error())
}
// Ensure expected.
suite.Equal(`[
{
"domain": "bumfaces.net",
"public_comment": "big jerks"
},
{
"domain": "peepee.poopoo",
"public_comment": "harassment"
},
{
"domain": "nothanks.com"
}
]`, dst.String())
// No permissions should be created
// since this is a dry run / test.
blocked, err := suite.state.DB.AreDomainsBlocked(
ctx,
[]string{"bumfaces.net", "peepee.poopoo", "nothanks.com"},
)
if err != nil {
suite.FailNow(err.Error())
}
suite.False(blocked)
}
func TestDomainPermissionSubscriptionTestTestSuite(t *testing.T) {
suite.Run(t, &DomainPermissionSubscriptionTestTestSuite{})
}

2
internal/api/client/bookmarks/bookmarks_test.go
View File

@ -28,6 +28,7 @@
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/bookmarks"
"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
@ -95,6 +96,7 @@ func (suite *BookmarkTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/favourites/favourites_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/favourites"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -79,6 +80,7 @@ func (suite *FavouritesStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/filters/v1/filter_test.go
View File

@ -23,6 +23,7 @@
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
filtersV1 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v1"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/filters/v2/filter_test.go
View File

@ -23,6 +23,7 @@
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
filtersV2 "github.com/superseriousbusiness/gotosocial/internal/api/client/filters/v2"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -90,6 +91,7 @@ func (suite *FiltersTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/followedtags/followedtags_test.go
View File

@ -21,6 +21,7 @@
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/followedtags"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -79,6 +80,7 @@ func (suite *FollowedTagsTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/followrequests/followrequest_test.go
View File

@ -24,6 +24,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequests"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -82,6 +83,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/instance/instance_test.go
View File

@ -24,6 +24,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -84,6 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/lists/lists_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/lists"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -85,6 +86,7 @@ func (suite *ListsStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/mutes/mutes_test.go
View File

@ -25,6 +25,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/mutes"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -81,6 +82,7 @@ func (suite *MutesTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/notifications/notifications_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/notifications"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -81,6 +82,7 @@ func (suite *NotificationsTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/polls/polls_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/polls"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -76,6 +77,7 @@ func (suite *PollsStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/reports/reports_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/reports"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -76,6 +77,7 @@ func (suite *ReportsStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/search/search_test.go
View File

@ -24,6 +24,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/search"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -80,6 +81,7 @@ func (suite *SearchStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/statuses/status_test.go
View File

@ -25,6 +25,7 @@
"strings"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/statuses"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -192,6 +193,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/streaming/streaming_test.go
View File

@ -31,6 +31,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/streaming"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -92,6 +93,7 @@ func (suite *StreamingTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/tags/tags_test.go
View File

@ -26,6 +26,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/tags"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
@ -87,6 +88,7 @@ func (suite *TagsTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/client/user/user_test.go
View File

@ -24,6 +24,7 @@
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/client/user"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation"
@ -72,6 +73,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/api/fileserver/fileserver_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -98,6 +99,7 @@ func (suite *FileserverTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")

2
internal/api/wellknown/webfinger/webfinger_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/api/wellknown/webfinger"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -79,6 +80,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.tc = typeutils.NewConverter(&suite.state)
testrig.StartTimelines(

2
internal/api/wellknown/webfinger/webfingerget_test.go
View File

@ -39,6 +39,7 @@
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -90,6 +91,7 @@ func (suite *WebfingerGetTestSuite) funkifyAccountDomain(host string, accountDom
suite.processor = processing.NewProcessor(
cleaner.New(&suite.state),
subscriptions.New(&suite.state, suite.federator.TransportController(), suite.tc),
suite.tc,
suite.federator,
testrig.NewTestOauthServer(suite.db),

2
internal/cleaner/media_test.go
View File

@ -26,6 +26,7 @@
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
@ -67,6 +68,7 @@ func (suite *MediaTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.storage = testrig.NewInMemoryStorage()
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.state.Storage = suite.storage
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

20
internal/config/config.go
View File

@ -78,15 +78,17 @@ type Configuration struct {
WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`
InstanceFederationMode string `name:"instance-federation-mode" usage:"Set instance federation mode."`
InstanceFederationSpamFilter bool `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"`
InstanceExposePeers bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
InstanceExposeSuspended bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
InstanceExposeSuspendedWeb bool `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"`
InstanceExposePublicTimeline bool `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"`
InstanceDeliverToSharedInboxes bool `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."`
InstanceInjectMastodonVersion bool `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"`
InstanceLanguages language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."`
InstanceFederationMode string `name:"instance-federation-mode" usage:"Set instance federation mode."`
InstanceFederationSpamFilter bool `name:"instance-federation-spam-filter" usage:"Enable basic spam filter heuristics for messages coming from other instances, and drop messages identified as spam"`
InstanceExposePeers bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
InstanceExposeSuspended bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
InstanceExposeSuspendedWeb bool `name:"instance-expose-suspended-web" usage:"Expose list of suspended instances as webpage on /about/suspended"`
InstanceExposePublicTimeline bool `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"`
InstanceDeliverToSharedInboxes bool `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."`
InstanceInjectMastodonVersion bool `name:"instance-inject-mastodon-version" usage:"This injects a Mastodon compatible version in /api/v1/instance to help Mastodon clients that use that version for feature detection"`
InstanceLanguages language.Languages `name:"instance-languages" usage:"BCP47 language tags for the instance. Used to indicate the preferred languages of instance residents (in order from most-preferred to least-preferred)."`
InstanceSubscriptionsProcessFrom string `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
InstanceSubscriptionsProcessEvery time.Duration `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."`
AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."`
AccountsReasonRequired bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`

16
internal/config/defaults.go
View File

@ -58,13 +58,15 @@
WebTemplateBaseDir: "./web/template/",
WebAssetBaseDir: "./web/assets/",
InstanceFederationMode: InstanceFederationModeDefault,
InstanceFederationSpamFilter: false,
InstanceExposePeers: false,
InstanceExposeSuspended: false,
InstanceExposeSuspendedWeb: false,
InstanceDeliverToSharedInboxes: true,
InstanceLanguages: make(language.Languages, 0),
InstanceFederationMode: InstanceFederationModeDefault,
InstanceFederationSpamFilter: false,
InstanceExposePeers: false,
InstanceExposeSuspended: false,
InstanceExposeSuspendedWeb: false,
InstanceDeliverToSharedInboxes: true,
InstanceLanguages: make(language.Languages, 0),
InstanceSubscriptionsProcessFrom: "23:00", // 11pm,
InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
AccountsRegistrationOpen: false,
AccountsReasonRequired: true,

2
internal/config/flags.go
View File

@ -90,6 +90,8 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
cmd.Flags().Bool(InstanceExposeSuspendedWebFlag(), cfg.InstanceExposeSuspendedWeb, fieldtag("InstanceExposeSuspendedWeb", "usage"))
cmd.Flags().Bool(InstanceDeliverToSharedInboxesFlag(), cfg.InstanceDeliverToSharedInboxes, fieldtag("InstanceDeliverToSharedInboxes", "usage"))
cmd.Flags().StringSlice(InstanceLanguagesFlag(), cfg.InstanceLanguages.TagStrs(), fieldtag("InstanceLanguages", "usage"))
cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage"))
cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage"))
// Accounts
cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage"))

56
internal/config/helpers.gen.go
View File

@ -1000,6 +1000,62 @@ func GetInstanceLanguages() language.Languages { return global.GetInstanceLangua
// SetInstanceLanguages safely sets the value for global configuration 'InstanceLanguages' field
func SetInstanceLanguages(v language.Languages) { global.SetInstanceLanguages(v) }
// GetInstanceSubscriptionsProcessFrom safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field
func (st *ConfigState) GetInstanceSubscriptionsProcessFrom() (v string) {
st.mutex.RLock()
v = st.config.InstanceSubscriptionsProcessFrom
st.mutex.RUnlock()
return
}
// SetInstanceSubscriptionsProcessFrom safely sets the Configuration value for state's 'InstanceSubscriptionsProcessFrom' field
func (st *ConfigState) SetInstanceSubscriptionsProcessFrom(v string) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.InstanceSubscriptionsProcessFrom = v
st.reloadToViper()
}
// InstanceSubscriptionsProcessFromFlag returns the flag name for the 'InstanceSubscriptionsProcessFrom' field
func InstanceSubscriptionsProcessFromFlag() string { return "instance-subscriptions-process-from" }
// GetInstanceSubscriptionsProcessFrom safely fetches the value for global configuration 'InstanceSubscriptionsProcessFrom' field
func GetInstanceSubscriptionsProcessFrom() string {
return global.GetInstanceSubscriptionsProcessFrom()
}
// SetInstanceSubscriptionsProcessFrom safely sets the value for global configuration 'InstanceSubscriptionsProcessFrom' field
func SetInstanceSubscriptionsProcessFrom(v string) { global.SetInstanceSubscriptionsProcessFrom(v) }
// GetInstanceSubscriptionsProcessEvery safely fetches the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field
func (st *ConfigState) GetInstanceSubscriptionsProcessEvery() (v time.Duration) {
st.mutex.RLock()
v = st.config.InstanceSubscriptionsProcessEvery
st.mutex.RUnlock()
return
}
// SetInstanceSubscriptionsProcessEvery safely sets the Configuration value for state's 'InstanceSubscriptionsProcessEvery' field
func (st *ConfigState) SetInstanceSubscriptionsProcessEvery(v time.Duration) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.InstanceSubscriptionsProcessEvery = v
st.reloadToViper()
}
// InstanceSubscriptionsProcessEveryFlag returns the flag name for the 'InstanceSubscriptionsProcessEvery' field
func InstanceSubscriptionsProcessEveryFlag() string { return "instance-subscriptions-process-every" }
// GetInstanceSubscriptionsProcessEvery safely fetches the value for global configuration 'InstanceSubscriptionsProcessEvery' field
func GetInstanceSubscriptionsProcessEvery() time.Duration {
return global.GetInstanceSubscriptionsProcessEvery()
}
// SetInstanceSubscriptionsProcessEvery safely sets the value for global configuration 'InstanceSubscriptionsProcessEvery' field
func SetInstanceSubscriptionsProcessEvery(v time.Duration) {
global.SetInstanceSubscriptionsProcessEvery(v)
}
// GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field
func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) {
st.mutex.RLock()

2
internal/federation/dereferencing/dereferencer_test.go
View File

@ -20,6 +20,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@ -77,6 +78,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
suite.client = testrig.NewMockHTTPClient(nil, "../../../testrig/media")
suite.storage = testrig.NewInMemoryStorage()
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.state.Storage = suite.storage
visFilter := visibility.NewFilter(&suite.state)

2
internal/federation/federatingdb/federatingdb_test.go
View File

@ -22,6 +22,7 @@
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
@ -91,6 +92,7 @@ func (suite *FederatingDBTestSuite) SetupTest() {
testrig.StandardDBSetup(suite.db, suite.testAccounts)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
}
func (suite *FederatingDBTestSuite) TearDownTest() {

2
internal/media/media_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -53,6 +54,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.storage = testrig.NewInMemoryStorage()
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.state.Storage = suite.storage
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

2
internal/oauth/clientstore_test.go
View File

@ -22,6 +22,7 @@
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/state"
@ -54,6 +55,7 @@ func (suite *PgClientStoreTestSuite) SetupTest() {
testrig.InitTestConfig()
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
testrig.StandardDBSetup(suite.db, nil)
}

2
internal/processing/account/account_test.go
View File

@ -22,6 +22,7 @@
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
@ -93,6 +94,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.tc = typeutils.NewConverter(&suite.state)
testrig.StartTimelines(

2
internal/processing/admin/account_test.go
View File

@ -53,7 +53,7 @@ func (suite *AccountTestSuite) TestAccountActionSuspend() {
// Wait for action to finish.
if !testrig.WaitFor(func() bool {
return suite.adminProcessor.Actions().TotalRunning() == 0
return suite.state.Actions.TotalRunning() == 0
}) {
suite.FailNow("timed out waiting for admin action(s) to finish")
}

2
internal/processing/admin/accountaction.go
View File

@ -68,7 +68,7 @@ func (p *Processor) accountActionSuspend(
) (string, gtserror.WithCode) {
actionID := id.NewULID()
errWithCode := p.actions.Run(
errWithCode := p.state.Actions.Run(
ctx,
&gtsmodel.AdminAction{
ID: actionID,

47
internal/processing/admin/admin.go
View File

@ -21,10 +21,10 @@
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/processing/common"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@ -33,21 +33,14 @@ type Processor struct {
// common processor logic
c *common.Processor
state *state.State
cleaner *cleaner.Cleaner
converter *typeutils.Converter
federator *federation.Federator
media *media.Manager
transport transport.Controller
email email.Sender
// admin Actions currently
// undergoing processing
actions *Actions
}
func (p *Processor) Actions() *Actions {
return p.actions
state *state.State
cleaner *cleaner.Cleaner
subscriptions *subscriptions.Subscriptions
converter *typeutils.Converter
federator *federation.Federator
media *media.Manager
transport transport.Controller
email email.Sender
}
// New returns a new admin processor.
@ -55,6 +48,7 @@ func New(
common *common.Processor,
state *state.State,
cleaner *cleaner.Cleaner,
subscriptions *subscriptions.Subscriptions,
federator *federation.Federator,
converter *typeutils.Converter,
mediaManager *media.Manager,
@ -62,17 +56,14 @@ func New(
emailSender email.Sender,
) Processor {
return Processor{
c: common,
state: state,
cleaner: cleaner,
converter: converter,
federator: federator,
media: mediaManager,
transport: transportController,
email: emailSender,
actions: &Actions{
r: make(map[string]*gtsmodel.AdminAction),
state: state,
},
c: common,
state: state,
cleaner: cleaner,
subscriptions: subscriptions,
converter: converter,
federator: federator,
media: mediaManager,
transport: transportController,
email: emailSender,
}
}

4
internal/processing/admin/admin_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -33,6 +34,7 @@
"github.com/superseriousbusiness/gotosocial/internal/processing/admin"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -89,6 +91,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.tc = typeutils.NewConverter(&suite.state)
testrig.StartTimelines(
@ -109,6 +112,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
suite.processor = processing.NewProcessor(
cleaner.New(&suite.state),
subscriptions.New(&suite.state, suite.transportController, suite.tc),
suite.tc,
suite.federator,
suite.oauthServer,

169
internal/processing/admin/domainallow.go
View File

@ -22,14 +22,11 @@
"errors"
"fmt"
"codeberg.org/gruf/go-kv"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/text"
)
@ -69,84 +66,30 @@ func (p *Processor) createDomainAllow(
}
}
actionID := id.NewULID()
// Run admin action to process
// side effects of allow.
action := &gtsmodel.AdminAction{
ID: id.NewULID(),
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domainAllow.Domain,
Type: gtsmodel.AdminActionUnsuspend,
AccountID: adminAcct.ID,
}
// Process domain allow side
// effects asynchronously.
if errWithCode := p.actions.Run(
if errWithCode := p.state.Actions.Run(
ctx,
&gtsmodel.AdminAction{
ID: actionID,
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domain,
Type: gtsmodel.AdminActionSuspend,
AccountID: adminAcct.ID,
Text: domainAllow.PrivateComment,
},
func(ctx context.Context) gtserror.MultiError {
// Log start + finish.
l := log.WithFields(kv.Fields{
{"domain", domain},
{"actionID", actionID},
}...).WithContext(ctx)
l.Info("processing domain allow side effects")
defer func() { l.Info("finished processing domain allow side effects") }()
return p.domainAllowSideEffects(ctx, domainAllow)
},
action,
p.state.Actions.DomainAllowF(action.ID, domainAllow),
); errWithCode != nil {
return nil, actionID, errWithCode
return nil, action.ID, errWithCode
}
apiDomainAllow, errWithCode := p.apiDomainPerm(ctx, domainAllow, false)
if errWithCode != nil {
return nil, actionID, errWithCode
return nil, action.ID, errWithCode
}
return apiDomainAllow, actionID, nil
}
func (p *Processor) domainAllowSideEffects(
ctx context.Context,
allow *gtsmodel.DomainAllow,
) gtserror.MultiError {
if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
// We're running in allowlist mode,
// so there are no side effects to
// process here.
return nil
}
// We're running in blocklist mode or
// some similar mode which necessitates
// domain allow side effects if a block
// was in place when the allow was created.
//
// So, check if there's a block.
block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs := gtserror.NewMultiError(1)
errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
return errs
}
if block == nil {
// No block?
// No problem!
return nil
}
// There was a block, over which the new
// allow ought to take precedence. To account
// for this, just run side effects as though
// the domain was being unblocked, while
// leaving the existing block in place.
//
// Any accounts that were suspended by
// the block will be unsuspended and be
// able to interact with the instance again.
return p.domainUnblockSideEffects(ctx, block)
return apiDomainAllow, action.ID, nil
}
func (p *Processor) deleteDomainAllow(
@ -179,77 +122,23 @@ func (p *Processor) deleteDomainAllow(
return nil, "", gtserror.NewErrorInternalError(err)
}
actionID := id.NewULID()
// Run admin action to process
// side effects of unallow.
action := &gtsmodel.AdminAction{
ID: id.NewULID(),
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domainAllow.Domain,
Type: gtsmodel.AdminActionUnsuspend,
AccountID: adminAcct.ID,
}
// Process domain unallow side
// effects asynchronously.
if errWithCode := p.actions.Run(
if errWithCode := p.state.Actions.Run(
ctx,
&gtsmodel.AdminAction{
ID: actionID,
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domainAllow.Domain,
Type: gtsmodel.AdminActionUnsuspend,
AccountID: adminAcct.ID,
},
func(ctx context.Context) gtserror.MultiError {
// Log start + finish.
l := log.WithFields(kv.Fields{
{"domain", domainAllow.Domain},
{"actionID", actionID},
}...).WithContext(ctx)
l.Info("processing domain unallow side effects")
defer func() { l.Info("finished processing domain unallow side effects") }()
return p.domainUnallowSideEffects(ctx, domainAllow)
},
action,
p.state.Actions.DomainUnallowF(action.ID, domainAllow),
); errWithCode != nil {
return nil, actionID, errWithCode
return nil, action.ID, errWithCode
}
return apiDomainAllow, actionID, nil
}
func (p *Processor) domainUnallowSideEffects(
ctx context.Context,
allow *gtsmodel.DomainAllow,
) gtserror.MultiError {
if config.GetInstanceFederationMode() == config.InstanceFederationModeAllowlist {
// We're running in allowlist mode,
// so there are no side effects to
// process here.
return nil
}
// We're running in blocklist mode or
// some similar mode which necessitates
// domain allow side effects if a block
// was in place when the allow was removed.
//
// So, check if there's a block.
block, err := p.state.DB.GetDomainBlock(ctx, allow.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs := gtserror.NewMultiError(1)
errs.Appendf("db error getting domain block %s: %w", allow.Domain, err)
return errs
}
if block == nil {
// No block?
// No problem!
return nil
}
// There was a block, over which the previous
// allow was taking precedence. Now that the
// allow has been removed, we should put the
// side effects of the block back in place.
//
// To do this, process the block side effects
// again as though the block were freshly
// created. This will mark all accounts from
// the blocked domain as suspended, and clean
// up their follows/following, media, etc.
return p.domainBlockSideEffects(ctx, block)
return apiDomainAllow, action.ID, nil
}

265
internal/processing/admin/domainblock.go
View File

@ -21,18 +21,12 @@
"context"
"errors"
"fmt"
"time"
"codeberg.org/gruf/go-kv"
"github.com/superseriousbusiness/gotosocial/internal/ap"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/text"
)
@ -72,149 +66,31 @@ func (p *Processor) createDomainBlock(
}
}
actionID := id.NewULID()
// Run admin action to process
// side effects of block.
action := &gtsmodel.AdminAction{
ID: id.NewULID(),
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domain,
Type: gtsmodel.AdminActionSuspend,
AccountID: adminAcct.ID,
Text: domainBlock.PrivateComment,
}
// Process domain block side
// effects asynchronously.
if errWithCode := p.actions.Run(
if errWithCode := p.state.Actions.Run(
ctx,
&gtsmodel.AdminAction{
ID: actionID,
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domain,
Type: gtsmodel.AdminActionSuspend,
AccountID: adminAcct.ID,
Text: domainBlock.PrivateComment,
},
func(ctx context.Context) gtserror.MultiError {
// Log start + finish.
l := log.WithFields(kv.Fields{
{"domain", domain},
{"actionID", actionID},
}...).WithContext(ctx)
skip, err := p.skipBlockSideEffects(ctx, domain)
if err != nil {
return err
}
if skip != "" {
l.Infof("skipping domain block side effects: %s", skip)
return nil
}
l.Info("processing domain block side effects")
defer func() { l.Info("finished processing domain block side effects") }()
return p.domainBlockSideEffects(ctx, domainBlock)
},
action,
p.state.Actions.DomainBlockF(action.ID, domainBlock),
); errWithCode != nil {
return nil, actionID, errWithCode
return nil, action.ID, errWithCode
}
apiDomainBlock, errWithCode := p.apiDomainPerm(ctx, domainBlock, false)
if errWithCode != nil {
return nil, actionID, errWithCode
return nil, action.ID, errWithCode
}
return apiDomainBlock, actionID, nil
}
// skipBlockSideEffects checks if side effects of block creation
// should be skipped for the given domain, taking account of
// instance federation mode, and existence of any allows
// which ought to "shield" this domain from being blocked.
//
// If the caller should skip, the returned string will be non-zero
// and will be set to a reason why side effects should be skipped.
//
// - blocklist mode + allow exists: "..." (skip)
// - blocklist mode + no allow: "" (don't skip)
// - allowlist mode + allow exists: "" (don't skip)
// - allowlist mode + no allow: "" (don't skip)
func (p *Processor) skipBlockSideEffects(
ctx context.Context,
domain string,
) (string, gtserror.MultiError) {
var (
skip string // Assume "" (don't skip).
errs gtserror.MultiError
)
// Never skip block side effects in allowlist mode.
fediMode := config.GetInstanceFederationMode()
if fediMode == config.InstanceFederationModeAllowlist {
return skip, errs
}
// We know we're in blocklist mode.
//
// We want to skip domain block side
// effects if an allow is already
// in place which overrides the block.
// Check if an explicit allow exists for this domain.
domainAllow, err := p.state.DB.GetDomainAllow(ctx, domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs.Appendf("error getting domain allow: %w", err)
return skip, errs
}
if domainAllow != nil {
skip = "running in blocklist mode, and an explicit allow exists for this domain"
return skip, errs
}
return skip, errs
}
// domainBlockSideEffects processes the side effects of a domain block:
//
// 1. Strip most info away from the instance entry for the domain.
// 2. Pass each account from the domain to the processor for deletion.
//
// It should be called asynchronously, since it can take a while when
// there are many accounts present on the given domain.
func (p *Processor) domainBlockSideEffects(
ctx context.Context,
block *gtsmodel.DomainBlock,
) gtserror.MultiError {
var errs gtserror.MultiError
// If we have an instance entry for this domain,
// update it with the new block ID and clear all fields
instance, err := p.state.DB.GetInstance(ctx, block.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs.Appendf("db error getting instance %s: %w", block.Domain, err)
return errs
}
if instance != nil {
// We had an entry for this domain.
columns := stubbifyInstance(instance, block.ID)
if err := p.state.DB.UpdateInstance(ctx, instance, columns...); err != nil {
errs.Appendf("db error updating instance: %w", err)
return errs
}
}
// For each account that belongs to this domain,
// process an account delete message to remove
// that account's posts, media, etc.
if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
if err := p.state.Workers.Client.Process(ctx, &messages.FromClientAPI{
APObjectType: ap.ActorPerson,
APActivityType: ap.ActivityDelete,
GTSModel: block,
Origin: account,
Target: account,
}); err != nil {
errs.Append(err)
}
}); err != nil {
errs.Appendf("db error ranging through accounts: %w", err)
}
return errs
return apiDomainBlock, action.ID, nil
}
func (p *Processor) deleteDomainBlock(
@ -247,104 +123,23 @@ func (p *Processor) deleteDomainBlock(
return nil, "", gtserror.NewErrorInternalError(err)
}
actionID := id.NewULID()
// Run admin action to process
// side effects of unblock.
action := &gtsmodel.AdminAction{
ID: id.NewULID(),
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domainBlock.Domain,
Type: gtsmodel.AdminActionUnsuspend,
AccountID: adminAcct.ID,
}
// Process domain unblock side
// effects asynchronously.
if errWithCode := p.actions.Run(
if errWithCode := p.state.Actions.Run(
ctx,
&gtsmodel.AdminAction{
ID: actionID,
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domainBlock.Domain,
Type: gtsmodel.AdminActionUnsuspend,
AccountID: adminAcct.ID,
},
func(ctx context.Context) gtserror.MultiError {
// Log start + finish.
l := log.WithFields(kv.Fields{
{"domain", domainBlock.Domain},
{"actionID", actionID},
}...).WithContext(ctx)
l.Info("processing domain unblock side effects")
defer func() { l.Info("finished processing domain unblock side effects") }()
return p.domainUnblockSideEffects(ctx, domainBlock)
},
action,
p.state.Actions.DomainUnblockF(action.ID, domainBlock),
); errWithCode != nil {
return nil, actionID, errWithCode
return nil, action.ID, errWithCode
}
return apiDomainBlock, actionID, nil
}
// domainUnblockSideEffects processes the side effects of undoing a
// domain block:
//
// 1. Mark instance entry as no longer suspended.
// 2. Mark each account from the domain as no longer suspended, if the
// suspension origin corresponds to the ID of the provided domain block.
//
// It should be called asynchronously, since it can take a while when
// there are many accounts present on the given domain.
func (p *Processor) domainUnblockSideEffects(
ctx context.Context,
block *gtsmodel.DomainBlock,
) gtserror.MultiError {
var errs gtserror.MultiError
// Update instance entry for this domain, if we have it.
instance, err := p.state.DB.GetInstance(ctx, block.Domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
errs.Appendf("db error getting instance %s: %w", block.Domain, err)
}
if instance != nil {
// We had an entry, update it to signal
// that it's no longer suspended.
instance.SuspendedAt = time.Time{}
instance.DomainBlockID = ""
if err := p.state.DB.UpdateInstance(
ctx,
instance,
"suspended_at",
"domain_block_id",
); err != nil {
errs.Appendf("db error updating instance: %w", err)
return errs
}
}
// Unsuspend all accounts whose suspension origin was this domain block.
if err := p.rangeDomainAccounts(ctx, block.Domain, func(account *gtsmodel.Account) {
if account.SuspensionOrigin == "" || account.SuspendedAt.IsZero() {
// Account wasn't suspended, nothing to do.
return
}
if account.SuspensionOrigin != block.ID {
// Account was suspended, but not by
// this domain block, leave it alone.
return
}
// Account was suspended by this domain
// block, mark it as unsuspended.
account.SuspendedAt = time.Time{}
account.SuspensionOrigin = ""
if err := p.state.DB.UpdateAccount(
ctx,
account,
"suspended_at",
"suspension_origin",
); err != nil {
errs.Appendf("db error updating account %s: %w", account.Username, err)
}
}); err != nil {
errs.Appendf("db error ranging through accounts: %w", err)
}
return errs
return apiDomainBlock, action.ID, nil
}

53
internal/processing/admin/domainkeysexpire.go
View File

@ -19,7 +19,6 @@
import (
"context"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -39,47 +38,23 @@ func (p *Processor) DomainKeysExpire(
adminAcct *gtsmodel.Account,
domain string,
) (string, gtserror.WithCode) {
actionID := id.NewULID()
// Run admin action to process
// side effects of key expiry.
action := &gtsmodel.AdminAction{
ID: id.NewULID(),
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domain,
Type: gtsmodel.AdminActionExpireKeys,
AccountID: adminAcct.ID,
}
// Process key expiration asynchronously.
if errWithCode := p.actions.Run(
if errWithCode := p.state.Actions.Run(
ctx,
&gtsmodel.AdminAction{
ID: actionID,
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domain,
Type: gtsmodel.AdminActionExpireKeys,
AccountID: adminAcct.ID,
},
func(ctx context.Context) gtserror.MultiError {
return p.domainKeysExpireSideEffects(ctx, domain)
},
action,
p.state.Actions.DomainKeysExpireF(domain),
); errWithCode != nil {
return actionID, errWithCode
return action.ID, errWithCode
}
return actionID, nil
}
func (p *Processor) domainKeysExpireSideEffects(ctx context.Context, domain string) gtserror.MultiError {
var (
expiresAt = time.Now()
errs gtserror.MultiError
)
// For each account on this domain, expire
// the public key and update the account.
if err := p.rangeDomainAccounts(ctx, domain, func(account *gtsmodel.Account) {
account.PublicKeyExpiresAt = expiresAt
if err := p.state.DB.UpdateAccount(ctx,
account,
"public_key_expires_at",
); err != nil {
errs.Appendf("db error updating account: %w", err)
}
}); err != nil {
errs.Appendf("db error ranging through accounts: %w", err)
}
return errs
return action.ID, nil
}

2
internal/processing/admin/domainpermission_test.go
View File

@ -186,7 +186,7 @@ func (suite *DomainBlockTestSuite) awaitAction(actionID string) {
ctx := context.Background()
if !testrig.WaitFor(func() bool {
return suite.adminProcessor.Actions().TotalRunning() == 0
return suite.state.Actions.TotalRunning() == 0
}) {
suite.FailNow("timed out waiting for admin action(s) to finish")
}

88
internal/processing/admin/domainpermissionsubscription.go
View File

@ -22,6 +22,7 @@
"errors"
"fmt"
"net/url"
"slices"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
@ -283,3 +284,90 @@ func (p *Processor) DomainPermissionSubscriptionRemove(
return p.apiDomainPermSub(ctx, permSub)
}
func (p *Processor) DomainPermissionSubscriptionTest(
ctx context.Context,
acct *gtsmodel.Account,
id string,
) (any, gtserror.WithCode) {
permSub, err := p.state.DB.GetDomainPermissionSubscriptionByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := gtserror.Newf("db error getting domain permission subscription %s: %w", id, err)
return nil, gtserror.NewErrorInternalError(err)
}
if permSub == nil {
err := fmt.Errorf("domain permission subscription %s not found", id)
return nil, gtserror.NewErrorNotFound(err, err.Error())
}
// To process the test/dry-run correctly, we need to get
// all domain perm subs of this type with a *higher* priority,
// to know whether we ought to create permissions or not.
permSubs, err := p.state.DB.GetDomainPermissionSubscriptionsByPriority(
ctx,
permSub.PermissionType,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := gtserror.Newf("db error: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Find the index of the targeted
// subscription in the slice.
index := slices.IndexFunc(
permSubs,
func(ps *gtsmodel.DomainPermissionSubscription) bool {
return ps.ID == permSub.ID
},
)
// Everything *before* the targeted subscription has a higher priority.
getHigherPrios := func() ([]*gtsmodel.DomainPermissionSubscription, error) {
return permSubs[:index], nil
}
// Get a transport for calling permSub.URI.
tsport, err := p.transport.NewTransportForUsername(ctx, acct.Username)
if err != nil {
err := gtserror.Newf("error getting transport: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Call the permSub.URI and parse a list of perms from it.
// Any error returned here is a "real" one, not an error
// from fetching / parsing the list.
createdPerms, err := p.subscriptions.ProcessDomainPermissionSubscription(
ctx,
permSub,
tsport,
getHigherPrios,
true, // Dry run.
)
if err != nil {
err := gtserror.Newf("error doing dry-run: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// If permSub has an error set on it now,
// we should return it to the caller.
if permSub.Error != "" {
return map[string]string{
"error": permSub.Error,
}, nil
}
// No error, so return the list of
// perms that would have been created.
apiPerms := make([]*apimodel.DomainPermission, 0, len(createdPerms))
for _, perm := range createdPerms {
apiPerm, errWithCode := p.apiDomainPerm(ctx, perm, false)
if errWithCode != nil {
return nil, errWithCode
}
apiPerms = append(apiPerms, apiPerm)
}
return apiPerms, nil
}

74
internal/processing/admin/util.go
View File

@ -19,86 +19,12 @@
import (
"context"
"errors"
"time"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// stubbifyInstance renders the given instance as a stub,
// removing most information from it and marking it as
// suspended.
//
// For caller's convenience, this function returns the db
// names of all columns that are updated by it.
func stubbifyInstance(instance *gtsmodel.Instance, domainBlockID string) []string {
instance.Title = ""
instance.SuspendedAt = time.Now()
instance.DomainBlockID = domainBlockID
instance.ShortDescription = ""
instance.Description = ""
instance.Terms = ""
instance.ContactEmail = ""
instance.ContactAccountUsername = ""
instance.ContactAccountID = ""
instance.Version = ""
return []string{
"title",
"suspended_at",
"domain_block_id",
"short_description",
"description",
"terms",
"contact_email",
"contact_account_username",
"contact_account_id",
"version",
}
}
// rangeDomainAccounts iterates through all accounts
// originating from the given domain, and calls the
// provided range function on each account.
//
// If an error is returned while selecting accounts,
// the loop will stop and return the error.
func (p *Processor) rangeDomainAccounts(
ctx context.Context,
domain string,
rangeF func(*gtsmodel.Account),
) error {
var (
limit = 50 // Limit selection to avoid spiking mem/cpu.
maxID string // Start with empty string to select from top.
)
for {
// Get (next) page of accounts.
accounts, err := p.state.DB.GetInstanceAccounts(ctx, domain, maxID, limit)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// Real db error.
return gtserror.Newf("db error getting instance accounts: %w", err)
}
if len(accounts) == 0 {
// No accounts left, we're done.
return nil
}
// Set next max ID for paging down.
maxID = accounts[len(accounts)-1].ID
// Call provided range function.
for _, account := range accounts {
rangeF(account)
}
}
}
// apiDomainPerm is a cheeky shortcut for returning
// the API version of the given domain permission,
// or an appropriate error if something goes wrong.

2
internal/processing/conversations/conversations_test.go
View File

@ -23,6 +23,7 @@
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
dbtest "github.com/superseriousbusiness/gotosocial/internal/db/test"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -103,6 +104,7 @@ func (suite *ConversationsTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.tc = typeutils.NewConverter(&suite.state)
suite.filter = visibility.NewFilter(&suite.state)

2
internal/processing/media/media_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -75,6 +76,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.tc = typeutils.NewConverter(&suite.state)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

4
internal/processing/processor.go
View File

@ -48,6 +48,7 @@
"github.com/superseriousbusiness/gotosocial/internal/processing/user"
"github.com/superseriousbusiness/gotosocial/internal/processing/workers"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@ -180,6 +181,7 @@ func (p *Processor) Workers() *workers.Processor {
// NewProcessor returns a new Processor.
func NewProcessor(
cleaner *cleaner.Cleaner,
subscriptions *subscriptions.Subscriptions,
converter *typeutils.Converter,
federator *federation.Federator,
oauthServer oauth.Server,
@ -210,7 +212,7 @@ func NewProcessor(
// Instantiate the rest of the sub
// processors + pin them to this struct.
processor.account = account.New(&common, state, converter, mediaManager, federator, visFilter, parseMentionFunc)
processor.admin = admin.New(&common, state, cleaner, federator, converter, mediaManager, federator.TransportController(), emailSender)
processor.admin = admin.New(&common, state, cleaner, subscriptions, federator, converter, mediaManager, federator.TransportController(), emailSender)
processor.conversations = conversations.New(state, converter, visFilter)
processor.fedi = fedi.New(state, &common, converter, federator, visFilter)
processor.filtersv1 = filtersv1.New(state, converter, &processor.stream)

4
internal/processing/processor_test.go
View File

@ -21,6 +21,7 @@
"context"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -34,6 +35,7 @@
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/stream"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -102,6 +104,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage
@ -125,6 +128,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.processor = processing.NewProcessor(
cleaner.New(&suite.state),
subscriptions.New(&suite.state, suite.transportController, suite.typeconverter),
suite.typeconverter,
suite.federator,
suite.oauthServer,

2
internal/processing/status/status_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@ -84,6 +85,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.typeConverter = typeutils.NewConverter(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.tc = testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media"))
suite.storage = testrig.NewInMemoryStorage()

2
internal/processing/stream/stream_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
@ -50,6 +51,7 @@ func (suite *StreamTestSuite) SetupTest() {
suite.testTokens = testrig.NewTestTokens()
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.streamProcessor = stream.New(&suite.state, suite.oauthServer)

2
internal/processing/timeline/timeline_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -55,6 +56,7 @@ func (suite *TimelineStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.timeline = timeline.New(
&suite.state,

2
internal/processing/user/user_test.go
View File

@ -19,6 +19,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -49,6 +50,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)

8
internal/state/state.go
View File

@ -19,6 +19,7 @@
import (
"codeberg.org/gruf/go-mutexes"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/cache"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/storage"
@ -61,9 +62,14 @@ type State struct {
// Storage provides access to the storage driver.
Storage *storage.Driver
// Workers provides access to this state's collection of worker pools.
// Workers provides access to this
// state's collection of worker pools.
Workers workers.Workers
// Struct to manage running admin
// actions (and locks thereupon).
Actions *actions.Actions
// prevent pass-by-value.
_ nocopy
}

804
internal/subscriptions/domainperms.go Normal file
View File

@ -0,0 +1,804 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package subscriptions
import (
"context"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io"
"slices"
"strconv"
"strings"
"time"
"codeberg.org/gruf/go-kv"
"github.com/miekg/dns"
"github.com/superseriousbusiness/gotosocial/internal/actions"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// ScheduleJobs schedules domain permission subscription
// fetching + updating using configured parameters.
//
// Returns an error if `MediaCleanupFrom`
// is not a valid format (hh:mm:ss).
func (s *Subscriptions) ScheduleJobs() error {
const hourMinute = "15:04"
var (
now = time.Now()
processEvery = config.GetInstanceSubscriptionsProcessEvery()
processFromStr = config.GetInstanceSubscriptionsProcessFrom()
)
// Parse processFromStr as hh:mm.
// Resulting time will be on 1 Jan year zero.
cleanupFrom, err := time.Parse(hourMinute, processFromStr)
if err != nil {
return gtserror.Newf(
"error parsing '%s' in time format 'hh:mm': %w",
processFromStr, err,
)
}
// Time travel from
// year zero, groovy.
firstProcessAt := time.Date(
now.Year(),
now.Month(),
now.Day(),
cleanupFrom.Hour(),
cleanupFrom.Minute(),
0,
0,
now.Location(),
)
// Ensure first processing is in the future.
for firstProcessAt.Before(now) {
firstProcessAt = firstProcessAt.Add(processEvery)
}
fn := func(ctx context.Context, start time.Time) {
log.Info(ctx, "starting instance subscriptions processing")
// In blocklist (default) mode, process allows
// first to provide immunity to block side effects.
//
// In allowlist mode, process blocks first to
// ensure allowlist doesn't override blocks.
var order [2]gtsmodel.DomainPermissionType
if config.GetInstanceFederationMode() == config.InstanceFederationModeBlocklist {
order = [2]gtsmodel.DomainPermissionType{
gtsmodel.DomainPermissionAllow,
gtsmodel.DomainPermissionBlock,
}
} else {
order = [2]gtsmodel.DomainPermissionType{
gtsmodel.DomainPermissionBlock,
gtsmodel.DomainPermissionAllow,
}
}
// Fetch + process subscribed perms in order.
for _, permType := range order {
s.ProcessDomainPermissionSubscriptions(ctx, permType)
}
log.Infof(ctx, "finished instance subscriptions processing after %s", time.Since(start))
}
log.Infof(nil,
"scheduling instance subscriptions processing to run every %s, starting from %s; next clean will run at %s",
processEvery, processFromStr, firstProcessAt,
)
// Schedule processing to execute according to schedule.
if !s.state.Workers.Scheduler.AddRecurring(
"@subsprocessing",
firstProcessAt,
processEvery,
fn,
) {
panic("failed to schedule @subsprocessing")
}
return nil
}
// ProcessDomainPermissionSubscriptions processes all domain permission
// subscriptions of the given permission type by, in turn, calling the
// URI of each subscription, parsing the result into a list of domain
// permissions, and creating (or skipping) each permission as appropriate.
func (s *Subscriptions) ProcessDomainPermissionSubscriptions(
ctx context.Context,
permType gtsmodel.DomainPermissionType,
) {
log.Info(ctx, "start")
// Get permission subscriptions in priority order (highest -> lowest).
permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// Real db error.
log.Error(ctx, err)
return
}
if len(permSubs) == 0 {
// No subscriptions of this
// type, so nothing to do.
return
}
// Get a transport using the instance account,
// we can reuse this for each HTTP call.
tsport, err := s.transportController.NewTransportForUsername(ctx, "")
if err != nil {
log.Error(ctx, err)
return
}
for i, permSub := range permSubs {
// Higher priority permission subs = everything
// above this permission sub in the slice.
getHigherPrios := func() ([]*gtsmodel.DomainPermissionSubscription, error) {
return permSubs[:i], nil
}
_, err := s.ProcessDomainPermissionSubscription(
ctx,
permSub,
tsport,
getHigherPrios,
false, // Not dry. Wet, if you will.
)
if err != nil {
// Real db error.
log.Error(ctx, err)
return
}
// Update this perm sub.
err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub)
if err != nil {
// Real db error.
log.Error(ctx, err)
return
}
}
log.Info(ctx, "finished")
}
// ProcessDomainPermissionSubscription processes one domain permission
// subscription by dereferencing the URI, parsing the response into a list
// of permissions, and for each discovered permission either creating an
// entry in the database, or ignoring it if it's excluded or already
// covered by a higher-priority subscription.
//
// On success, the slice of discovered DomainPermissions will be returned.
// In case of parsing error, or error on the remote side, permSub.Error
// will be updated with the calling/parsing error, and `nil, nil` will be
// returned. In case of an actual db error, `nil, err` will be returned and
// the caller should handle it.
//
// getHigherPrios should be a function for returning a slice of domain
// permission subscriptions with a higher priority than the given permSub.
//
// If dry == true, then the URI will still be called, and permissions
// will be parsed, but they will not actually be created.
//
// Note that while this function modifies fields on the given permSub,
// it's up to the caller to update it in the database (if desired).
func (s *Subscriptions) ProcessDomainPermissionSubscription(
ctx context.Context,
permSub *gtsmodel.DomainPermissionSubscription,
tsport transport.Transport,
getHigherPrios func() ([]*gtsmodel.DomainPermissionSubscription, error),
dry bool,
) ([]gtsmodel.DomainPermission, error) {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"permType", permSub.PermissionType.String()},
{"permSubURI", permSub.URI},
}...)
// Set FetchedAt as we're
// going to attempt this now.
permSub.FetchedAt = time.Now()
// Call the URI, only force
// if we're doing a dry run.
resp, err := tsport.DereferenceDomainPermissions(
ctx, permSub, !dry,
)
if err != nil {
// Couldn't get this one,
// set error + return.
errStr := err.Error()
l.Warnf("couldn't dereference permSubURI: %+v", err)
permSub.Error = errStr
return nil, nil
}
// If the permissions at URI weren't modified
// since last time, just update some metadata
// to indicate a successful fetch, and return.
if resp.Unmodified {
l.Debug("received 304 Not Modified from remote")
permSub.SuccessfullyFetchedAt = permSub.FetchedAt
if permSub.ETag == "" && resp.ETag != "" {
// We didn't have an ETag before but
// we have one now: probably the remote
// added ETag support in the meantime.
permSub.ETag = resp.ETag
}
return nil, nil
}
// At this point we know we got a 200 OK
// from the URI, so we've got a live body!
// Try to parse the body as a list of wantedPerms
// that the subscription wants to create.
var wantedPerms []gtsmodel.DomainPermission
switch permSub.ContentType {
// text/csv
case gtsmodel.DomainPermSubContentTypeCSV:
wantedPerms, err = s.permsFromCSV(l, permSub.PermissionType, resp.Body)
// application/json
case gtsmodel.DomainPermSubContentTypeJSON:
wantedPerms, err = s.permsFromJSON(l, permSub.PermissionType, resp.Body)
// text/plain
case gtsmodel.DomainPermSubContentTypePlain:
wantedPerms, err = s.permsFromPlain(l, permSub.PermissionType, resp.Body)
}
if err != nil {
// We retrieved the permissions from remote but
// the connection died halfway through transfer,
// or we couldn't parse the results, or something.
// Just set error and return.
errStr := err.Error()
l.Warnf("couldn't parse results: %+v", err)
permSub.Error = errStr
return nil, nil
}
if len(wantedPerms) == 0 {
// Fetch was OK, and parsing was, on the surface at
// least, OK, but we didn't get any perms. Consider
// this an error as users will probably want to know.
const errStr = "fetch successful but parsed zero usable results"
l.Warn(errStr)
permSub.Error = errStr
return nil, nil
}
// This can now be considered a successful fetch.
permSub.SuccessfullyFetchedAt = permSub.FetchedAt
permSub.ETag = resp.ETag
permSub.Error = ""
// Need a list of higher priority subscriptions
// to ensure we don't create permissions wrongly.
higherPrios, err := getHigherPrios()
if err != nil {
// Proper db error.
return nil, err
}
// Keep track of which domain perms are
// created (or would be, if dry == true).
createdPerms := make([]gtsmodel.DomainPermission, 0, len(wantedPerms))
// Iterate through wantedPerms and
// create (or dry create) each one.
for _, wantedPerm := range wantedPerms {
l = l.WithField("domain", wantedPerm.GetDomain())
created, err := s.processDomainPermission(
ctx, l,
wantedPerm,
permSub,
higherPrios,
dry,
)
if err != nil {
// Proper db error.
return nil, err
}
if !created {
continue
}
createdPerms = append(createdPerms, wantedPerm)
}
return createdPerms, nil
}
// processDomainPermission processes one wanted domain
// permission discovered via a domain permission sub's URI.
//
// Error will only be returned in case of an actual database
// error, else the error will be logged and nil returned.
func (s *Subscriptions) processDomainPermission(
ctx context.Context,
l log.Entry,
wantedPerm gtsmodel.DomainPermission,
permSub *gtsmodel.DomainPermissionSubscription,
higherPrios []*gtsmodel.DomainPermissionSubscription,
dry bool,
) (bool, error) {
// Set to true if domain permission
// actually (would be) created.
var created bool
// If domain is excluded from automatic
// permission creation, don't process it.
domain := wantedPerm.GetDomain()
excluded, err := s.state.DB.IsDomainPermissionExcluded(ctx, domain)
if err != nil {
// Proper db error.
return created, err
}
if excluded {
l.Debug("domain is excluded, skipping")
return created, nil
}
// Check if a permission already exists for
// this domain, and if it's covered already
// by a higher-priority subscription.
existingPerm, covered, err := s.existingCovered(
ctx, permSub.PermissionType, domain, higherPrios,
)
if err != nil {
// Proper db error.
return created, err
}
if covered {
l.Debug("domain is covered by a higher-priority subscription, skipping")
return created, nil
}
// At this point we know we
// should create the perm.
created = true
if dry {
// Don't do creation or side
// effects if we're dry running.
return created, nil
}
// Handle perm creation differently depending
// on whether or not a perm already existed.
existing := !util.IsNil(existingPerm)
switch {
case !existing && *permSub.AsDraft:
// No existing perm, create as draft.
err = s.state.DB.PutDomainPermissionDraft(
ctx,
&gtsmodel.DomainPermissionDraft{
ID: id.NewULID(),
PermissionType: permSub.PermissionType,
Domain: domain,
CreatedByAccountID: permSub.CreatedByAccount.ID,
CreatedByAccount: permSub.CreatedByAccount,
PrivateComment: permSub.URI,
PublicComment: wantedPerm.GetPublicComment(),
Obfuscate: wantedPerm.GetObfuscate(),
SubscriptionID: permSub.ID,
},
)
case !existing && !*permSub.AsDraft:
// No existing perm, create a new one of the
// appropriate type, and process side effects.
var (
insertF func() error
action *gtsmodel.AdminAction
actionF actions.AdminActionF
)
if permSub.PermissionType == gtsmodel.DomainPermissionBlock {
// Prepare to insert + process a block.
domainBlock := &gtsmodel.DomainBlock{
ID: id.NewULID(),
Domain: domain,
CreatedByAccountID: permSub.CreatedByAccount.ID,
CreatedByAccount: permSub.CreatedByAccount,
PrivateComment: permSub.URI,
PublicComment: wantedPerm.GetPublicComment(),
Obfuscate: wantedPerm.GetObfuscate(),
SubscriptionID: permSub.ID,
}
insertF = func() error { return s.state.DB.CreateDomainBlock(ctx, domainBlock) }
action = &gtsmodel.AdminAction{
ID: id.NewULID(),
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domain,
Type: gtsmodel.AdminActionSuspend,
AccountID: permSub.CreatedByAccountID,
}
actionF = s.state.Actions.DomainBlockF(action.ID, domainBlock)
} else {
// Prepare to insert + process an allow.
domainAllow := &gtsmodel.DomainAllow{
ID: id.NewULID(),
Domain: domain,
CreatedByAccountID: permSub.CreatedByAccount.ID,
CreatedByAccount: permSub.CreatedByAccount,
PrivateComment: permSub.URI,
PublicComment: wantedPerm.GetPublicComment(),
Obfuscate: wantedPerm.GetObfuscate(),
SubscriptionID: permSub.ID,
}
insertF = func() error { return s.state.DB.CreateDomainAllow(ctx, domainAllow) }
action = &gtsmodel.AdminAction{
ID: id.NewULID(),
TargetCategory: gtsmodel.AdminActionCategoryDomain,
TargetID: domain,
Type: gtsmodel.AdminActionUnsuspend,
AccountID: permSub.CreatedByAccountID,
}
actionF = s.state.Actions.DomainAllowF(action.ID, domainAllow)
}
// Insert the new perm in the db.
if err = insertF(); err != nil {
// Couldn't insert wanted perm,
// don't process side effects.
break
}
// Run admin action to process
// side effects of permission.
err = s.state.Actions.Run(ctx, action, actionF)
case existingPerm.GetSubscriptionID() != "" || *permSub.AdoptOrphans:
// Perm exists but we should adopt/take
// it by copying over desired fields.
existingPerm.SetCreatedByAccountID(wantedPerm.GetCreatedByAccountID())
existingPerm.SetCreatedByAccount(wantedPerm.GetCreatedByAccount())
existingPerm.SetSubscriptionID(permSub.ID)
existingPerm.SetObfuscate(wantedPerm.GetObfuscate())
existingPerm.SetPrivateComment(wantedPerm.GetPrivateComment())
existingPerm.SetPublicComment(wantedPerm.GetPublicComment())
switch p := existingPerm.(type) {
case *gtsmodel.DomainBlock:
err = s.state.DB.UpdateDomainBlock(ctx, p)
case *gtsmodel.DomainAllow:
err = s.state.DB.UpdateDomainAllow(ctx, p)
}
default:
// Perm exists but we should leave it alone.
l.Debug("domain is covered by a higher-priority subscription, skipping")
}
if err != nil && !errors.Is(err, db.ErrAlreadyExists) {
// Proper db error.
return created, err
}
created = true
return created, nil
}
func (s *Subscriptions) permsFromCSV(
l log.Entry,
permType gtsmodel.DomainPermissionType,
body io.ReadCloser,
) ([]gtsmodel.DomainPermission, error) {
// Read body into memory as slice of CSV records.
records, err := csv.NewReader(body).ReadAll()
// Whatever happened, we're
// done with the body now.
body.Close()
// Check if error reading body.
if err != nil {
return nil, gtserror.NewfAt(3, "error decoding into csv: %w", err)
}
// Make sure we actually
// have some records.
if len(records) == 0 {
return nil, nil
}
// Validate column headers.
columnHeaders := records[0]
if !slices.Equal(
columnHeaders,
[]string{
"#domain",
"#severity",
"#reject_media",
"#reject_reports",
"#public_comment",
"#obfuscate",
},
) {
return nil, gtserror.Newf(
"unexpected column headers in csv: %+v",
columnHeaders,
)
}
// Trim off column headers
// now they're validated.
records = records[1:]
// Convert records to permissions slice.
perms := make([]gtsmodel.DomainPermission, 0, len(records))
for _, record := range records {
if len(record) != 6 {
l.Warnf("skipping invalid-length record: %+v", record)
continue
}
var (
domainRaw = record[0]
severity = record[1]
publicComment = record[4]
obfuscate, err = strconv.ParseBool(record[5])
)
if severity != "suspend" {
l.Warnf("skipping non-suspend record: %+v", record)
continue
}
if err != nil {
l.Warnf("couldn't parse obfuscate field of record: %+v", record)
continue
}
// Normalize + validate domain.
domain, err := validateDomain(domainRaw)
if err != nil {
l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
continue
}
// Instantiate the permission
// as either block or allow.
var perm gtsmodel.DomainPermission
if permType == gtsmodel.DomainPermissionBlock {
perm = &gtsmodel.DomainBlock{Domain: domain}
} else {
perm = &gtsmodel.DomainAllow{Domain: domain}
}
// Set remaining fields.
perm.SetPublicComment(publicComment)
perm.SetObfuscate(&obfuscate)
// We're done.
perms = append(perms, perm)
}
return perms, nil
}
func (s *Subscriptions) permsFromJSON(
l log.Entry,
permType gtsmodel.DomainPermissionType,
body io.ReadCloser,
) ([]gtsmodel.DomainPermission, error) {
var (
dec = json.NewDecoder(body)
apiPerms = make([]*apimodel.DomainPermission, 0)
)
// Read body into memory as
// slice of domain permissions.
if err := dec.Decode(&apiPerms); err != nil {
_ = body.Close() // ensure closed.
return nil, gtserror.NewfAt(3, "error decoding into json: %w", err)
}
// Perform a secondary decode just to ensure we drained the
// entirety of the data source. Error indicates either extra
// trailing garbage, or multiple JSON values (invalid data).
if err := dec.Decode(&struct{}{}); err != io.EOF {
_ = body.Close() // ensure closed.
return nil, gtserror.NewfAt(3, "data remaining after json")
}
// Done with body.
_ = body.Close()
// Convert apimodel perms to barebones internal perms.
perms := make([]gtsmodel.DomainPermission, 0, len(apiPerms))
for _, apiPerm := range apiPerms {
// Normalize + validate domain.
domainRaw := apiPerm.Domain.Domain
domain, err := validateDomain(domainRaw)
if err != nil {
l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
continue
}
// Instantiate the permission
// as either block or allow.
var perm gtsmodel.DomainPermission
if permType == gtsmodel.DomainPermissionBlock {
perm = &gtsmodel.DomainBlock{Domain: domain}
} else {
perm = &gtsmodel.DomainAllow{Domain: domain}
}
// Set remaining fields.
perm.SetPublicComment(apiPerm.PublicComment)
perm.SetObfuscate(&apiPerm.Obfuscate)
// We're done.
perms = append(perms, perm)
}
return perms, nil
}
func (s *Subscriptions) permsFromPlain(
l log.Entry,
permType gtsmodel.DomainPermissionType,
body io.ReadCloser,
) ([]gtsmodel.DomainPermission, error) {
// Read body into memory as bytes.
b, err := io.ReadAll(body)
// Whatever happened, we're
// done with the body now.
body.Close()
// Check if error reading body.
if err != nil {
return nil, gtserror.NewfAt(3, "error decoding into plain: %w", err)
}
// Coerce to newline-separated list of domains.
domains := strings.Split(string(b), "\n")
// Convert raw domains to permissions.
perms := make([]gtsmodel.DomainPermission, 0, len(domains))
for _, domainRaw := range domains {
// Normalize + validate domain.
domain, err := validateDomain(domainRaw)
if err != nil {
l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
continue
}
// Instantiate the permission
// as either block or allow.
var perm gtsmodel.DomainPermission
if permType == gtsmodel.DomainPermissionBlock {
perm = &gtsmodel.DomainBlock{Domain: domain}
} else {
perm = &gtsmodel.DomainAllow{Domain: domain}
}
// We're done.
perms = append(perms, perm)
}
return perms, nil
}
func validateDomain(domain string) (string, error) {
// Basic validation.
if _, ok := dns.IsDomainName(domain); !ok {
err := fmt.Errorf("invalid domain name")
return "", err
}
// Convert to punycode.
domain, err := util.Punify(domain)
if err != nil {
err := fmt.Errorf("could not punify domain: %w", err)
return "", err
}
// Check for invalid characters
// after the punification process.
if strings.ContainsAny(domain, "*, \n") {
err := fmt.Errorf("invalid char(s) in domain")
return "", err
}
return domain, nil
}
func (s *Subscriptions) existingCovered(
ctx context.Context,
permType gtsmodel.DomainPermissionType,
domain string,
higherPrios []*gtsmodel.DomainPermissionSubscription,
) (
existingPerm gtsmodel.DomainPermission,
covered bool,
err error,
) {
// Check for existing permission of appropriate type.
var dbErr error
if permType == gtsmodel.DomainPermissionBlock {
existingPerm, dbErr = s.state.DB.GetDomainBlock(ctx, domain)
} else {
existingPerm, dbErr = s.state.DB.GetDomainAllow(ctx, domain)
}
if dbErr != nil && !errors.Is(dbErr, db.ErrNoEntries) {
// Real db error.
err = dbErr
return
}
if util.IsNil(existingPerm) {
// Can't be covered if
// no existing perm.
return
}
subscriptionID := existingPerm.GetSubscriptionID()
if subscriptionID == "" {
// Can't be covered if
// no subscription ID.
return
}
// Covered if subscription ID is in the slice
// of higher-priority permission subscriptions.
covered = slices.ContainsFunc(
higherPrios,
func(permSub *gtsmodel.DomainPermissionSubscription) bool {
return permSub.ID == subscriptionID
},
)
return
}

42
internal/subscriptions/subscriptions.go Normal file
View File

@ -0,0 +1,42 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package subscriptions
import (
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
type Subscriptions struct {
state *state.State
transportController transport.Controller
tc *typeutils.Converter
}
func New(
state *state.State,
transportController transport.Controller,
tc *typeutils.Converter,
) *Subscriptions {
return &Subscriptions{
state: state,
transportController: transportController,
tc: tc,
}
}

496
internal/subscriptions/subscriptions_test.go Normal file
View File

@ -0,0 +1,496 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package subscriptions_test
import (
"context"
"errors"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/util"
"github.com/superseriousbusiness/gotosocial/testrig"
)
const (
rMediaPath = "../../testrig/media"
rTemplatePath = "../../web/template"
)
type SubscriptionsTestSuite struct {
suite.Suite
testAccounts map[string]*gtsmodel.Account
}
func (suite *SubscriptionsTestSuite) SetupSuite() {
testrig.InitTestConfig()
testrig.InitTestLog()
suite.testAccounts = testrig.NewTestAccounts()
}
func (suite *SubscriptionsTestSuite) TestDomainBlocksCSV() {
var (
ctx = context.Background()
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
testAccount = suite.testAccounts["admin_account"]
subscriptions = subscriptions.New(
testStructs.State,
testStructs.TransportController,
testStructs.TypeConverter,
)
// Create a subscription for a CSV list of baddies.
testSubscription = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/baddies.csv",
ContentType: gtsmodel.DomainPermSubContentTypeCSV,
}
)
defer testrig.TearDownTestStructs(testStructs)
// Store test subscription.
if err := testStructs.State.DB.PutDomainPermissionSubscription(
ctx, testSubscription,
); err != nil {
suite.FailNow(err.Error())
}
// Process all subscriptions.
subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
// We should now have blocks for
// each domain on the subscribed list.
for _, domain := range []string{
"bumfaces.net",
"peepee.poopoo",
"nothanks.com",
} {
var (
perm gtsmodel.DomainPermission
err error
)
if !testrig.WaitFor(func() bool {
perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
return err == nil
}) {
suite.FailNowf("", "timed out waiting for domain %s", domain)
}
suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
}
// The just-fetched perm sub should
// have ETag and count etc set now.
permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
ctx, testSubscription.ID,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.Equal("bigbums6969", permSub.ETag)
suite.EqualValues(3, permSub.Count)
suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
}
func (suite *SubscriptionsTestSuite) TestDomainBlocksJSON() {
var (
ctx = context.Background()
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
testAccount = suite.testAccounts["admin_account"]
subscriptions = subscriptions.New(
testStructs.State,
testStructs.TransportController,
testStructs.TypeConverter,
)
// Create a subscription for a JSON list of baddies.
testSubscription = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/baddies.json",
ContentType: gtsmodel.DomainPermSubContentTypeJSON,
}
)
defer testrig.TearDownTestStructs(testStructs)
// Store test subscription.
if err := testStructs.State.DB.PutDomainPermissionSubscription(
ctx, testSubscription,
); err != nil {
suite.FailNow(err.Error())
}
// Process all subscriptions.
subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
// We should now have blocks for
// each domain on the subscribed list.
for _, domain := range []string{
"bumfaces.net",
"peepee.poopoo",
"nothanks.com",
} {
var (
perm gtsmodel.DomainPermission
err error
)
if !testrig.WaitFor(func() bool {
perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
return err == nil
}) {
suite.FailNowf("", "timed out waiting for domain %s", domain)
}
suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
}
// The just-fetched perm sub should
// have ETag and count etc set now.
permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
ctx, testSubscription.ID,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.Equal("don't modify me daddy", permSub.ETag)
suite.EqualValues(3, permSub.Count)
suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
}
func (suite *SubscriptionsTestSuite) TestDomainBlocksPlain() {
var (
ctx = context.Background()
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
testAccount = suite.testAccounts["admin_account"]
subscriptions = subscriptions.New(
testStructs.State,
testStructs.TransportController,
testStructs.TypeConverter,
)
// Create a subscription for a plain list of baddies.
testSubscription = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/baddies.txt",
ContentType: gtsmodel.DomainPermSubContentTypePlain,
}
)
defer testrig.TearDownTestStructs(testStructs)
// Store test subscription.
if err := testStructs.State.DB.PutDomainPermissionSubscription(
ctx, testSubscription,
); err != nil {
suite.FailNow(err.Error())
}
// Process all subscriptions.
subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
// We should now have blocks for
// each domain on the subscribed list.
for _, domain := range []string{
"bumfaces.net",
"peepee.poopoo",
"nothanks.com",
} {
var (
perm gtsmodel.DomainPermission
err error
)
if !testrig.WaitFor(func() bool {
perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
return err == nil
}) {
suite.FailNowf("", "timed out waiting for domain %s", domain)
}
suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
}
// The just-fetched perm sub should
// have ETag and count etc set now.
permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
ctx, testSubscription.ID,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.Equal("this is a legit etag i swear", permSub.ETag)
suite.EqualValues(3, permSub.Count)
suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
}
func (suite *SubscriptionsTestSuite) TestDomainBlocksCSVETag() {
var (
ctx = context.Background()
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
testAccount = suite.testAccounts["admin_account"]
subscriptions = subscriptions.New(
testStructs.State,
testStructs.TransportController,
testStructs.TypeConverter,
)
// Create a subscription for a CSV list of baddies.
// Include the ETag so it gets sent with the request.
testSubscription = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/baddies.csv",
ContentType: gtsmodel.DomainPermSubContentTypeCSV,
ETag: "bigbums6969",
}
)
defer testrig.TearDownTestStructs(testStructs)
// Store test subscription.
if err := testStructs.State.DB.PutDomainPermissionSubscription(
ctx, testSubscription,
); err != nil {
suite.FailNow(err.Error())
}
// Process all subscriptions.
subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
// We should now NOT have blocks for the domains
// on the list, as the remote will have returned
// 304, indicating we should do nothing.
for _, domain := range []string{
"bumfaces.net",
"peepee.poopoo",
"nothanks.com",
} {
_, err := testStructs.State.DB.GetDomainBlock(ctx, domain)
if !errors.Is(err, db.ErrNoEntries) {
suite.FailNowf("", "domain perm %s created when it shouldn't be")
}
}
// The just-fetched perm sub should
// have ETag and count etc set now.
permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
ctx, testSubscription.ID,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.Equal("bigbums6969", permSub.ETag)
suite.EqualValues(0, permSub.Count) // Should be 0.
suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
}
func (suite *SubscriptionsTestSuite) TestDomainBlocks404() {
var (
ctx = context.Background()
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
testAccount = suite.testAccounts["admin_account"]
subscriptions = subscriptions.New(
testStructs.State,
testStructs.TransportController,
testStructs.TypeConverter,
)
// Create a subscription for a CSV list of baddies.
// The endpoint will return a 404 so we can test erroring.
testSubscription = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/does_not_exist.csv",
ContentType: gtsmodel.DomainPermSubContentTypeCSV,
}
)
defer testrig.TearDownTestStructs(testStructs)
// Store test subscription.
if err := testStructs.State.DB.PutDomainPermissionSubscription(
ctx, testSubscription,
); err != nil {
suite.FailNow(err.Error())
}
// Process all subscriptions.
subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
// The just-fetched perm sub should have an error set on it.
permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
ctx, testSubscription.ID,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.EqualValues(0, permSub.Count)
suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
suite.Zero(permSub.SuccessfullyFetchedAt)
suite.Equal(`DereferenceDomainPermissions: GET request to https://lists.example.org/does_not_exist.csv failed: status="" body="{"error":"not found"}"`, permSub.Error)
}
func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypeCSV() {
var (
ctx = context.Background()
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
testAccount = suite.testAccounts["admin_account"]
subscriptions = subscriptions.New(
testStructs.State,
testStructs.TransportController,
testStructs.TypeConverter,
)
// Create a subscription for a plaintext list of baddies,
// but try to parse as CSV content type (shouldn't work).
testSubscription = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/baddies.txt",
ContentType: gtsmodel.DomainPermSubContentTypeCSV,
}
)
defer testrig.TearDownTestStructs(testStructs)
// Store test subscription.
if err := testStructs.State.DB.PutDomainPermissionSubscription(
ctx, testSubscription,
); err != nil {
suite.FailNow(err.Error())
}
// Process all subscriptions.
subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
// The just-fetched perm sub should have an error set on it.
permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
ctx, testSubscription.ID,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.Zero(permSub.Count)
suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
suite.Zero(permSub.SuccessfullyFetchedAt)
suite.Equal(`permsFromCSV: unexpected column headers in csv: [bumfaces.net]`, permSub.Error)
}
func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypePlain() {
var (
ctx = context.Background()
testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
testAccount = suite.testAccounts["admin_account"]
subscriptions = subscriptions.New(
testStructs.State,
testStructs.TransportController,
testStructs.TypeConverter,
)
// Create a subscription for a plaintext list of baddies,
// but try to parse as CSV content type (shouldn't work).
testSubscription = &gtsmodel.DomainPermissionSubscription{
ID: "01JGE681TQSBPAV59GZXPKE62H",
Priority: 255,
Title: "whatever!",
PermissionType: gtsmodel.DomainPermissionBlock,
AsDraft: util.Ptr(false),
AdoptOrphans: util.Ptr(true),
CreatedByAccountID: testAccount.ID,
CreatedByAccount: testAccount,
URI: "https://lists.example.org/baddies.csv",
ContentType: gtsmodel.DomainPermSubContentTypePlain,
}
)
defer testrig.TearDownTestStructs(testStructs)
// Store test subscription.
if err := testStructs.State.DB.PutDomainPermissionSubscription(
ctx, testSubscription,
); err != nil {
suite.FailNow(err.Error())
}
// Process all subscriptions.
subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
// The just-fetched perm sub should have an error set on it.
permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
ctx, testSubscription.ID,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.Zero(permSub.Count)
suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
suite.Zero(permSub.SuccessfullyFetchedAt)
suite.Equal(`fetch successful but parsed zero usable results`, permSub.Error)
}
func TestSubscriptionTestSuite(t *testing.T) {
suite.Run(t, new(SubscriptionsTestSuite))
}

121
internal/transport/derefdomainpermlist.go Normal file
View File

@ -0,0 +1,121 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transport
import (
"context"
"io"
"net/http"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
type DereferenceDomainPermissionsResp struct {
// Set only if response was 200 OK.
// It's up to the caller to close
// this when they're done with it.
Body io.ReadCloser
// True if response
// was 304 Not Modified.
Unmodified bool
// May be set
// if 200 or 304.
ETag string
}
func (t *transport) DereferenceDomainPermissions(
ctx context.Context,
permSub *gtsmodel.DomainPermissionSubscription,
force bool,
) (*DereferenceDomainPermissionsResp, error) {
// Prepare new HTTP request to endpoint
req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil)
if err != nil {
return nil, err
}
// Set basic auth header if necessary.
if permSub.FetchUsername != "" || permSub.FetchPassword != "" {
req.SetBasicAuth(permSub.FetchUsername, permSub.FetchPassword)
}
// Set relevant Accept headers.
// Allow fallback in case target doesn't
// negotiate content type correctly.
req.Header.Add("Accept-Charset", "utf-8")
req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*")
// If force is true, we want to skip setting Cache
// headers so that we definitely don't get a 304 back.
if !force {
// If we've successfully fetched this list
// before, set If-Modified-Since to last
// success to make the request conditional.
//
// See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since
if !permSub.SuccessfullyFetchedAt.IsZero() {
timeStr := permSub.SuccessfullyFetchedAt.Format(http.TimeFormat)
req.Header.Add("If-Modified-Since", timeStr)
}
// If we've got an ETag stored for this list, set
// If-None-Match to make the request conditional.
// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#caching_of_unchanged_resources.
if len(permSub.ETag) != 0 {
req.Header.Add("If-None-Match", permSub.ETag)
}
}
// Perform the HTTP request
rsp, err := t.GET(req)
if err != nil {
return nil, err
}
// If we have an unexpected / error response,
// wrap + return as error. This will also drain
// and close the response body for us.
if rsp.StatusCode != http.StatusOK &&
rsp.StatusCode != http.StatusNotModified {
err := gtserror.NewFromResponse(rsp)
return nil, err
}
// Check already if we were given an ETag
// we can use, as ETag is often returned
// even on 304 Not Modified responses.
permsResp := &DereferenceDomainPermissionsResp{
ETag: rsp.Header.Get("Etag"),
}
if rsp.StatusCode == http.StatusNotModified {
// Nothing has changed on the remote side
// since we last fetched, so there's nothing
// to do and we don't need to read the body.
rsp.Body.Close()
permsResp.Unmodified = true
} else {
// Return the live body to the caller.
permsResp.Body = rsp.Body
}
return permsResp, nil
}

14
internal/transport/transport.go
View File

@ -78,6 +78,20 @@ type Transport interface {
// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
// DereferenceDomainPermissions dereferences the
// permissions list present at the given permSub's URI.
//
// If "force", then If-Modified-Since and If-None-Match
// headers will *NOT* be sent with the outgoing request.
//
// If err == nil and Unmodified == false, then it's up
// to the caller to close the returned io.ReadCloser.
DereferenceDomainPermissions(
ctx context.Context,
permSub *gtsmodel.DomainPermissionSubscription,
force bool,
) (*DereferenceDomainPermissionsResp, error)
// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error)
}

2
internal/transport/transport_test.go
View File

@ -21,6 +21,7 @@
"context"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
@ -74,6 +75,7 @@ func (suite *TransportTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
suite.storage = testrig.NewInMemoryStorage()
suite.state.Storage = suite.storage

2
internal/typeutils/converter_test.go
View File

@ -20,6 +20,7 @@
import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -499,6 +500,7 @@ func (suite *TypeUtilsTestSuite) SetupTest() {
suite.db = testrig.NewTestDB(&suite.state)
suite.state.DB = suite.db
suite.state.Actions = actions.New(suite.state.DB, &suite.state.Workers)
storage := testrig.NewInMemoryStorage()
suite.state.Storage = storage

4
internal/typeutils/internaltofrontend.go
View File

@ -2121,7 +2121,9 @@ func (c *Converter) DomainPermToAPIDomainPerm(
domainPerm.PrivateComment = d.GetPrivateComment()
domainPerm.SubscriptionID = d.GetSubscriptionID()
domainPerm.CreatedBy = d.GetCreatedByAccountID()
domainPerm.CreatedAt = util.FormatISO8601(d.GetCreatedAt())
if createdAt := d.GetCreatedAt(); !createdAt.IsZero() {
domainPerm.CreatedAt = util.FormatISO8601(createdAt)
}
// If this is a draft, also add the permission type.
if _, ok := d.(*gtsmodel.DomainPermissionDraft); ok {

2
testrig/config.go
View File

@ -99,6 +99,8 @@ func testDefaults() config.Configuration {
TagStr: "en-gb",
},
},
InstanceSubscriptionsProcessFrom: "23:00", // 11pm,
InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
AccountsRegistrationOpen: true,
AccountsReasonRequired: true,

14
testrig/processor.go
View File

@ -26,15 +26,27 @@
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
// NewTestProcessor returns a Processor suitable for testing purposes.
// The passed in state will have its worker functions set appropriately,
// but the state will not be initialized.
func NewTestProcessor(state *state.State, federator *federation.Federator, emailSender email.Sender, mediaManager *media.Manager) *processing.Processor {
func NewTestProcessor(
state *state.State,
federator *federation.Federator,
emailSender email.Sender,
mediaManager *media.Manager,
) *processing.Processor {
return processing.NewProcessor(
cleaner.New(state),
subscriptions.New(
state,
federator.TransportController(),
typeutils.NewConverter(state),
),
typeutils.NewConverter(state),
federator,
NewTestOauthServer(state.DB),

31
testrig/teststructs.go
View File

@ -18,6 +18,7 @@
package testrig
import (
"github.com/superseriousbusiness/gotosocial/internal/actions"
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@ -25,6 +26,8 @@
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/common"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@ -38,12 +41,13 @@
// and worker queues, which was causing issues
// when running all tests at once.
type TestStructs struct {
State *state.State
Common *common.Processor
Processor *processing.Processor
HTTPClient *MockHTTPClient
TypeConverter *typeutils.Converter
EmailSender email.Sender
State *state.State
Common *common.Processor
Processor *processing.Processor
HTTPClient *MockHTTPClient
TypeConverter *typeutils.Converter
EmailSender email.Sender
TransportController transport.Controller
}
func SetupTestStructs(
@ -56,6 +60,7 @@ func SetupTestStructs(
db := NewTestDB(&state)
state.DB = db
state.Actions = actions.New(db, &state.Workers)
storage := NewInMemoryStorage()
state.Storage = storage
@ -89,6 +94,7 @@ func SetupTestStructs(
processor := processing.NewProcessor(
cleaner.New(&state),
subscriptions.New(&state, transportController, typeconverter),
typeconverter,
federator,
oauthServer,
@ -105,12 +111,13 @@ func SetupTestStructs(
StandardStorageSetup(storage, rMediaPath)
return &TestStructs{
State: &state,
Common: &common,
Processor: processor,
HTTPClient: httpClient,
TypeConverter: typeconverter,
EmailSender: emailSender,
State: &state,
Common: &common,
Processor: processor,
HTTPClient: httpClient,
TypeConverter: typeconverter,
EmailSender: emailSender,
TransportController: transportController,
}
}

124
testrig/transportcontroller.go
View File

@ -41,6 +41,8 @@
const (
applicationJSON = "application/json"
applicationActivityJSON = "application/activity+json"
textCSV = "text/csv"
textPlain = "text/plain"
)
// NewTestTransportController returns a test transport controller with the given http client.
@ -101,6 +103,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
responseBytes = []byte(`{"error":"404 not found"}`)
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
extraHeaders = make(map[string]string, 0)
reqURLString = req.URL.String()
)
@ -124,11 +127,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
} else if strings.Contains(reqURLString, ".well-known/webfinger") {
responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
} else if strings.Contains(reqURLString, ".weird-webfinger-location/webfinger") {
responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
} else if strings.Contains(reqURLString, ".well-known/host-meta") {
responseCode, responseBytes, responseContentType, responseContentLength = HostMetaResponse(req)
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = HostMetaResponse(req)
} else if strings.Contains(reqURLString, "lists.example.org") {
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = DomainPermissionSubscriptionResponse(req)
} else if note, ok := mockHTTPClient.TestRemoteStatuses[reqURLString]; ok {
// the request is for a note that we have stored
noteI, err := streams.Serialize(note)
@ -239,14 +244,23 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
}
log.Debugf(nil, "returning response %s", string(responseBytes))
reader := bytes.NewReader(responseBytes)
readCloser := io.NopCloser(reader)
header := http.Header{
"Content-Type": {responseContentType},
}
for k, v := range extraHeaders {
header.Add(k, v)
}
return &http.Response{
Request: req,
StatusCode: responseCode,
Body: readCloser,
ContentLength: int64(responseContentLength),
Header: http.Header{"Content-Type": {responseContentType}},
Header: header,
}, nil
}
@ -261,7 +275,13 @@ func (m *MockHTTPClient) DoSigned(req *http.Request, sign httpclient.SignFunc) (
return m.do(req)
}
func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
func HostMetaResponse(req *http.Request) (
responseCode int,
responseBytes []byte,
responseContentType string,
responseContentLength int,
extraHeaders map[string]string,
) {
var hm *apimodel.HostMeta
if req.URL.String() == "https://misconfigured-instance.com/.well-known/host-meta" {
@ -297,7 +317,13 @@ func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte
return
}
func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
func WebfingerResponse(req *http.Request) (
responseCode int,
responseBytes []byte,
responseContentType string,
responseContentLength int,
extraHeaders map[string]string,
) {
var wfr *apimodel.WellKnownResponse
switch req.URL.String() {
@ -410,3 +436,89 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt
responseContentLength = len(wfrJSON)
return
}
func DomainPermissionSubscriptionResponse(req *http.Request) (
responseCode int,
responseBytes []byte,
responseContentType string,
responseContentLength int,
extraHeaders map[string]string,
) {
const (
csvResp = `#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate
bumfaces.net,suspend,false,false,big jerks,false
peepee.poopoo,suspend,false,false,harassment,false
nothanks.com,suspend,false,false,,false`
csvRespETag = "bigbums6969"
textResp = `bumfaces.net
peepee.poopoo
nothanks.com`
textRespETag = "this is a legit etag i swear"
jsonResp = `[
{
"domain": "bumfaces.net",
"suspended_at": "2020-05-13T13:29:12.000Z",
"public_comment": "big jerks"
},
{
"domain": "peepee.poopoo",
"suspended_at": "2020-05-13T13:29:12.000Z",
"public_comment": "harassment"
},
{
"domain": "nothanks.com",
"suspended_at": "2020-05-13T13:29:12.000Z"
}
]`
jsonRespETag = "don't modify me daddy"
)
switch req.URL.String() {
case "https://lists.example.org/baddies.csv":
extraHeaders = map[string]string{"ETag": csvRespETag}
if req.Header.Get("If-None-Match") == csvRespETag {
// Cached.
responseCode = http.StatusNotModified
} else {
responseBytes = []byte(csvResp)
responseContentType = textCSV
responseCode = http.StatusOK
}
responseContentLength = len(responseBytes)
case "https://lists.example.org/baddies.txt":
extraHeaders = map[string]string{"ETag": textRespETag}
if req.Header.Get("If-None-Match") == textRespETag {
// Cached.
responseCode = http.StatusNotModified
} else {
responseBytes = []byte(textResp)
responseContentType = textPlain
responseCode = http.StatusOK
}
responseContentLength = len(responseBytes)
case "https://lists.example.org/baddies.json":
extraHeaders = map[string]string{"ETag": jsonRespETag}
if req.Header.Get("If-None-Match") == jsonRespETag {
// Cached.
responseCode = http.StatusNotModified
} else {
responseBytes = []byte(jsonResp)
responseContentType = applicationJSON
responseCode = http.StatusOK
}
responseContentLength = len(responseBytes)
default:
responseCode = http.StatusNotFound
responseBytes = []byte(`{"error":"not found"}`)
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
}
return
}