[feature] Dereference remote mentions when the account is not already known (#442)

* remove mention util function from db

* add ParseMentionFunc to gtsmodel

* add parseMentionFunc to processor

* refactor search to simplify it a bit

* add parseMentionFunc to account

* add parseMentionFunc to status

* some renaming for clarity

* test dereference of unknown mentioned account

internal/api/client/status/status_test.go

@@ -19,8 +19,18 @@
 package status_test
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/activity/pub"
+	"github.com/superseriousbusiness/activity/streams"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -73,7 +83,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db), suite.storage, suite.mediaManager)
 	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
 	suite.statusModule = status.New(suite.processor).(*status.Module)
@@ -85,3 +95,89 @@ func (suite *StatusStandardTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 	testrig.StandardStorageTeardown(suite.storage)
 }
+
+func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient {
+	remoteAccount := suite.testAccounts["remote_account_1"]
+	remoteAccountNamestring := fmt.Sprintf("acct:%s@%s", remoteAccount.Username, remoteAccount.Domain)
+	remoteAccountWebfingerURI := fmt.Sprintf("https://%s/.well-known/webfinger?resource=%s", remoteAccount.Domain, remoteAccountNamestring)
+
+	fmt.Println(remoteAccountWebfingerURI)
+
+	httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+
+		// respond correctly to a webfinger lookup
+		if req.URL.String() == remoteAccountWebfingerURI {
+			responseJson := fmt.Sprintf(`
+			{
+				"subject": "%s",
+				"aliases": [
+				  "%s",
+				  "%s"
+				],
+				"links": [
+				  {
+					"rel": "http://webfinger.net/rel/profile-page",
+					"type": "text/html",
+					"href": "%s"
+				  },
+				  {
+					"rel": "self",
+					"type": "application/activity+json",
+					"href": "%s"
+				  }
+				]
+			}`, remoteAccountNamestring, remoteAccount.URI, remoteAccount.URL, remoteAccount.URL, remoteAccount.URI)
+			responseType := "application/json"
+
+			reader := bytes.NewReader([]byte(responseJson))
+			readCloser := io.NopCloser(reader)
+			response := &http.Response{
+				StatusCode:    200,
+				Body:          readCloser,
+				ContentLength: int64(len(responseJson)),
+				Header: http.Header{
+					"content-type": {responseType},
+				},
+			}
+			return response, nil
+		}
+
+		// respond correctly to an account dereference
+		if req.URL.String() == remoteAccount.URI {
+			satanAS, err := suite.tc.AccountToAS(context.Background(), remoteAccount)
+			if err != nil {
+				panic(err)
+			}
+
+			satanI, err := streams.Serialize(satanAS)
+			if err != nil {
+				panic(err)
+			}
+			satanJson, err := json.Marshal(satanI)
+			if err != nil {
+				panic(err)
+			}
+			responseType := "application/activity+json"
+
+			reader := bytes.NewReader(satanJson)
+			readCloser := io.NopCloser(reader)
+			response := &http.Response{
+				StatusCode:    200,
+				Body:          readCloser,
+				ContentLength: int64(len(satanJson)),
+				Header: http.Header{
+					"content-type": {responseType},
+				},
+			}
+			return response, nil
+		}
+
+		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
+		return &http.Response{
+			StatusCode: 200,
+			Body:       r,
+		}, nil
+	})
+
+	return httpClient
+}

internal/api/client/status/statuscreate_test.go

@@ -29,7 +29,6 @@ import (
 	"testing"
 
 	"github.com/gin-gonic/gin"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
@@ -85,26 +84,69 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 	result := recorder.Result()
 	defer result.Body.Close()
 	b, err := ioutil.ReadAll(result.Body)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	statusReply := &model.Status{}
 	err = json.Unmarshal(b, statusReply)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
-	assert.Equal(suite.T(), "hello hello", statusReply.SpoilerText)
+	suite.Equal("hello hello", statusReply.SpoilerText)
-	assert.Equal(suite.T(), "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", statusReply.Content)
+	suite.Equal("<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", statusReply.Content)
-	assert.True(suite.T(), statusReply.Sensitive)
+	suite.True(statusReply.Sensitive)
-	assert.Equal(suite.T(), model.VisibilityPrivate, statusReply.Visibility) // even though we set this status to mutuals only, it should serialize to private, because the mastodon api has no idea about mutuals_only
+	suite.Equal(model.VisibilityPrivate, statusReply.Visibility) // even though we set this status to mutuals only, it should serialize to private, because the mastodon api has no idea about mutuals_only
-	assert.Len(suite.T(), statusReply.Tags, 1)
+	suite.Len(statusReply.Tags, 1)
-	assert.Equal(suite.T(), model.Tag{
+	suite.Equal(model.Tag{
 		Name: "helloworld",
 		URL:  "http://localhost:8080/tags/helloworld",
 	}, statusReply.Tags[0])
 
 	gtsTag := &gtsmodel.Tag{}
 	err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "helloworld"}}, gtsTag)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
-	assert.Equal(suite.T(), statusReply.Account.ID, gtsTag.FirstSeenFromAccountID)
+	suite.Equal(statusReply.Account.ID, gtsTag.FirstSeenFromAccountID)
+}
+
+// mention an account that is not yet known to the instance -- it should be looked up and put in the db
+func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
+
+	// first remove remote account 1 from the database so it gets looked up again
+	remoteAccount := suite.testAccounts["remote_account_1"]
+	if err := suite.db.DeleteByID(context.Background(), remoteAccount.ID, &gtsmodel.Account{}); err != nil {
+		panic(err)
+	}
+
+	t := suite.testTokens["local_account_1"]
+	oauthToken := oauth.DBTokenToToken(t)
+
+	// setup
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+	ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
+	ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
+	ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
+	ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("accept", "application/json")
+	ctx.Request.Form = url.Values{
+		"status":     {"hello @foss_satan@fossbros-anonymous.io"},
+		"visibility": {string(model.VisibilityPublic)},
+	}
+	suite.statusModule.StatusCreatePOSTHandler(ctx)
+
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+
+	statusReply := &model.Status{}
+	err = json.Unmarshal(b, statusReply)
+	suite.NoError(err)
+
+	// if the status is properly formatted, that means the account has been put in the db
+	suite.Equal("<p>hello <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span></p>", statusReply.Content)
+	suite.Equal(model.VisibilityPublic, statusReply.Visibility)
 }
 
 func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() {
@@ -134,13 +176,13 @@ func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() {
 	result := recorder.Result()
 	defer result.Body.Close()
 	b, err := ioutil.ReadAll(result.Body)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	statusReply := &model.Status{}
 	err = json.Unmarshal(b, statusReply)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
-	assert.Equal(suite.T(), "<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let's see........<br><br><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\">docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br><br><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br><br>(tobi remember to pull the docker image challenge)</p>", statusReply.Content)
+	suite.Equal("<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let's see........<br><br><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\">docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br><br><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br><br>(tobi remember to pull the docker image challenge)</p>", statusReply.Content)
 }
 
 func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
@@ -167,22 +209,22 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
 	result := recorder.Result()
 	defer result.Body.Close()
 	b, err := ioutil.ReadAll(result.Body)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	statusReply := &model.Status{}
 	err = json.Unmarshal(b, statusReply)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
-	assert.Equal(suite.T(), "", statusReply.SpoilerText)
+	suite.Equal("", statusReply.SpoilerText)
-	assert.Equal(suite.T(), "<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:<br>here's an emoji that isn't in the db: :test_emoji:</p>", statusReply.Content)
+	suite.Equal("<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:<br>here's an emoji that isn't in the db: :test_emoji:</p>", statusReply.Content)
 
-	assert.Len(suite.T(), statusReply.Emojis, 1)
+	suite.Len(statusReply.Emojis, 1)
 	apiEmoji := statusReply.Emojis[0]
 	gtsEmoji := testrig.NewTestEmojis()["rainbow"]
 
-	assert.Equal(suite.T(), gtsEmoji.Shortcode, apiEmoji.Shortcode)
+	suite.Equal(gtsEmoji.Shortcode, apiEmoji.Shortcode)
-	assert.Equal(suite.T(), gtsEmoji.ImageURL, apiEmoji.URL)
+	suite.Equal(gtsEmoji.ImageURL, apiEmoji.URL)
-	assert.Equal(suite.T(), gtsEmoji.ImageStaticURL, apiEmoji.StaticURL)
+	suite.Equal(gtsEmoji.ImageStaticURL, apiEmoji.StaticURL)
 }
 
 // Try to reply to a status that doesn't exist
@@ -213,8 +255,8 @@ func (suite *StatusCreateTestSuite) TestReplyToNonexistentStatus() {
 	result := recorder.Result()
 	defer result.Body.Close()
 	b, err := ioutil.ReadAll(result.Body)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
-	assert.Equal(suite.T(), `{"error":"bad request"}`, string(b))
+	suite.Equal(`{"error":"bad request"}`, string(b))
 }
 
 // Post a reply to the status of a local user that allows replies.
@@ -243,19 +285,19 @@ func (suite *StatusCreateTestSuite) TestReplyToLocalStatus() {
 	result := recorder.Result()
 	defer result.Body.Close()
 	b, err := ioutil.ReadAll(result.Body)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	statusReply := &model.Status{}
 	err = json.Unmarshal(b, statusReply)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
-	assert.Equal(suite.T(), "", statusReply.SpoilerText)
+	suite.Equal("", statusReply.SpoilerText)
-	assert.Equal(suite.T(), fmt.Sprintf("<p>hello <span class=\"h-card\"><a href=\"http://localhost:8080/@%s\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>%s</span></a></span> this reply should work!</p>", testrig.NewTestAccounts()["local_account_2"].Username, testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content)
+	suite.Equal(fmt.Sprintf("<p>hello <span class=\"h-card\"><a href=\"http://localhost:8080/@%s\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>%s</span></a></span> this reply should work!</p>", testrig.NewTestAccounts()["local_account_2"].Username, testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content)
-	assert.False(suite.T(), statusReply.Sensitive)
+	suite.False(statusReply.Sensitive)
-	assert.Equal(suite.T(), model.VisibilityPublic, statusReply.Visibility)
+	suite.Equal(model.VisibilityPublic, statusReply.Visibility)
-	assert.Equal(suite.T(), testrig.NewTestStatuses()["local_account_2_status_1"].ID, statusReply.InReplyToID)
+	suite.Equal(testrig.NewTestStatuses()["local_account_2_status_1"].ID, statusReply.InReplyToID)
-	assert.Equal(suite.T(), testrig.NewTestAccounts()["local_account_2"].ID, statusReply.InReplyToAccountID)
+	suite.Equal(testrig.NewTestAccounts()["local_account_2"].ID, statusReply.InReplyToAccountID)
-	assert.Len(suite.T(), statusReply.Mentions, 1)
+	suite.Len(statusReply.Mentions, 1)
 }
 
 // Take a media file which is currently not associated with a status, and attach it to a new status.
@@ -286,33 +328,33 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
 	result := recorder.Result()
 	defer result.Body.Close()
 	b, err := ioutil.ReadAll(result.Body)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	statusResponse := &model.Status{}
 	err = json.Unmarshal(b, statusResponse)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
-	assert.Equal(suite.T(), "", statusResponse.SpoilerText)
+	suite.Equal("", statusResponse.SpoilerText)
-	assert.Equal(suite.T(), "<p>here's an image attachment</p>", statusResponse.Content)
+	suite.Equal("<p>here's an image attachment</p>", statusResponse.Content)
-	assert.False(suite.T(), statusResponse.Sensitive)
+	suite.False(statusResponse.Sensitive)
-	assert.Equal(suite.T(), model.VisibilityPublic, statusResponse.Visibility)
+	suite.Equal(model.VisibilityPublic, statusResponse.Visibility)
 
 	// there should be one media attachment
-	assert.Len(suite.T(), statusResponse.MediaAttachments, 1)
+	suite.Len(statusResponse.MediaAttachments, 1)
 
 	// get the updated media attachment from the database
 	gtsAttachment, err := suite.db.GetAttachmentByID(context.Background(), statusResponse.MediaAttachments[0].ID)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	// convert it to a api attachment
 	gtsAttachmentAsapi, err := suite.tc.AttachmentToAPIAttachment(context.Background(), gtsAttachment)
-	assert.NoError(suite.T(), err)
+	suite.NoError(err)
 
 	// compare it with what we have now
-	assert.EqualValues(suite.T(), statusResponse.MediaAttachments[0], gtsAttachmentAsapi)
+	suite.EqualValues(statusResponse.MediaAttachments[0], gtsAttachmentAsapi)
 
 	// the status id of the attachment should now be set to the id of the status we just created
-	assert.Equal(suite.T(), statusResponse.ID, gtsAttachment.StatusID)
+	suite.Equal(statusResponse.ID, gtsAttachment.StatusID)
 }
 
 func TestStatusCreateTestSuite(t *testing.T) {

internal/db/bundb/bundb.go

@@ -382,86 +382,6 @@ func tweakConnectionValues(sqldb *sql.DB) {
 	CONVERSION FUNCTIONS
 */
 
-// TODO: move these to the type converter, it's bananas that they're here and not there
-
-func (ps *bunDBService) MentionStringsToMentions(ctx context.Context, targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error) {
-	ogAccount := &gtsmodel.Account{}
-	if err := ps.conn.NewSelect().Model(ogAccount).Where("id = ?", originAccountID).Scan(ctx); err != nil {
-		return nil, err
-	}
-
-	menchies := []*gtsmodel.Mention{}
-	for _, a := range targetAccounts {
-		// A mentioned account looks like "@test@example.org" or just "@test" for a local account
-		// -- we can guarantee this from the regex that targetAccounts should have been derived from.
-		// But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
-
-		// 1.  trim off the first @
-		t := strings.TrimPrefix(a, "@")
-
-		// 2. split the username and domain
-		s := strings.Split(t, "@")
-
-		// 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
-		var local bool
-		switch len(s) {
-		case 1:
-			local = true
-		case 2:
-			local = false
-		default:
-			return nil, fmt.Errorf("mentioned account format '%s' was not valid", a)
-		}
-
-		var username, domain string
-		username = s[0]
-		if !local {
-			domain = s[1]
-		}
-
-		// 4. check we now have a proper username and domain
-		if username == "" || (!local && domain == "") {
-			return nil, fmt.Errorf("username or domain for '%s' was nil", a)
-		}
-
-		// okay we're good now, we can start pulling accounts out of the database
-		mentionedAccount := &gtsmodel.Account{}
-		var err error
-
-		// match username + account, case insensitive
-		if local {
-			// local user -- should have a null domain
-			err = ps.conn.NewSelect().Model(mentionedAccount).Where("LOWER(?) = LOWER(?)", bun.Ident("username"), username).Where("? IS NULL", bun.Ident("domain")).Scan(ctx)
-		} else {
-			// remote user -- should have domain defined
-			err = ps.conn.NewSelect().Model(mentionedAccount).Where("LOWER(?) = LOWER(?)", bun.Ident("username"), username).Where("LOWER(?) = LOWER(?)", bun.Ident("domain"), domain).Scan(ctx)
-		}
-
-		if err != nil {
-			if err == sql.ErrNoRows {
-				// no result found for this username/domain so just don't include it as a mencho and carry on about our business
-				logrus.Debugf("no account found with username '%s' and domain '%s', skipping it", username, domain)
-				continue
-			}
-			// a serious error has happened so bail
-			return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err)
-		}
-
-		// id, createdAt and updatedAt will be populated by the db, so we have everything we need!
-		menchies = append(menchies, &gtsmodel.Mention{
-			StatusID:         statusID,
-			OriginAccountID:  ogAccount.ID,
-			OriginAccountURI: ogAccount.URI,
-			TargetAccountID:  mentionedAccount.ID,
-			NameString:       a,
-			TargetAccountURI: mentionedAccount.URI,
-			TargetAccountURL: mentionedAccount.URL,
-			OriginAccount:    mentionedAccount,
-		})
-	}
-	return menchies, nil
-}
-
 func (ps *bunDBService) TagStringsToTags(ctx context.Context, tags []string, originAccountID string) ([]*gtsmodel.Tag, error) {
 	protocol := viper.GetString(config.Keys.Protocol)
 	host := viper.GetString(config.Keys.Host)

internal/db/db.go

@@ -48,15 +48,6 @@ type DB interface {
 		USEFUL CONVERSION FUNCTIONS
 	*/
 
-	// MentionStringsToMentions takes a slice of deduplicated, lowercase account names in the form "@test@whatever.example.org" for a remote account,
-	// or @test for a local account, which have been mentioned in a status.
-	// It takes the id of the account that wrote the status, and the id of the status itself, and then
-	// checks in the database for the mentioned accounts, and returns a slice of mentions generated based on the given parameters.
-	//
-	// Note: this func doesn't/shouldn't do any manipulation of the accounts in the DB, it's just for checking
-	// if they exist in the db and conveniently returning them if they do.
-	MentionStringsToMentions(ctx context.Context, targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error)
-
 	// TagStringsToTags takes a slice of deduplicated, lowercase tags in the form "somehashtag", which have been
 	// used in a status. It takes the id of the account that wrote the status, and the id of the status itself, and then
 	// returns a slice of *model.Tag corresponding to the given tags. If the tag already exists in database, that tag

internal/gtsmodel/mention.go

@@ -18,7 +18,10 @@
 
 package gtsmodel
 
-import "time"
+import (
+	"context"
+	"time"
+)
 
 // Mention refers to the 'tagging' or 'mention' of a user within a status.
 type Mention struct {
@@ -57,3 +60,15 @@ type Mention struct {
 	TargetAccountURL string `validate:"-" bun:"-"`
 	// A pointer to the gtsmodel account of the mentioned account.
 }
+
+// ParseMentionFunc describes a function that takes a lowercase account name
+// in the form "@test@whatever.example.org" for a remote account, or "@test"
+// for a local account, and returns a fully populated mention for that account,
+// with the given origin status ID and origin account ID.
+//
+// If the account is remote and not yet found in the database, then ParseMentionFunc
+// will try to webfinger the remote account and put it in the database before returning.
+//
+// Mentions generated by this function are not put in the database, that's still up to
+// the caller to do.
+type ParseMentionFunc func(ctx context.Context, targetAccount string, originAccountID string, statusID string) (*Mention, error)

internal/processing/account/account.go

@@ -87,10 +87,11 @@ type processor struct {
 	formatter     text.Formatter
 	db            db.DB
 	federator     federation.Federator
+	parseMention  gtsmodel.ParseMentionFunc
 }
 
 // New returns a new account processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor {
 	return &processor{
 		tc:            tc,
 		mediaManager:  mediaManager,
@@ -100,5 +101,6 @@ func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauth
 		formatter:     text.NewFormatter(db),
 		db:            db,
 		federator:     federator,
+		parseMention:  parseMention,
 	}
 }

internal/processing/account/account_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/account"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -88,7 +89,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
 	suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager)
 	suite.sentEmails = make(map[string]string)
 	suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
-	suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator)
+	suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator))
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
 }

internal/processing/account/update.go

@@ -199,10 +199,14 @@ func (p *processor) processNote(ctx context.Context, note string, accountID stri
 		return "", err
 	}
 
-	mentionStrings := util.DeriveMentionsFromText(note)
+	mentionStrings := util.DeriveMentionNamesFromText(note)
-	mentions, err := p.db.MentionStringsToMentions(ctx, mentionStrings, accountID, "")
+	mentions := []*gtsmodel.Mention{}
-	if err != nil {
+	for _, mentionString := range mentionStrings {
-		return "", err
+		mention, err := p.parseMention(ctx, mentionString, accountID, "")
+		if err != nil {
+			continue
+		}
+		mentions = append(mentions, mention)
 	}
 
 	// TODO: support emojis in account notes

internal/processing/processor.go

@@ -267,12 +267,14 @@ func NewProcessor(
 	storage *kv.KVStore,
 	db db.DB,
 	emailSender email.Sender) Processor {
+
 	fromClientAPI := make(chan messages.FromClientAPI, 1000)
 	fromFederator := make(chan messages.FromFederator, 1000)
+	parseMentionFunc := GetParseMentionFunc(db, federator)
 
-	statusProcessor := status.New(db, tc, fromClientAPI)
+	statusProcessor := status.New(db, tc, fromClientAPI, parseMentionFunc)
 	streamingProcessor := streaming.New(db, oauthServer)
-	accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator)
+	accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator, parseMentionFunc)
 	adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI)
 	mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage)
 	userProcessor := user.New(db, emailSender)

internal/processing/search.go

@@ -194,20 +194,15 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au
 	// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
 	if resolve {
 		// we're allowed to resolve it so let's try
-
 		// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
 		acctURI, err := p.federator.FingerRemoteAccount(ctx, authed.Account.Username, username, domain)
 		if err != nil {
 			// something went wrong doing the webfinger lookup so we can't process the request
-			return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err)
+			return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
 		}
 
-		// we don't have it locally so try and dereference it
+		// return the attempt to get the remove account
-		account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
+		return p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
-		if err != nil {
-			return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err)
-		}
-		return account, nil
 	}
 
 	return nil, nil

internal/processing/status/status.go

@@ -74,15 +74,17 @@ type processor struct {
 	filter        visibility.Filter
 	formatter     text.Formatter
 	fromClientAPI chan messages.FromClientAPI
+	parseMention  gtsmodel.ParseMentionFunc
 }
 
 // New returns a new status processor.
-func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI, parseMention gtsmodel.ParseMentionFunc) Processor {
 	return &processor{
 		tc:            tc,
 		db:            db,
 		filter:        visibility.NewFilter(db),
 		formatter:     text.NewFormatter(db),
 		fromClientAPI: fromClientAPI,
+		parseMention:  parseMention,
 	}
 }

internal/processing/status/status_test.go

@@ -19,11 +19,16 @@
 package status_test
 
 import (
+	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/processing"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/status"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
@@ -32,6 +37,10 @@ type StatusStandardTestSuite struct {
 	suite.Suite
 	db                db.DB
 	typeConverter     typeutils.TypeConverter
+	tc                transport.Controller
+	storage           *kv.KVStore
+	mediaManager      media.Manager
+	federator         federation.Federator
 	fromClientAPIChan chan messages.FromClientAPI
 
 	// standard suite models
@@ -62,17 +71,23 @@ func (suite *StatusStandardTestSuite) SetupSuite() {
 }
 
 func (suite *StatusStandardTestSuite) SetupTest() {
-	testrig.InitTestLog()
 	testrig.InitTestConfig()
+	testrig.InitTestLog()
 
 	suite.db = testrig.NewTestDB()
 	suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
 	suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
-	suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan)
+	suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	suite.storage = testrig.NewTestStorage()
+	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager)
+	suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan, processing.GetParseMentionFunc(suite.db, suite.federator))
 
-	testrig.StandardDBSetup(suite.db, nil)
+	testrig.StandardDBSetup(suite.db, suite.testAccounts)
+	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
 }
 
 func (suite *StatusStandardTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
 }

internal/processing/status/util.go

@@ -23,10 +23,10 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/text"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
@@ -192,27 +192,30 @@ func (p *processor) ProcessLanguage(ctx context.Context, form *apimodel.Advanced
 }
 
 func (p *processor) ProcessMentions(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
-	menchies := []string{}
+	mentionedAccountNames := util.DeriveMentionNamesFromText(form.Status)
-	gtsMenchies, err := p.db.MentionStringsToMentions(ctx, util.DeriveMentionsFromText(form.Status), accountID, status.ID)
+	mentions := []*gtsmodel.Mention{}
-	if err != nil {
+	mentionIDs := []string{}
-		return fmt.Errorf("error generating mentions from status: %s", err)
-	}
-	for _, menchie := range gtsMenchies {
-		menchieID, err := id.NewRandomULID()
-		if err != nil {
-			return err
-		}
-		menchie.ID = menchieID
 
-		if err := p.db.Put(ctx, menchie); err != nil {
+	for _, mentionedAccountName := range mentionedAccountNames {
-			return fmt.Errorf("error putting mentions in db: %s", err)
+		gtsMention, err := p.parseMention(ctx, mentionedAccountName, accountID, status.ID)
+		if err != nil {
+			logrus.Errorf("ProcessMentions: error parsing mention %s from status: %s", mentionedAccountName, err)
+			continue
 		}
-		menchies = append(menchies, menchie.ID)
+
+		if err := p.db.Put(ctx, gtsMention); err != nil {
+			logrus.Errorf("ProcessMentions: error putting mention in db: %s", err)
+		}
+
+		mentions = append(mentions, gtsMention)
+		mentionIDs = append(mentionIDs, gtsMention.ID)
 	}
+
 	// add full populated gts menchies to the status for passing them around conveniently
-	status.Mentions = gtsMenchies
+	status.Mentions = mentions
 	// add just the ids of the mentioned accounts to the status for putting in the db
-	status.MentionIDs = menchies
+	status.MentionIDs = mentionIDs
+
 	return nil
 }
 

internal/processing/util.go

@@ -0,0 +1,151 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+)
+
+func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.ParseMentionFunc {
+	return func(ctx context.Context, targetAccount string, originAccountID string, statusID string) (*gtsmodel.Mention, error) {
+		// get the origin account first since we'll need it to create the mention
+		originAccount, err := dbConn.GetAccountByID(ctx, originAccountID)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't get mention origin account with id %s", originAccountID)
+		}
+
+		// A mentioned account looks like "@test@example.org" or just "@test" for a local account
+		// -- we can guarantee this from the regex that targetAccounts should have been derived from.
+		// But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
+
+		// 1.  trim off the first @
+		trimmed := strings.TrimPrefix(targetAccount, "@")
+
+		// 2. split the username and domain
+		split := strings.Split(trimmed, "@")
+
+		// 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
+
+		var local bool
+		switch len(split) {
+		case 1:
+			local = true
+		case 2:
+			local = false
+		default:
+			return nil, fmt.Errorf("mentioned account format '%s' was not valid", targetAccount)
+		}
+
+		var username, domain string
+		username = split[0]
+		if !local {
+			domain = split[1]
+		}
+
+		// 4. check we now have a proper username and domain
+		if username == "" || (!local && domain == "") {
+			return nil, fmt.Errorf("username or domain for '%s' was nil", targetAccount)
+		}
+
+		var mentionedAccount *gtsmodel.Account
+
+		if local {
+			localAccount, err := dbConn.GetLocalAccountByUsername(ctx, username)
+			if err != nil {
+				return nil, err
+			}
+			mentionedAccount = localAccount
+		} else {
+			remoteAccount := &gtsmodel.Account{}
+
+			where := []db.Where{
+				{
+					Key:             "username",
+					Value:           username,
+					CaseInsensitive: true,
+				},
+				{
+					Key:             "domain",
+					Value:           domain,
+					CaseInsensitive: true,
+				},
+			}
+
+			err := dbConn.GetWhere(ctx, where, remoteAccount)
+			if err == nil {
+				// the account was already in the database
+				mentionedAccount = remoteAccount
+			} else {
+				// we couldn't get it from the database
+				if err != db.ErrNoEntries {
+					// a serious error has happened so bail
+					return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err)
+				}
+
+				// We just don't have the account, so try webfingering it.
+				//
+				// If the mention originates from our instance we should use the username of the origin account to do the dereferencing,
+				// otherwise we should just use our instance account (that is, provide an empty string), since obviously we can't use
+				// a remote account to do remote dereferencing!
+				var fingeringUsername string
+				if originAccount.Domain == "" {
+					fingeringUsername = originAccount.Username
+				}
+
+				acctURI, err := federator.FingerRemoteAccount(ctx, fingeringUsername, username, domain)
+				if err != nil {
+					// something went wrong doing the webfinger lookup so we can't process the request
+					return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
+				}
+
+				resolvedAccount, err := federator.GetRemoteAccount(ctx, fingeringUsername, acctURI, true, true)
+				if err != nil {
+					return nil, fmt.Errorf("error dereferencing account with uri %s: %s", acctURI.String(), err)
+				}
+
+				// we were able to resolve it!
+				mentionedAccount = resolvedAccount
+			}
+		}
+
+		mentionID, err := id.NewRandomULID()
+		if err != nil {
+			return nil, err
+		}
+
+		return &gtsmodel.Mention{
+			ID:               mentionID,
+			StatusID:         statusID,
+			OriginAccountID:  originAccount.ID,
+			OriginAccountURI: originAccount.URI,
+			TargetAccountID:  mentionedAccount.ID,
+			NameString:       targetAccount,
+			TargetAccountURI: mentionedAccount.URI,
+			TargetAccountURL: mentionedAccount.URL,
+			OriginAccount:    mentionedAccount,
+		}, nil
+	}
+}

internal/util/statustools.go

@@ -25,13 +25,11 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/regexes"
 )
 
-// DeriveMentionsFromText takes a plaintext (ie., not html-formatted) text,
+// DeriveMentionNamesFromText takes a plaintext (ie., not html-formatted) text,
-// and applies a regex to it to return a deduplicated list of accounts
+// and applies a regex to it to return a deduplicated list of account names
-// mentioned in that text.
+// mentioned in that text, in the format "@user@example.org" or "@username" for
-//
+// local users.
-// It will look for fully-qualified account names in the form "@user@example.org".
+func DeriveMentionNamesFromText(text string) []string {
-// or the form "@username" for local users.
-func DeriveMentionsFromText(text string) []string {
 	mentionedAccounts := []string{}
 	for _, m := range regexes.MentionFinder.FindAllStringSubmatch(text, -1) {
 		mentionedAccounts = append(mentionedAccounts, m[1])

internal/util/statustools_test.go

@@ -37,7 +37,7 @@ https://localhost:8080/@the_mighty_zork/statuses/01FGVP55XMF2K6316MQRX6PFG1
 
 that link shouldn't come out formatted as a mention!`
 
-	menchies := util.DeriveMentionsFromText(statusText)
+	menchies := util.DeriveMentionNamesFromText(statusText)
 	suite.Empty(menchies)
 }
 
@@ -56,7 +56,7 @@ func (suite *StatusTestSuite) TestDeriveMentionsOK() {
 
 	`
 
-	menchies := util.DeriveMentionsFromText(statusText)
+	menchies := util.DeriveMentionNamesFromText(statusText)
 	assert.Len(suite.T(), menchies, 6)
 	assert.Equal(suite.T(), "@dumpsterqueer@example.org", menchies[0])
 	assert.Equal(suite.T(), "@someone_else@testing.best-horse.com", menchies[1])
@@ -68,7 +68,7 @@ func (suite *StatusTestSuite) TestDeriveMentionsOK() {
 
 func (suite *StatusTestSuite) TestDeriveMentionsEmpty() {
 	statusText := ``
-	menchies := util.DeriveMentionsFromText(statusText)
+	menchies := util.DeriveMentionNamesFromText(statusText)
 	assert.Len(suite.T(), menchies, 0)
 }
 
@@ -126,7 +126,7 @@ func (suite *StatusTestSuite) TestDeriveMultiple() {
 
 	Text`
 
-	ms := util.DeriveMentionsFromText(statusText)
+	ms := util.DeriveMentionNamesFromText(statusText)
 	hs := util.DeriveHashtagsFromText(statusText)
 	es := util.DeriveEmojisFromText(statusText)