diff --git a/internal/api/client/admin/domainpermissiondraftcreate.go b/internal/api/client/admin/domainpermissiondraftcreate.go index ec94f947b..a36c2ca33 100644 --- a/internal/api/client/admin/domainpermissiondraftcreate.go +++ b/internal/api/client/admin/domainpermissiondraftcreate.go @@ -149,6 +149,7 @@ func (m *Module) DomainPermissionDraftsPOSTHandler(c *gin.Context) { form.Obfuscate, form.PublicComment, form.PrivateComment, + "", // No subscription ID. ) if errWithCode != nil { apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) diff --git a/internal/processing/admin/domainpermissiondraft.go b/internal/processing/admin/domainpermissiondraft.go index 0dc17a45a..62d77c7bc 100644 --- a/internal/processing/admin/domainpermissiondraft.go +++ b/internal/processing/admin/domainpermissiondraft.go @@ -125,6 +125,7 @@ func (p *Processor) DomainPermissionDraftCreate( obfuscate bool, publicComment string, privateComment string, + subscriptionID string, ) (*apimodel.DomainPermission, gtserror.WithCode) { permDraft := >smodel.DomainPermissionDraft{ ID: id.NewULID(), @@ -135,6 +136,7 @@ func (p *Processor) DomainPermissionDraftCreate( PrivateComment: privateComment, PublicComment: publicComment, Obfuscate: &obfuscate, + SubscriptionID: subscriptionID, } if err := p.state.DB.PutDomainPermissionDraft(ctx, permDraft); err != nil { diff --git a/internal/subscriptions/subscriptions.go b/internal/subscriptions/subscriptions.go new file mode 100644 index 000000000..73bfa748f --- /dev/null +++ b/internal/subscriptions/subscriptions.go @@ -0,0 +1,511 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subscriptions + +import ( + "context" + "encoding/csv" + "encoding/json" + "errors" + "io" + "slices" + "strings" + "time" + + "codeberg.org/gruf/go-kv" + + "github.com/miekg/dns" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/processing/admin" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +type Subscriptions struct { + state *state.State + transportController transport.Controller + tc *typeutils.Converter + admin *admin.Processor +} + +// ProcessDomainPermissionSubscriptions processes all domain permission +// subscriptions of the given permission type by, in turn, calling the +// URI of each subscription, parsing the result into a list of domain +// permissions, and creating (or skipping) each permission as appropriate. +func (s *Subscriptions) ProcessDomainPermissionSubscriptions( + ctx context.Context, + permType gtsmodel.DomainPermissionType, +) { + log.Info(ctx, "start") + + // Get permission subscriptions in priority order (highest -> lowest). + permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Real db error. + log.Error(ctx, err) + return + } + + if len(permSubs) == 0 { + // No subscriptions of this + // type, so nothing to do. + return + } + + // Get a transport using the instance account, + // we can reuse this for each HTTP call. + tsport, err := s.transportController.NewTransportForUsername(ctx, "") + if err != nil { + log.Error(ctx, err) + return + } + + for i, permSub := range permSubs { + // Higher priority permission subs = everything + // above this permission sub in the slice. + getHigherPrios := func() ([]*gtsmodel.DomainPermissionSubscription, error) { + return permSubs[:i], nil + } + + perms, err := s.ProcessDomainPermissionSubscription( + ctx, + permSub, + tsport, + getHigherPrios, + ) + if err != nil { + // Real db error. + log.Error(ctx, err) + return + } + + // Update count if necessary. + if count := len(perms); count != 0 { + permSub.Count = uint64(count) + } + + // Update this perm sub. + err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub) + if err != nil { + // Real db error. + log.Error(ctx, err) + return + } + } + + log.Info(ctx, "finished") +} + +// ProcessDomainPermissionSubscription processes one domain permission +// subscription by dereferencing the URI, parsing the response into a list +// of permissions, and for each discovered permission either creating an +// entry in the database, or ignoring it if it's excluded or already +// covered by a higher-priority subscription. +// +// On success, the slice of discovered DomainPermissions will be returned. +// In case of parsing error, or error on the remote side, permSub.Error +// will be updated with the calling/parsing error, and `nil, nil` will be +// returned. In case of an actual db error, `nil, err` will be returned and +// the caller should handle it. +// +// getHigherPrios should be a function for returning a slice of domain +// permission subscriptions with a higher priority than the given permSub. +func (s *Subscriptions) ProcessDomainPermissionSubscription( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + tsport transport.Transport, + getHigherPrios func() ([]*gtsmodel.DomainPermissionSubscription, error), +) ([]gtsmodel.DomainPermission, error) { + l := log. + WithContext(ctx). + WithFields(kv.Fields{ + {"permType", permSub.PermissionType.String()}, + {"permSubURI", permSub.URI}, + }...) + + // Set FetchedAt as we're + // going to attempt this now. + permSub.FetchedAt = time.Now() + + // Call the URI but don't force. + resp, err := tsport.DereferenceDomainPermissions( + ctx, permSub, false, + ) + if err != nil { + // Couldn't get this one, + // set error + return. + errStr := err.Error() + l.Warnf("couldn't dereference permSubURI: %+v", err) + permSub.Error = errStr + return nil, nil + } + + // If the permissions at URI weren't modified + // since last time, just update some metadata + // to indicate a successful fetch, and return. + if resp.Unmodified { + l.Debug("received 304 Not Modified from remote") + permSub.SuccessfullyFetchedAt = time.Now() + if permSub.ETag == "" && resp.ETag != "" { + // We didn't have an ETag before but + // we have one now: probably the remote + // added ETag support in the meantime. + permSub.ETag = resp.ETag + } + return nil, nil + } + + // At this point we know we got a 200 OK + // from the URI, so we've got a live body! + // Try to parse the body as a list of perms. + var perms []gtsmodel.DomainPermission + + switch permSub.ContentType { + + // text/csv + case gtsmodel.DomainPermSubContentTypeCSV: + perms, err = s.permsFromCSV(ctx, permSub.PermissionType, resp.Body) + + // application/json + case gtsmodel.DomainPermSubContentTypeJSON: + perms, err = s.permsFromJSON(ctx, permSub.PermissionType, resp.Body) + + // text/plain + case gtsmodel.DomainPermSubContentTypePlain: + perms, err = s.permsFromPlain(ctx, permSub.PermissionType, resp.Body) + } + + if err != nil { + // We retrieved the permissions from remote + // but we couldn't parse them into anything + // workable, or the connection died halfway + // through transfer, or some other annoyance. + // Just set error and return. + errStr := err.Error() + l.Warnf("couldn't parse results: %+v", err) + permSub.Error = errStr + return nil, nil + } + + higherPrios, err := getHigherPrios() + if err != nil { + // Proper db error. + return nil, err + } + + // Process each domain permission. + for _, perm := range perms { + l = l.WithField("domain", perm.GetDomain()) + if err := s.processDomainPermission( + ctx, + l, + perm, + permSub, + higherPrios, + ); err != nil { + // Proper db error. + return nil, err + } + } + + return perms, nil +} + +// processDomainPermission processes one domain permission +// discovered via a domain permission subscription's URI. +// +// Error will only be returned in case of an actual database +// error, else the error will be logged and nil returned. +func (s *Subscriptions) processDomainPermission( + ctx context.Context, + l log.Entry, + perm gtsmodel.DomainPermission, + permSub *gtsmodel.DomainPermissionSubscription, + higherPrios []*gtsmodel.DomainPermissionSubscription, +) error { + // If domain contains wildcard/obfuscation + // characters, we can't do anything with it. + domain := perm.GetDomain() + if strings.Contains(domain, "*") { + l.Warn("'*' char(s) in domain, skipping") + return nil + } + + // Basic validation. + if _, ok := dns.IsDomainName(domain); !ok { + l.Warn("invalid domain, skipping") + return nil + } + + // Convert to punycode. + domain, err := util.Punify(domain) + if err != nil { + l.Warnf("could not punify domain (%+v), skipping", err) + return nil + } + + // Domain looks good. Check if it's excluded. + excluded, err := s.state.DB.IsDomainPermissionExcluded(ctx, domain) + if err != nil { + // Proper db error. + return err + } + + if excluded { + l.Debug("domain is excluded, skipping") + return nil + } + + // Check if a permission already exists for + // this domain, and if it's covered already + // by a higher-priority subscription. + existingPerm, covered, err := s.existingCovered( + ctx, permSub.PermissionType, domain, higherPrios, + ) + if err != nil { + // Proper db error. + return err + } + + if covered { + l.Debug("domain is covered by a higher-priority subscription, skipping") + return nil + } + + existing := !util.IsNil(existingPerm) + switch { + + // No existing perm, create draft. + case !existing && *permSub.AsDraft: + _, err = s.admin.DomainPermissionDraftCreate( + ctx, + perm.GetCreatedByAccount(), + domain, + permSub.PermissionType, + util.PtrOrZero(perm.GetObfuscate()), + perm.GetPublicComment(), + permSub.URI, + permSub.ID, + ) + + // No existing perm, create straight up. + case !existing && !*permSub.AsDraft: + _, _, err = s.admin.DomainPermissionCreate( + ctx, + permSub.PermissionType, + perm.GetCreatedByAccount(), + domain, + util.PtrOrZero(perm.GetObfuscate()), + perm.GetPublicComment(), + permSub.URI, + permSub.ID, + ) + + // Exists but we should adopt/take it. + case existingPerm.GetSubscriptionID() != "" || *permSub.AdoptOrphans: + existingPerm.SetCreatedByAccountID(perm.GetCreatedByAccountID()) + existingPerm.SetCreatedByAccount(perm.GetCreatedByAccount()) + existingPerm.SetSubscriptionID(permSub.ID) + existingPerm.SetObfuscate(perm.GetObfuscate()) + existingPerm.SetPrivateComment(perm.GetPrivateComment()) + existingPerm.SetPublicComment(perm.GetPublicComment()) + + switch p := existingPerm.(type) { + case *gtsmodel.DomainBlock: + err = s.state.DB.UpdateDomainBlock(ctx, p) + case *gtsmodel.DomainAllow: + err = s.state.DB.UpdateDomainAllow(ctx, p) + } + + // Exists but we should leave it alone. + default: + l.Debug("domain is covered by a higher-priority subscription, skipping") + } + + if err != nil && !errors.Is(err, db.ErrAlreadyExists) { + // Proper db error. + return err + } + + return nil +} + +func (s *Subscriptions) permsFromCSV( + ctx context.Context, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + // Read body into memory as slice of CSV records. + records, err := csv.NewReader(body).ReadAll() + + // Whatever happened, we're + // done with the body now. + body.Close() + + // Check if error reading body. + if err != nil { + return nil, gtserror.NewfAt(3, "error decoding into csv: %w", err) + } + + // Convert CSV records to perms. + return s.tc.CSVToDomainPerms(ctx, records, permType) +} + +func (s *Subscriptions) permsFromJSON( + _ context.Context, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + var ( + dec = json.NewDecoder(body) + apiPerms = make([]*apimodel.DomainPermission, 0) + ) + + // Read body into memory as + // slice of domain permissions. + if err := dec.Decode(&apiPerms); err != nil { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "error decoding into json: %w", err) + } + + // Perform a secondary decode just to ensure we drained the + // entirety of the data source. Error indicates either extra + // trailing garbage, or multiple JSON values (invalid data). + if err := dec.Decode(&struct{}{}); err != io.EOF { + _ = body.Close() // ensure closed. + return nil, gtserror.NewfAt(3, "data remaining after json") + } + + // Done with body. + _ = body.Close() + + // Convert apimodel perms to barebones internal perms. + perms := make([]gtsmodel.DomainPermission, 0, len(apiPerms)) + for _, apiPerm := range apiPerms { + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + if permType == gtsmodel.DomainPermissionBlock { + perm = >smodel.DomainBlock{Domain: apiPerm.Domain.Domain} + } else { + perm = >smodel.DomainAllow{Domain: apiPerm.Domain.Domain} + } + + // Set remaining fields. + perm.SetPublicComment(apiPerm.PublicComment) + perm.SetObfuscate(&apiPerm.Obfuscate) + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +} + +func (s *Subscriptions) permsFromPlain( + _ context.Context, + permType gtsmodel.DomainPermissionType, + body io.ReadCloser, +) ([]gtsmodel.DomainPermission, error) { + // Read body into memory as bytes. + b, err := io.ReadAll(body) + + // Whatever happened, we're + // done with the body now. + body.Close() + + // Check if error reading body. + if err != nil { + return nil, gtserror.NewfAt(3, "error decoding into plain: %w", err) + } + + // Coerce to newline-separated list of domains. + domains := strings.Split(string(b), "\n") + + // Convert raw domains to permissions. + perms := make([]gtsmodel.DomainPermission, 0, len(domains)) + for _, domain := range domains { + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + if permType == gtsmodel.DomainPermissionBlock { + perm = >smodel.DomainBlock{Domain: domain} + } else { + perm = >smodel.DomainAllow{Domain: domain} + } + + // We're done. + perms = append(perms, perm) + } + + return nil, nil +} + +func (s *Subscriptions) existingCovered( + ctx context.Context, + permType gtsmodel.DomainPermissionType, + domain string, + higherPrios []*gtsmodel.DomainPermissionSubscription, +) ( + existingPerm gtsmodel.DomainPermission, + covered bool, + err error, +) { + // Check for existing permission of appropriate type. + var dbErr error + if permType == gtsmodel.DomainPermissionBlock { + existingPerm, dbErr = s.state.DB.GetDomainBlock(ctx, domain) + } else { + existingPerm, dbErr = s.state.DB.GetDomainAllow(ctx, domain) + } + if dbErr != nil && !errors.Is(dbErr, db.ErrNoEntries) { + // Real db error. + err = dbErr + return + } + + if util.IsNil(existingPerm) { + // Can't be covered if + // no existing perm. + return + } + + subscriptionID := existingPerm.GetSubscriptionID() + if subscriptionID == "" { + // Can't be covered if + // no subscription ID. + return + } + + // Covered if subscription ID is in the slice + // of higher-priority permission subscriptions. + covered = slices.ContainsFunc( + higherPrios, + func(permSub *gtsmodel.DomainPermissionSubscription) bool { + return permSub.ID == subscriptionID + }, + ) + + return +} diff --git a/internal/transport/derefdomainpermlist.go b/internal/transport/derefdomainpermlist.go new file mode 100644 index 000000000..b20e2bd2e --- /dev/null +++ b/internal/transport/derefdomainpermlist.go @@ -0,0 +1,123 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transport + +import ( + "context" + "io" + "net/http" + + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type DereferenceDomainPermissionsResp struct { + // Set only if response was 200 OK. + // It's up to the caller to close + // this when they're done with it. + Body io.ReadCloser + + // True if response + // was 304 Not Modified. + Unmodified bool + + // May be set + // if 200 or 304. + ETag string +} + +func (t *transport) DereferenceDomainPermissions( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + force bool, +) (*DereferenceDomainPermissionsResp, error) { + // Prepare new HTTP request to endpoint + req, err := http.NewRequestWithContext(ctx, "GET", permSub.URI, nil) + if err != nil { + return nil, err + } + + // Set basic auth header if necessary. + if permSub.FetchUsername != "" || permSub.FetchPassword != "" { + req.SetBasicAuth(permSub.FetchUsername, permSub.FetchPassword) + } + + // Set relevant Accept headers. + // Allow fallback in case target doesn't + // negotiate content type correctly. + req.Header.Add("Accept-Charset", "utf-8") + req.Header.Add("Accept", permSub.ContentType.String()+","+"*/*") + + // If force is true, we want to skip setting Cache + // headers so that we definitely don't get a 304 back. + if !force { + // If we've successfully fetched this list + // before, set If-Modified-Since to last + // success to make the request conditional. + // + // See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since + if !permSub.SuccessfullyFetchedAt.IsZero() { + timeStr := permSub.SuccessfullyFetchedAt.Format(http.TimeFormat) + req.Header.Add("If-Modified-Since", timeStr) + } + + // If we've got an ETag stored for this list, set + // If-None-Match to make the request conditional. + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#caching_of_unchanged_resources. + if len(permSub.ETag) != 0 { + req.Header.Add("If-None-Match", permSub.ETag) + } + } + + // Perform the HTTP request + rsp, err := t.GET(req) + if err != nil { + return nil, err + } + + // If we have an unexpected / error response, + // wrap + return as error. This will also drain + // and close the response body for us. + if rsp.StatusCode != http.StatusOK && + rsp.StatusCode != http.StatusNotModified { + err := gtserror.NewFromResponse(rsp) + return nil, err + } + + // Check already if we were given an ETag + // we can use, as ETag is often returned + // even on 304 Not Modified responses. + eTag := rsp.Header.Get("ETag") + + if rsp.StatusCode == http.StatusNotModified { + // Nothing has changed on the remote side + // since we last fetched, so there's nothing + // to do and we don't need to read the body. + rsp.Body.Close() + return &DereferenceDomainPermissionsResp{ + Unmodified: true, + ETag: eTag, + }, nil + } + + // Return the body + ETag to the caller. + return &DereferenceDomainPermissionsResp{ + Body: rsp.Body, + ETag: eTag, + }, nil +} diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 7f7e985fc..45d43ff18 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -78,6 +78,20 @@ type Transport interface { // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) + // DereferenceDomainPermissions dereferences the + // permissions list present at the given permSub's URI. + // + // If "force", then If-Modified-Since and If-None-Match + // headers will *NOT* be sent with the outgoing request. + // + // If err == nil and Unmodified == false, then it's up + // to the caller to close the returned io.ReadCloser. + DereferenceDomainPermissions( + ctx context.Context, + permSub *gtsmodel.DomainPermissionSubscription, + force bool, + ) (*DereferenceDomainPermissionsResp, error) + // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. Finger(ctx context.Context, targetUsername string, targetDomain string) ([]byte, error) } diff --git a/internal/typeutils/csv.go b/internal/typeutils/csv.go index 7211d5c9c..4986f5902 100644 --- a/internal/typeutils/csv.go +++ b/internal/typeutils/csv.go @@ -27,6 +27,7 @@ "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -553,3 +554,94 @@ func (c *Converter) CSVToBlocks( return blocks, nil } + +// CSVToDomainPerms converts a slice of Mastodon-style +// CSV records to barebones gtsmodel.DomainPermission's, +// ready for further processing. +// +// Expected column names are: +// +// #domain +// #severity +// #reject_media +// #reject_reports +// #public_comment +// #obfuscate +func (c *Converter) CSVToDomainPerms( + ctx context.Context, + records [][]string, + permType gtsmodel.DomainPermissionType, +) ([]gtsmodel.DomainPermission, error) { + // Make sure we actually + // have some records. + if len(records) == 0 { + return nil, nil + } + + // Validate column headers. + columnHeaders := records[0] + if !slices.Equal( + columnHeaders, + []string{ + "#domain", + "#severity", + "#reject_media", + "#reject_reports", + "#public_comment", + "#obfuscate", + }, + ) { + return nil, gtserror.Newf( + "unexpected column headers in csv: %+v", + columnHeaders, + ) + } + + // Trim off column headers + // now they're validated. + records = records[1:] + + // Convert records to permissions slice. + perms := make([]gtsmodel.DomainPermission, 0, len(records)) + for _, record := range records { + if len(record) != 6 { + log.Warnf(ctx, "skipping invalid-length record: %+v", record) + continue + } + + var ( + domain = record[0] + severity = record[1] + publicComment = record[4] + obfuscate, err = strconv.ParseBool(record[5]) + ) + + if severity != "suspend" { + log.Warnf(ctx, "skipping non-suspend record: %+v", record) + continue + } + + if err != nil { + log.Warnf(ctx, "couldn't parse obfuscate field of record: %+v", record) + continue + } + + // Instantiate the permission + // as either block or allow. + var perm gtsmodel.DomainPermission + if permType == gtsmodel.DomainPermissionBlock { + perm = >smodel.DomainBlock{Domain: domain} + } else { + perm = >smodel.DomainAllow{Domain: domain} + } + + // Set remaining fields. + perm.SetPublicComment(publicComment) + perm.SetObfuscate(&obfuscate) + + // We're done. + perms = append(perms, perm) + } + + return perms, nil +}