From 1ce9c524420bc6a7b08748ca736284382581ab56 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 4 Dec 2020 10:41:07 +0000 Subject: [PATCH 01/13] Don't recalculate event IDs so often (#1610) * Don't recalculate event IDs so often * Revert invite change * Make sure we're using the right NIDs * Update gomatrixserverlib * Update to NewEventFromTrustedJSONWithEventID * Fix go.mod * Update gomatrixserverlib to matrix-org/gomatrixserverlib#243 * Use BulkSelectEventID --- go.mod | 2 +- go.sum | 4 ++-- roomserver/storage/shared/storage.go | 28 +++++++++++++++++++++++----- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index d01c80860..54a139321 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201202134418-2ba106a5bca3 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 18715f86a..9780a688d 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201202134418-2ba106a5bca3 h1:+45Q/5FybBhHPMr10YdzJNFYO/6RRgkBcZbMzIRq5Ck= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201202134418-2ba106a5bca3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb h1:cN+v/rDbGg2p5e8xyxfATtwXeW7tI4aUn7slLXvuNyw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 2548980d4..83982299a 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -309,6 +309,10 @@ func (d *Database) Events( if err != nil { return nil, err } + eventIDs, _ := d.EventsTable.BulkSelectEventID(ctx, eventNIDs) + if err != nil { + eventIDs = map[types.EventNID]string{} + } results := make([]types.Event, len(eventJSONs)) for i, eventJSON := range eventJSONs { var roomNID types.RoomNID @@ -328,8 +332,8 @@ func (d *Database) Events( return nil, err } } - result.Event, err = gomatrixserverlib.NewEventFromTrustedJSON( - eventJSON.EventJSON, false, roomVersion, + result.Event, err = gomatrixserverlib.NewEventFromTrustedJSONWithEventID( + eventIDs[eventJSON.EventNID], eventJSON.EventJSON, false, roomVersion, ) if err != nil { return nil, err @@ -779,6 +783,7 @@ func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event { // GetStateEvent returns the current state event of a given type for a given room with a given state key // If no event could be found, returns nil // If there was an issue during the retrieval, returns an error +// nolint:gocyclo func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) { roomInfo, err := d.RoomInfo(ctx, roomID) if err != nil { @@ -800,6 +805,16 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s if err != nil { return nil, err } + var eventNIDs []types.EventNID + for _, e := range entries { + if e.EventTypeNID == eventTypeNID && e.EventStateKeyNID == stateKeyNID { + eventNIDs = append(eventNIDs, e.EventNID) + } + } + eventIDs, _ := d.EventsTable.BulkSelectEventID(ctx, eventNIDs) + if err != nil { + eventIDs = map[types.EventNID]string{} + } // return the event requested for _, e := range entries { if e.EventTypeNID == eventTypeNID && e.EventStateKeyNID == stateKeyNID { @@ -810,7 +825,7 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s if len(data) == 0 { return nil, fmt.Errorf("GetStateEvent: no json for event nid %d", e.EventNID) } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(data[0].EventJSON, false, roomInfo.RoomVersion) + ev, err := gomatrixserverlib.NewEventFromTrustedJSONWithEventID(eventIDs[e.EventNID], data[0].EventJSON, false, roomInfo.RoomVersion) if err != nil { return nil, err } @@ -921,7 +936,10 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu } } } - + eventIDs, _ := d.EventsTable.BulkSelectEventID(ctx, eventNIDs) + if err != nil { + eventIDs = map[types.EventNID]string{} + } events, err := d.EventJSONTable.BulkSelectEventJSON(ctx, eventNIDs) if err != nil { return nil, fmt.Errorf("GetBulkStateContent: failed to load event JSON for event nids: %w", err) @@ -929,7 +947,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu result := make([]tables.StrippedEvent, len(events)) for i := range events { roomVer := eventNIDToVer[events[i].EventNID] - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(events[i].EventJSON, false, roomVer) + ev, err := gomatrixserverlib.NewEventFromTrustedJSONWithEventID(eventIDs[events[i].EventNID], events[i].EventJSON, false, roomVer) if err != nil { return nil, fmt.Errorf("GetBulkStateContent: failed to load event JSON for event NID %v : %w", events[i].EventNID, err) } From a677a288bd3344a5d31080be047125043ada75dc Mon Sep 17 00:00:00 2001 From: Ronnie Ebrin Date: Fri, 4 Dec 2020 15:08:17 +0100 Subject: [PATCH 02/13] federationsender/roomserver: don't panic while federation is disabled (#1615) --- federationsender/consumers/roomserver.go | 21 ++++++++++++++------- federationsender/queue/queue.go | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 13c415bc7..b53cb251b 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -94,13 +94,20 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { } if err := s.processMessage(*output.NewRoomEvent); err != nil { - // panic rather than continue with an inconsistent database - log.WithFields(log.Fields{ - "event": string(ev.JSON()), - "add": output.NewRoomEvent.AddsStateEventIDs, - "del": output.NewRoomEvent.RemovesStateEventIDs, - log.ErrorKey: err, - }).Panicf("roomserver output log: write room event failure") + switch err.(type) { + case *queue.ErrorFederationDisabled: + log.WithField("error", output.Type).Info( + err.Error(), + ) + default: + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + "event": string(ev.JSON()), + "add": output.NewRoomEvent.AddsStateEventIDs, + "del": output.NewRoomEvent.RemovesStateEventIDs, + log.ErrorKey: err, + }).Panicf("roomserver output log: write room event failure") + } return nil } default: diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 95d37b2b8..d45af8686 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -122,13 +122,23 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d return oq } +type ErrorFederationDisabled struct { + Message string +} + +func (e *ErrorFederationDisabled) Error() string { + return e.Message +} + // SendEvent sends an event to the destinations func (oqs *OutgoingQueues) SendEvent( ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName, ) error { if oqs.disabled { - return fmt.Errorf("federation is disabled") + return &ErrorFederationDisabled{ + Message: "Federation disabled", + } } if origin != oqs.origin { // TODO: Support virtual hosting; gh issue #577. @@ -190,7 +200,9 @@ func (oqs *OutgoingQueues) SendEDU( destinations []gomatrixserverlib.ServerName, ) error { if oqs.disabled { - return fmt.Errorf("federation is disabled") + return &ErrorFederationDisabled{ + Message: "Federation disabled", + } } if origin != oqs.origin { // TODO: Support virtual hosting; gh issue #577. From 68d92fd3b4bfdc5a8e13ec66905bc7c0a2b951c9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 4 Dec 2020 14:09:21 +0000 Subject: [PATCH 03/13] Update sytest-whitelist/blacklist --- sytest-blacklist | 5 ++++- sytest-whitelist | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sytest-blacklist b/sytest-blacklist index f42d5f822..601a3f705 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -63,4 +63,7 @@ If user leaves room, remote user changes device and rejoins we see update in /sy A prev_batch token from incremental sync can be used in the v1 messages API # Blacklisted due to flakiness -Forgotten room messages cannot be paginated \ No newline at end of file +Forgotten room messages cannot be paginated + +# Blacklisted due to flakiness +Can re-join room if re-invited \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index 17027041d..da4b201ce 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -501,6 +501,5 @@ Can get rooms/{roomId}/state for a departed room (SPEC-216) Users cannot set notifications powerlevel higher than their own Forgetting room does not show up in v2 /sync Can forget room you've been kicked from -Can re-join room if re-invited /whois /joined_members return joined members From c052edafdd765d821f9732add4f5d33962ba5ba4 Mon Sep 17 00:00:00 2001 From: TR_SLimey <37966924+TR-SLimey@users.noreply.github.com> Date: Fri, 4 Dec 2020 14:10:01 +0000 Subject: [PATCH 04/13] Fix broken image in CONTRIBUTING.md (#1612) --- docs/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 1ab885ebc..ea4b2b27d 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -37,7 +37,7 @@ If a job fails, click the "details" button and you should be taken to the job's logs. ![Click the details button on the failing build -step](docs/images/details-button-location.jpg) +step](https://raw.githubusercontent.com/matrix-org/dendrite/master/docs/images/details-button-location.jpg) Scroll down to the failing step and you should see some log output. Scan the logs until you find what it's complaining about, fix it, submit a new commit, From b507312d4cf9d35b5d4eaaa01a7f74d095b825f8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 4 Dec 2020 14:11:01 +0000 Subject: [PATCH 05/13] MSC2836 threading: part 2 (#1596) * Update GMSL * Add MSC2836EventRelationships to fedsender * Call MSC2836EventRelationships in reqCtx * auth remote servers * Extract room ID and servers from previous events; refactor a bit * initial cut of federated threading * Use the right client/fed struct in the response * Add QueryAuthChain for use with MSC2836 * Add auth chain to federated response * Fix pointers * under CI: more logging and enable mscs, nil fix * Handle direction: up * Actually send message events to the roomserver.. * Add children and children_hash to unsigned, with tests * Add logic for exploring threads and tracking children; missing storage functions * Implement storage functions for children * Add fetchUnknownEvent * Do federated hits for include_children if we have unexplored children * Use /ev_rel rather than /event as the former includes child metadata * Remove cross-room threading impl * Enable MSC2836 in the p2p demo * Namespace mscs db * Enable msc2836 for ygg Co-authored-by: Neil Alexander --- cmd/dendrite-demo-libp2p/main.go | 6 + cmd/dendrite-demo-yggdrasil/main.go | 6 + cmd/generate-config/main.go | 2 + federationapi/routing/send_test.go | 52 +-- federationsender/api/api.go | 1 + federationsender/internal/api.go | 15 + federationsender/inthttp/client.go | 51 ++- federationsender/inthttp/server.go | 22 + go.mod | 9 +- go.sum | 13 + roomserver/api/api.go | 9 + roomserver/api/api_trace.go | 10 + roomserver/api/query.go | 8 + roomserver/internal/input/input.go | 4 +- roomserver/internal/query/query.go | 13 + roomserver/inthttp/client.go | 11 + roomserver/inthttp/server.go | 13 + setup/mscs/msc2836/msc2836.go | 618 +++++++++++++++++++++------- setup/mscs/msc2836/msc2836_test.go | 123 ++++-- setup/mscs/msc2836/storage.go | 159 ++++++- setup/mscs/mscs.go | 3 + 21 files changed, 896 insertions(+), 252 deletions(-) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 9aae34e38..92c283b52 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -36,6 +36,7 @@ import ( "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/mscs" "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -130,6 +131,8 @@ func main() { cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName)) + cfg.MSCs.MSCs = []string{"msc2836"} + cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) if err = cfg.Derive(); err != nil { panic(err) } @@ -190,6 +193,9 @@ func main() { base.Base.PublicKeyAPIMux, base.Base.PublicMediaAPIMux, ) + if err := mscs.Enable(&base.Base, &monolith); err != nil { + logrus.WithError(err).Fatalf("Failed to enable MSCs") + } httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.Base.InternalAPIMux) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 39643cc2f..16f92cfa1 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -39,6 +39,7 @@ import ( "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/mscs" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -83,6 +84,8 @@ func main() { cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) + cfg.MSCs.MSCs = []string{"msc2836"} + cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) if err = cfg.Derive(); err != nil { panic(err) } @@ -151,6 +154,9 @@ func main() { base.PublicKeyAPIMux, base.PublicMediaAPIMux, ) + if err := mscs.Enable(base, &monolith); err != nil { + logrus.WithError(err).Fatalf("Failed to enable MSCs") + } httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index 3233cf809..c4cb4abbb 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -63,6 +63,8 @@ func main() { if *defaultsForCI { cfg.ClientAPI.RateLimiting.Enabled = false cfg.FederationSender.DisableTLSValidation = true + cfg.MSCs.MSCs = []string{"msc2836"} + cfg.Logging[0].Level = "trace" } j, err := yaml.Marshal(cfg) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index a9ee78830..8bdf54c4a 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -9,7 +9,6 @@ import ( "time" eduAPI "github.com/matrix-org/dendrite/eduserver/api" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -85,6 +84,7 @@ func (o *testEDUProducer) InputReceiptEvent( } type testRoomserverAPI struct { + api.RoomserverInternalAPITrace inputRoomEvents []api.InputRoomEvent queryMissingAuthPrevEvents func(*api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse @@ -92,12 +92,6 @@ type testRoomserverAPI struct { queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse } -func (t *testRoomserverAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, resp *api.PerformForgetResponse) error { - return nil -} - -func (t *testRoomserverAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {} - func (t *testRoomserverAPI) InputRoomEvents( ctx context.Context, request *api.InputRoomEventsRequest, @@ -109,50 +103,6 @@ func (t *testRoomserverAPI) InputRoomEvents( } } -func (t *testRoomserverAPI) PerformInvite( - ctx context.Context, - req *api.PerformInviteRequest, - res *api.PerformInviteResponse, -) error { - return nil -} - -func (t *testRoomserverAPI) PerformJoin( - ctx context.Context, - req *api.PerformJoinRequest, - res *api.PerformJoinResponse, -) { -} - -func (t *testRoomserverAPI) PerformPeek( - ctx context.Context, - req *api.PerformPeekRequest, - res *api.PerformPeekResponse, -) { -} - -func (t *testRoomserverAPI) PerformUnpeek( - ctx context.Context, - req *api.PerformUnpeekRequest, - res *api.PerformUnpeekResponse, -) { -} - -func (t *testRoomserverAPI) PerformPublish( - ctx context.Context, - req *api.PerformPublishRequest, - res *api.PerformPublishResponse, -) { -} - -func (t *testRoomserverAPI) PerformLeave( - ctx context.Context, - req *api.PerformLeaveRequest, - res *api.PerformLeaveResponse, -) error { - return nil -} - // Query the latest events and state for a room from the room server. func (t *testRoomserverAPI) QueryLatestEventsAndState( ctx context.Context, diff --git a/federationsender/api/api.go b/federationsender/api/api.go index a4d15f1f5..e4d176b16 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -21,6 +21,7 @@ type FederationClient interface { QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error) GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) + MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error) } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index a14cf3caa..407e7ffec 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -229,3 +229,18 @@ func (a *FederationSenderInternalAPI) LookupServerKeys( } return ires.([]gomatrixserverlib.ServerKeys), nil } + +func (a *FederationSenderInternalAPI) MSC2836EventRelationships( + ctx context.Context, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, + roomVersion gomatrixserverlib.RoomVersion, +) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + ires, err := a.doRequest(s, func() (interface{}, error) { + return a.federation.MSC2836EventRelationships(ctx, s, r, roomVersion) + }) + if err != nil { + return res, err + } + return ires.(gomatrixserverlib.MSC2836EventRelationshipsResponse), nil +} diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go index e0783ee1b..fe98ff33d 100644 --- a/federationsender/inthttp/client.go +++ b/federationsender/inthttp/client.go @@ -23,15 +23,16 @@ const ( FederationSenderPerformServersAlivePath = "/federationsender/performServersAlive" FederationSenderPerformBroadcastEDUPath = "/federationsender/performBroadcastEDU" - FederationSenderGetUserDevicesPath = "/federationsender/client/getUserDevices" - FederationSenderClaimKeysPath = "/federationsender/client/claimKeys" - FederationSenderQueryKeysPath = "/federationsender/client/queryKeys" - FederationSenderBackfillPath = "/federationsender/client/backfill" - FederationSenderLookupStatePath = "/federationsender/client/lookupState" - FederationSenderLookupStateIDsPath = "/federationsender/client/lookupStateIDs" - FederationSenderGetEventPath = "/federationsender/client/getEvent" - FederationSenderGetServerKeysPath = "/federationsender/client/getServerKeys" - FederationSenderLookupServerKeysPath = "/federationsender/client/lookupServerKeys" + FederationSenderGetUserDevicesPath = "/federationsender/client/getUserDevices" + FederationSenderClaimKeysPath = "/federationsender/client/claimKeys" + FederationSenderQueryKeysPath = "/federationsender/client/queryKeys" + FederationSenderBackfillPath = "/federationsender/client/backfill" + FederationSenderLookupStatePath = "/federationsender/client/lookupState" + FederationSenderLookupStateIDsPath = "/federationsender/client/lookupStateIDs" + FederationSenderGetEventPath = "/federationsender/client/getEvent" + FederationSenderGetServerKeysPath = "/federationsender/client/getServerKeys" + FederationSenderLookupServerKeysPath = "/federationsender/client/lookupServerKeys" + FederationSenderEventRelationshipsPath = "/federationsender/client/msc2836eventRelationships" ) // NewFederationSenderClient creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. @@ -416,3 +417,35 @@ func (h *httpFederationSenderInternalAPI) LookupServerKeys( } return response.ServerKeys, nil } + +type eventRelationships struct { + S gomatrixserverlib.ServerName + Req gomatrixserverlib.MSC2836EventRelationshipsRequest + RoomVer gomatrixserverlib.RoomVersion + Res gomatrixserverlib.MSC2836EventRelationshipsResponse + Err *api.FederationClientError +} + +func (h *httpFederationSenderInternalAPI) MSC2836EventRelationships( + ctx context.Context, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, + roomVersion gomatrixserverlib.RoomVersion, +) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "MSC2836EventRelationships") + defer span.Finish() + + request := eventRelationships{ + S: s, + Req: r, + RoomVer: roomVersion, + } + var response eventRelationships + apiURL := h.federationSenderURL + FederationSenderEventRelationshipsPath + err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) + if err != nil { + return res, err + } + if response.Err != nil { + return res, response.Err + } + return response.Res, nil +} diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go index 53e1183e4..293fb4209 100644 --- a/federationsender/inthttp/server.go +++ b/federationsender/inthttp/server.go @@ -307,4 +307,26 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route return util.JSONResponse{Code: http.StatusOK, JSON: request} }), ) + internalAPIMux.Handle( + FederationSenderEventRelationshipsPath, + httputil.MakeInternalAPI("MSC2836EventRelationships", func(req *http.Request) util.JSONResponse { + var request eventRelationships + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + res, err := intAPI.MSC2836EventRelationships(req.Context(), request.S, request.Req, request.RoomVer) + if err != nil { + ferr, ok := err.(*api.FederationClientError) + if ok { + request.Err = ferr + } else { + request.Err = &api.FederationClientError{ + Err: err.Error(), + } + } + } + request.Res = res + return util.JSONResponse{Code: http.StatusOK, JSON: request} + }), + ) } diff --git a/go.mod b/go.mod index 54a139321..0d39cf03b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/golang-lru v0.5.4 + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/lib/pq v1.8.0 github.com/libp2p/go-libp2p v0.11.0 github.com/libp2p/go-libp2p-circuit v0.3.1 @@ -32,15 +33,17 @@ require ( github.com/pkg/errors v0.9.1 github.com/pressly/goose v2.7.0-rc5+incompatible github.com/prometheus/client_golang v1.7.1 - github.com/sirupsen/logrus v1.6.0 + github.com/sirupsen/logrus v1.7.0 github.com/tidwall/gjson v1.6.3 - github.com/tidwall/sjson v1.1.1 + github.com/tidwall/match v1.0.2 // indirect + github.com/tidwall/sjson v1.1.2 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee go.uber.org/atomic v1.6.0 - golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a + golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 golang.org/x/net v0.0.0-20200528225125-3c3fba18258b + golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect gopkg.in/h2non/bimg.v1 v1.1.4 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 9780a688d..3bd186ce9 100644 --- a/go.sum +++ b/go.sum @@ -781,6 +781,8 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= @@ -812,10 +814,13 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI= github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/match v1.0.2 h1:uuqvHuBGSedK7awZ2YoAtpnimfwBGFjHuWLuLqQj+bU= +github.com/tidwall/match v1.0.2/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -825,6 +830,8 @@ github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= +github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= +github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= @@ -907,6 +914,8 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 h1:phUcVbl53swtrUN8kQEXFhUxPlIlWyBfKmidCu7P95o= +golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -982,6 +991,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -994,6 +1004,9 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/roomserver/api/api.go b/roomserver/api/api.go index bef2bb3fa..ebc068ac8 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -132,6 +132,15 @@ type RoomserverInternalAPI interface { response *QueryStateAndAuthChainResponse, ) error + // QueryAuthChain returns the entire auth chain for the event IDs given. + // The response includes the events in the request. + // Omits without error for any missing auth events. There will be no duplicates. + QueryAuthChain( + ctx context.Context, + request *QueryAuthChainRequest, + response *QueryAuthChainResponse, + ) error + // QueryCurrentState retrieves the requested state events. If state events are not found, they will be missing from // the response. QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index eb2b2e1d4..c279807e5 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -324,6 +324,16 @@ func (t *RoomserverInternalAPITrace) QueryServerBannedFromRoom(ctx context.Conte return err } +func (t *RoomserverInternalAPITrace) QueryAuthChain( + ctx context.Context, + request *QueryAuthChainRequest, + response *QueryAuthChainResponse, +) error { + err := t.Impl.QueryAuthChain(ctx, request, response) + util.GetLogger(ctx).WithError(err).Infof("QueryAuthChain req=%+v res=%+v", js(request), js(response)) + return err +} + func js(thing interface{}) string { b, err := json.Marshal(thing) if err != nil { diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 55922b7fe..43e562a98 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -275,6 +275,14 @@ type QueryPublishedRoomsResponse struct { RoomIDs []string } +type QueryAuthChainRequest struct { + EventIDs []string +} + +type QueryAuthChainResponse struct { + AuthChain []*gomatrixserverlib.HeaderedEvent +} + type QuerySharedUsersRequest struct { UserID string ExcludeRoomIDs []string diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 79dc2fe14..2bed0c7f4 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -62,10 +62,10 @@ func (w *inputWorker) start() { for { select { case task := <-w.input: - hooks.Run(hooks.KindNewEventReceived, &task.event.Event) + hooks.Run(hooks.KindNewEventReceived, task.event.Event) _, task.err = w.r.processRoomEvent(task.ctx, task.event) if task.err == nil { - hooks.Run(hooks.KindNewEventPersisted, &task.event.Event) + hooks.Run(hooks.KindNewEventPersisted, task.event.Event) } task.wg.Done() case <-time.After(time.Second * 5): diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index c9940c3c2..7346c7a77 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -716,3 +716,16 @@ func (r *Queryer) QueryServerBannedFromRoom(ctx context.Context, req *api.QueryS res.Banned = r.ServerACLs.IsServerBannedFromRoom(req.ServerName, req.RoomID) return nil } + +func (r *Queryer) QueryAuthChain(ctx context.Context, req *api.QueryAuthChainRequest, res *api.QueryAuthChainResponse) error { + chain, err := getAuthChain(ctx, r.DB.EventsFromIDs, req.EventIDs) + if err != nil { + return err + } + hchain := make([]*gomatrixserverlib.HeaderedEvent, len(chain)) + for i := range chain { + hchain[i] = chain[i].Headered(chain[i].Version()) + } + res.AuthChain = hchain + return nil +} diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index e496b81e0..8a1c91d28 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -54,6 +54,7 @@ const ( RoomserverQuerySharedUsersPath = "/roomserver/querySharedUsers" RoomserverQueryKnownUsersPath = "/roomserver/queryKnownUsers" RoomserverQueryServerBannedFromRoomPath = "/roomserver/queryServerBannedFromRoom" + RoomserverQueryAuthChainPath = "/roomserver/queryAuthChain" ) type httpRoomserverInternalAPI struct { @@ -502,6 +503,16 @@ func (h *httpRoomserverInternalAPI) QueryKnownUsers( return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) } +func (h *httpRoomserverInternalAPI) QueryAuthChain( + ctx context.Context, req *api.QueryAuthChainRequest, res *api.QueryAuthChainResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryAuthChain") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryAuthChainPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom( ctx context.Context, req *api.QueryServerBannedFromRoomRequest, res *api.QueryServerBannedFromRoomResponse, ) error { diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index ac1fc25b6..f9c8ef9fd 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -452,4 +452,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(RoomserverQueryAuthChainPath, + httputil.MakeInternalAPI("queryAuthChain", func(req *http.Request) util.JSONResponse { + request := api.QueryAuthChainRequest{} + response := api.QueryAuthChainResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.QueryAuthChain(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index 33a65c8f4..95473f97c 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -18,10 +18,13 @@ package msc2836 import ( "bytes" "context" + "crypto/sha256" "encoding/json" "fmt" "io" "net/http" + "sort" + "strings" "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" @@ -36,13 +39,12 @@ import ( ) const ( - constRelType = "m.reference" - constRoomIDKey = "relationship_room_id" - constRoomServers = "relationship_servers" + constRelType = "m.reference" ) type EventRelationshipRequest struct { EventID string `json:"event_id"` + RoomID string `json:"room_id"` MaxDepth int `json:"max_depth"` MaxBreadth int `json:"max_breadth"` Limit int `json:"limit"` @@ -52,7 +54,6 @@ type EventRelationshipRequest struct { IncludeChildren bool `json:"include_children"` Direction string `json:"direction"` Batch string `json:"batch"` - AutoJoin bool `json:"auto_join"` } func NewEventRelationshipRequest(body io.Reader) (*EventRelationshipRequest, error) { @@ -81,8 +82,16 @@ type EventRelationshipResponse struct { Limited bool `json:"limited"` } +func toClientResponse(res *gomatrixserverlib.MSC2836EventRelationshipsResponse) *EventRelationshipResponse { + out := &EventRelationshipResponse{ + Events: gomatrixserverlib.ToClientEvents(res.Events, gomatrixserverlib.FormatAll), + Limited: res.Limited, + NextBatch: res.NextBatch, + } + return out +} + // Enable this MSC -// nolint:gocyclo func Enable( base *setup.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, userAPI userapi.UserInternalAPI, keyRing gomatrixserverlib.JSONVerifier, @@ -96,63 +105,22 @@ func Enable( he := headeredEvent.(*gomatrixserverlib.HeaderedEvent) hookErr := db.StoreRelation(context.Background(), he) if hookErr != nil { - util.GetLogger(context.Background()).WithError(hookErr).Error( + util.GetLogger(context.Background()).WithError(hookErr).WithField("event_id", he.EventID()).Error( "failed to StoreRelation", ) } - }) - hooks.Attach(hooks.KindNewEventReceived, func(headeredEvent interface{}) { - he := headeredEvent.(*gomatrixserverlib.HeaderedEvent) - ctx := context.Background() - // we only inject metadata for events our server sends - userID := he.Sender() - _, domain, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - return - } - if domain != base.Cfg.Global.ServerName { - return - } - // if this event has an m.relationship, add on the room_id and servers to unsigned - parent, child, relType := parentChildEventIDs(he) - if parent == "" || child == "" || relType == "" { - return - } - event, joinedToRoom := getEventIfVisible(ctx, rsAPI, parent, userID) - if !joinedToRoom { - return - } - err = he.SetUnsignedField(constRoomIDKey, event.RoomID()) - if err != nil { - util.GetLogger(context.Background()).WithError(err).Warn("Failed to SetUnsignedField") - return - } - - var servers []gomatrixserverlib.ServerName - if fsAPI != nil { - var res fs.QueryJoinedHostServerNamesInRoomResponse - err = fsAPI.QueryJoinedHostServerNamesInRoom(ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: event.RoomID(), - }, &res) - if err != nil { - util.GetLogger(context.Background()).WithError(err).Warn("Failed to QueryJoinedHostServerNamesInRoom") - return - } - servers = res.ServerNames - } else { - servers = []gomatrixserverlib.ServerName{ - base.Cfg.Global.ServerName, - } - } - err = he.SetUnsignedField(constRoomServers, servers) - if err != nil { - util.GetLogger(context.Background()).WithError(err).Warn("Failed to SetUnsignedField") - return + // we need to update child metadata here as well as after doing remote /event_relationships requests + // so we catch child metadata originating from /send transactions + hookErr = db.UpdateChildMetadata(context.Background(), he) + if hookErr != nil { + util.GetLogger(context.Background()).WithError(err).WithField("event_id", he.EventID()).Warn( + "failed to update child metadata for event", + ) } }) base.PublicClientAPIMux.Handle("/unstable/event_relationships", - httputil.MakeAuthAPI("eventRelationships", userAPI, eventRelationshipHandler(db, rsAPI)), + httputil.MakeAuthAPI("eventRelationships", userAPI, eventRelationshipHandler(db, rsAPI, fsAPI)), ).Methods(http.MethodPost, http.MethodOptions) base.PublicFederationAPIMux.Handle("/unstable/event_relationships", httputil.MakeExternalAPI( @@ -163,22 +131,27 @@ func Enable( if fedReq == nil { return errResp } - return federatedEventRelationship(req.Context(), fedReq, db, rsAPI) + return federatedEventRelationship(req.Context(), fedReq, db, rsAPI, fsAPI) }, )).Methods(http.MethodPost, http.MethodOptions) return nil } type reqCtx struct { - ctx context.Context - rsAPI roomserver.RoomserverInternalAPI - db Database - req *EventRelationshipRequest - userID string + ctx context.Context + rsAPI roomserver.RoomserverInternalAPI + db Database + req *EventRelationshipRequest + userID string + roomVersion gomatrixserverlib.RoomVersion + + // federated request args isFederatedRequest bool + serverName gomatrixserverlib.ServerName + fsAPI fs.FederationSenderInternalAPI } -func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAPI) func(*http.Request, *userapi.Device) util.JSONResponse { +func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI) func(*http.Request, *userapi.Device) util.JSONResponse { return func(req *http.Request, device *userapi.Device) util.JSONResponse { relation, err := NewEventRelationshipRequest(req.Body) if err != nil { @@ -193,6 +166,7 @@ func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAP req: relation, userID: device.UserID, rsAPI: rsAPI, + fsAPI: fsAPI, isFederatedRequest: false, db: db, } @@ -203,12 +177,14 @@ func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAP return util.JSONResponse{ Code: 200, - JSON: res, + JSON: toClientResponse(res), } } } -func federatedEventRelationship(ctx context.Context, fedReq *gomatrixserverlib.FederationRequest, db Database, rsAPI roomserver.RoomserverInternalAPI) util.JSONResponse { +func federatedEventRelationship( + ctx context.Context, fedReq *gomatrixserverlib.FederationRequest, db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, +) util.JSONResponse { relation, err := NewEventRelationshipRequest(bytes.NewBuffer(fedReq.Content())) if err != nil { util.GetLogger(ctx).WithError(err).Error("failed to decode HTTP request as JSON") @@ -218,17 +194,43 @@ func federatedEventRelationship(ctx context.Context, fedReq *gomatrixserverlib.F } } rc := reqCtx{ - ctx: ctx, - req: relation, - userID: "", - rsAPI: rsAPI, + ctx: ctx, + req: relation, + rsAPI: rsAPI, + db: db, + // federation args isFederatedRequest: true, - db: db, + fsAPI: fsAPI, + serverName: fedReq.Origin(), } res, resErr := rc.process() if resErr != nil { return *resErr } + // add auth chain information + requiredAuthEventsSet := make(map[string]bool) + var requiredAuthEvents []string + for _, ev := range res.Events { + for _, a := range ev.AuthEventIDs() { + if requiredAuthEventsSet[a] { + continue + } + requiredAuthEvents = append(requiredAuthEvents, a) + requiredAuthEventsSet[a] = true + } + } + var queryRes roomserver.QueryAuthChainResponse + err = rsAPI.QueryAuthChain(ctx, &roomserver.QueryAuthChainRequest{ + EventIDs: requiredAuthEvents, + }, &queryRes) + if err != nil { + // they may already have the auth events so don't fail this request + util.GetLogger(ctx).WithError(err).Error("Failed to QueryAuthChain") + } + res.AuthChain = make([]*gomatrixserverlib.Event, len(queryRes.AuthChain)) + for i := range queryRes.AuthChain { + res.AuthChain[i] = queryRes.AuthChain[i].Unwrap() + } return util.JSONResponse{ Code: 200, @@ -236,18 +238,25 @@ func federatedEventRelationship(ctx context.Context, fedReq *gomatrixserverlib.F } } -func (rc *reqCtx) process() (*EventRelationshipResponse, *util.JSONResponse) { - var res EventRelationshipResponse +// nolint:gocyclo +func (rc *reqCtx) process() (*gomatrixserverlib.MSC2836EventRelationshipsResponse, *util.JSONResponse) { + var res gomatrixserverlib.MSC2836EventRelationshipsResponse var returnEvents []*gomatrixserverlib.HeaderedEvent // Can the user see (according to history visibility) event_id? If no, reject the request, else continue. - // We should have the event being referenced so don't give any claimed room ID / servers - event := rc.getEventIfVisible(rc.req.EventID, "", nil) + event := rc.getLocalEvent(rc.req.EventID) if event == nil { + event = rc.fetchUnknownEvent(rc.req.EventID, rc.req.RoomID) + } + if rc.req.RoomID == "" && event != nil { + rc.req.RoomID = event.RoomID() + } + if event == nil || !rc.authorisedToSeeEvent(event) { return nil, &util.JSONResponse{ Code: 403, JSON: jsonerror.Forbidden("Event does not exist or you are not authorised to see it"), } } + rc.roomVersion = event.Version() // Retrieve the event. Add it to response array. returnEvents = append(returnEvents, event) @@ -282,29 +291,122 @@ func (rc *reqCtx) process() (*EventRelationshipResponse, *util.JSONResponse) { ) returnEvents = append(returnEvents, events...) } - res.Events = make([]gomatrixserverlib.ClientEvent, len(returnEvents)) + res.Events = make([]*gomatrixserverlib.Event, len(returnEvents)) for i, ev := range returnEvents { - res.Events[i] = gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll) + // for each event, extract the children_count | hash and add it as unsigned data. + rc.addChildMetadata(ev) + res.Events[i] = ev.Unwrap() } res.Limited = remaining == 0 || walkLimited return &res, nil } +// fetchUnknownEvent retrieves an unknown event from the room specified. This server must +// be joined to the room in question. This has the side effect of injecting surround threaded +// events into the roomserver. +func (rc *reqCtx) fetchUnknownEvent(eventID, roomID string) *gomatrixserverlib.HeaderedEvent { + if rc.isFederatedRequest || roomID == "" { + // we don't do fed hits for fed requests, and we can't ask servers without a room ID! + return nil + } + logger := util.GetLogger(rc.ctx).WithField("room_id", roomID) + // if they supplied a room_id, check the room exists. + var queryVerRes roomserver.QueryRoomVersionForRoomResponse + err := rc.rsAPI.QueryRoomVersionForRoom(rc.ctx, &roomserver.QueryRoomVersionForRoomRequest{ + RoomID: roomID, + }, &queryVerRes) + if err != nil { + logger.WithError(err).Warn("failed to query room version for room, does this room exist?") + return nil + } + + // check the user is joined to that room + var queryMemRes roomserver.QueryMembershipForUserResponse + err = rc.rsAPI.QueryMembershipForUser(rc.ctx, &roomserver.QueryMembershipForUserRequest{ + RoomID: roomID, + UserID: rc.userID, + }, &queryMemRes) + if err != nil { + logger.WithError(err).Warn("failed to query membership for user in room") + return nil + } + if !queryMemRes.IsInRoom { + return nil + } + + // ask one of the servers in the room for the event + var queryRes fs.QueryJoinedHostServerNamesInRoomResponse + err = rc.fsAPI.QueryJoinedHostServerNamesInRoom(rc.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ + RoomID: roomID, + }, &queryRes) + if err != nil { + logger.WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") + return nil + } + // query up to 5 servers + serversToQuery := queryRes.ServerNames + if len(serversToQuery) > 5 { + serversToQuery = serversToQuery[:5] + } + + // fetch the event, along with some of the surrounding thread (if it's threaded) and the auth chain. + // Inject the response into the roomserver to remember the event across multiple calls and to set + // unexplored flags correctly. + for _, srv := range serversToQuery { + res, err := rc.MSC2836EventRelationships(eventID, srv, queryVerRes.RoomVersion) + if err != nil { + continue + } + rc.injectResponseToRoomserver(res) + for _, ev := range res.Events { + if ev.EventID() == eventID { + return ev.Headered(ev.Version()) + } + } + } + logger.WithField("servers", serversToQuery).Warn("failed to query event relationships") + return nil +} + // If include_parent: true and there is a valid m.relationship field in the event, // retrieve the referenced event. Apply history visibility check to that event and if it passes, add it to the response array. -func (rc *reqCtx) includeParent(event *gomatrixserverlib.HeaderedEvent) (parent *gomatrixserverlib.HeaderedEvent) { - parentID, _, _ := parentChildEventIDs(event) +func (rc *reqCtx) includeParent(childEvent *gomatrixserverlib.HeaderedEvent) (parent *gomatrixserverlib.HeaderedEvent) { + parentID, _, _ := parentChildEventIDs(childEvent) if parentID == "" { return nil } - claimedRoomID, claimedServers := roomIDAndServers(event) - return rc.getEventIfVisible(parentID, claimedRoomID, claimedServers) + return rc.lookForEvent(parentID) } // If include_children: true, lookup all events which have event_id as an m.relationship // Apply history visibility checks to all these events and add the ones which pass into the response array, // honouring the recent_first flag and the limit. func (rc *reqCtx) includeChildren(db Database, parentID string, limit int, recentFirst bool) ([]*gomatrixserverlib.HeaderedEvent, *util.JSONResponse) { + if rc.hasUnexploredChildren(parentID) { + // we need to do a remote request to pull in the children as we are missing them locally. + serversToQuery := rc.getServersForEventID(parentID) + var result *gomatrixserverlib.MSC2836EventRelationshipsResponse + for _, srv := range serversToQuery { + res, err := rc.fsAPI.MSC2836EventRelationships(rc.ctx, srv, gomatrixserverlib.MSC2836EventRelationshipsRequest{ + EventID: parentID, + Direction: "down", + Limit: 100, + MaxBreadth: -1, + MaxDepth: 1, // we just want the children from this parent + RecentFirst: true, + }, rc.roomVersion) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).WithField("server", srv).Error("includeChildren: failed to call MSC2836EventRelationships") + } else { + result = &res + break + } + } + if result != nil { + rc.injectResponseToRoomserver(result) + } + // fallthrough to pull these new events from the DB + } children, err := db.ChildrenForParent(rc.ctx, parentID, constRelType, recentFirst) if err != nil { util.GetLogger(rc.ctx).WithError(err).Error("failed to get ChildrenForParent") @@ -313,8 +415,7 @@ func (rc *reqCtx) includeChildren(db Database, parentID string, limit int, recen } var childEvents []*gomatrixserverlib.HeaderedEvent for _, child := range children { - // in order for us to even know about the children the server must be joined to those rooms, hence pass no claimed room ID or servers. - childEvent := rc.getEventIfVisible(child.EventID, "", nil) + childEvent := rc.lookForEvent(child.EventID) if childEvent != nil { childEvents = append(childEvents, childEvent) } @@ -327,14 +428,9 @@ func (rc *reqCtx) includeChildren(db Database, parentID string, limit int, recen // Begin to walk the thread DAG in the direction specified, either depth or breadth first according to the depth_first flag, // honouring the limit, max_depth and max_breadth values according to the following rules -// nolint: unparam func walkThread( ctx context.Context, db Database, rc *reqCtx, included map[string]bool, limit int, ) ([]*gomatrixserverlib.HeaderedEvent, bool) { - if rc.req.Direction != "down" { - util.GetLogger(ctx).Error("not implemented: direction=up") - return nil, false - } var result []*gomatrixserverlib.HeaderedEvent eventWalker := walker{ ctx: ctx, @@ -352,8 +448,11 @@ func walkThread( } // Process the event. - // TODO: Include edge information: room ID and servers - event := rc.getEventIfVisible(wi.EventID, "", nil) + // if event is not found, use remoteEventRelationships to explore that part of the thread remotely. + // This will probably be easiest if the event relationships response is directly pumped into the database + // so the next walk will do the right thing. This requires those events to be authed and likely injected as + // outliers into the roomserver DB, which will de-dupe appropriately. + event := rc.lookForEvent(wi.EventID) if event != nil { result = append(result, event) } @@ -368,74 +467,280 @@ func walkThread( return result, limited } -func (rc *reqCtx) getEventIfVisible(eventID string, claimedRoomID string, claimedServers []string) *gomatrixserverlib.HeaderedEvent { - event, joinedToRoom := getEventIfVisible(rc.ctx, rc.rsAPI, eventID, rc.userID) - if event != nil && joinedToRoom { - return event +// MSC2836EventRelationships performs an /event_relationships request to a remote server +func (rc *reqCtx) MSC2836EventRelationships(eventID string, srv gomatrixserverlib.ServerName, ver gomatrixserverlib.RoomVersion) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) { + res, err := rc.fsAPI.MSC2836EventRelationships(rc.ctx, srv, gomatrixserverlib.MSC2836EventRelationshipsRequest{ + EventID: eventID, + DepthFirst: rc.req.DepthFirst, + Direction: rc.req.Direction, + Limit: rc.req.Limit, + MaxBreadth: rc.req.MaxBreadth, + MaxDepth: rc.req.MaxDepth, + RecentFirst: rc.req.RecentFirst, + }, ver) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Error("Failed to call MSC2836EventRelationships") + return nil, err } - // either we don't have the event or we aren't joined to the room, regardless we should try joining if auto join is enabled - if !rc.req.AutoJoin { - return nil - } - // if we're doing this on behalf of a random server don't auto-join rooms regardless of what the request says - if rc.isFederatedRequest { - return nil - } - roomID := claimedRoomID - var servers []gomatrixserverlib.ServerName - if event != nil { - roomID = event.RoomID() - } - for _, s := range claimedServers { - servers = append(servers, gomatrixserverlib.ServerName(s)) - } - var joinRes roomserver.PerformJoinResponse - rc.rsAPI.PerformJoin(rc.ctx, &roomserver.PerformJoinRequest{ - UserID: rc.userID, - Content: map[string]interface{}{}, - RoomIDOrAlias: roomID, - ServerNames: servers, - }, &joinRes) - if joinRes.Error != nil { - util.GetLogger(rc.ctx).WithError(joinRes.Error).WithField("room_id", roomID).Error("Failed to auto-join room") - return nil - } - if event != nil { - return event - } - // TODO: hit /event_relationships on the server we joined via - util.GetLogger(rc.ctx).Infof("joined room but need to fetch event TODO") - return nil + return &res, nil + } -func getEventIfVisible(ctx context.Context, rsAPI roomserver.RoomserverInternalAPI, eventID, userID string) (*gomatrixserverlib.HeaderedEvent, bool) { - var queryEventsRes roomserver.QueryEventsByIDResponse - err := rsAPI.QueryEventsByID(ctx, &roomserver.QueryEventsByIDRequest{ - EventIDs: []string{eventID}, - }, &queryEventsRes) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("getEventIfVisible: failed to QueryEventsByID") - return nil, false +// authorisedToSeeEvent checks that the user or server is allowed to see this event. Returns true if allowed to +// see this request. This only needs to be done once per room at present as we just check for joined status. +func (rc *reqCtx) authorisedToSeeEvent(event *gomatrixserverlib.HeaderedEvent) bool { + if rc.isFederatedRequest { + // make sure the server is in this room + var res fs.QueryJoinedHostServerNamesInRoomResponse + err := rc.fsAPI.QueryJoinedHostServerNamesInRoom(rc.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ + RoomID: event.RoomID(), + }, &res) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Error("authorisedToSeeEvent: failed to QueryJoinedHostServerNamesInRoom") + return false + } + for _, srv := range res.ServerNames { + if srv == rc.serverName { + return true + } + } + return false } - if len(queryEventsRes.Events) == 0 { - util.GetLogger(ctx).Infof("event does not exist") - return nil, false // event does not exist - } - event := queryEventsRes.Events[0] - + // make sure the user is in this room // Allow events if the member is in the room // TODO: This does not honour history_visibility // TODO: This does not honour m.room.create content var queryMembershipRes roomserver.QueryMembershipForUserResponse - err = rsAPI.QueryMembershipForUser(ctx, &roomserver.QueryMembershipForUserRequest{ + err := rc.rsAPI.QueryMembershipForUser(rc.ctx, &roomserver.QueryMembershipForUserRequest{ RoomID: event.RoomID(), - UserID: userID, + UserID: rc.userID, }, &queryMembershipRes) if err != nil { - util.GetLogger(ctx).WithError(err).Error("getEventIfVisible: failed to QueryMembershipForUser") - return nil, false + util.GetLogger(rc.ctx).WithError(err).Error("authorisedToSeeEvent: failed to QueryMembershipForUser") + return false } - return event, queryMembershipRes.IsInRoom + return queryMembershipRes.IsInRoom +} + +func (rc *reqCtx) getServersForEventID(eventID string) []gomatrixserverlib.ServerName { + if rc.req.RoomID == "" { + util.GetLogger(rc.ctx).WithField("event_id", eventID).Error( + "getServersForEventID: event exists in unknown room", + ) + return nil + } + if rc.roomVersion == "" { + util.GetLogger(rc.ctx).WithField("event_id", eventID).Errorf( + "getServersForEventID: event exists in %s with unknown room version", rc.req.RoomID, + ) + return nil + } + var queryRes fs.QueryJoinedHostServerNamesInRoomResponse + err := rc.fsAPI.QueryJoinedHostServerNamesInRoom(rc.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ + RoomID: rc.req.RoomID, + }, &queryRes) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Error("getServersForEventID: failed to QueryJoinedHostServerNamesInRoom") + return nil + } + // query up to 5 servers + serversToQuery := queryRes.ServerNames + if len(serversToQuery) > 5 { + serversToQuery = serversToQuery[:5] + } + return serversToQuery +} + +func (rc *reqCtx) remoteEventRelationships(eventID string) *gomatrixserverlib.MSC2836EventRelationshipsResponse { + if rc.isFederatedRequest { + return nil // we don't query remote servers for remote requests + } + serversToQuery := rc.getServersForEventID(eventID) + var res *gomatrixserverlib.MSC2836EventRelationshipsResponse + var err error + for _, srv := range serversToQuery { + res, err = rc.MSC2836EventRelationships(eventID, srv, rc.roomVersion) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).WithField("server", srv).Error("remoteEventRelationships: failed to call MSC2836EventRelationships") + } else { + break + } + } + return res +} + +// lookForEvent returns the event for the event ID given, by trying to query remote servers +// if the event ID is unknown via /event_relationships. +func (rc *reqCtx) lookForEvent(eventID string) *gomatrixserverlib.HeaderedEvent { + event := rc.getLocalEvent(eventID) + if event == nil { + queryRes := rc.remoteEventRelationships(eventID) + if queryRes != nil { + // inject all the events into the roomserver then return the event in question + rc.injectResponseToRoomserver(queryRes) + for _, ev := range queryRes.Events { + if ev.EventID() == eventID && rc.req.RoomID == ev.RoomID() { + return ev.Headered(ev.Version()) + } + } + } + } else if rc.hasUnexploredChildren(eventID) { + // we have the local event but we may need to do a remote hit anyway if we are exploring the thread and have unknown children. + // If we don't do this then we risk never fetching the children. + queryRes := rc.remoteEventRelationships(eventID) + if queryRes != nil { + rc.injectResponseToRoomserver(queryRes) + err := rc.db.MarkChildrenExplored(context.Background(), eventID) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Warnf("failed to mark children of %s as explored", eventID) + } + } + } + if rc.req.RoomID == event.RoomID() { + return event + } + return nil +} + +func (rc *reqCtx) getLocalEvent(eventID string) *gomatrixserverlib.HeaderedEvent { + var queryEventsRes roomserver.QueryEventsByIDResponse + err := rc.rsAPI.QueryEventsByID(rc.ctx, &roomserver.QueryEventsByIDRequest{ + EventIDs: []string{eventID}, + }, &queryEventsRes) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Error("getLocalEvent: failed to QueryEventsByID") + return nil + } + if len(queryEventsRes.Events) == 0 { + util.GetLogger(rc.ctx).WithField("event_id", eventID).Infof("getLocalEvent: event does not exist") + return nil // event does not exist + } + return queryEventsRes.Events[0] +} + +// injectResponseToRoomserver injects the events +// into the roomserver as KindOutlier, with auth chains. +func (rc *reqCtx) injectResponseToRoomserver(res *gomatrixserverlib.MSC2836EventRelationshipsResponse) { + var stateEvents []*gomatrixserverlib.Event + var messageEvents []*gomatrixserverlib.Event + for _, ev := range res.Events { + if ev.StateKey() != nil { + stateEvents = append(stateEvents, ev) + } else { + messageEvents = append(messageEvents, ev) + } + } + respState := gomatrixserverlib.RespState{ + AuthEvents: res.AuthChain, + StateEvents: stateEvents, + } + eventsInOrder, err := respState.Events() + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Error("failed to calculate order to send events in MSC2836EventRelationshipsResponse") + return + } + // everything gets sent as an outlier because auth chain events may be disjoint from the DAG + // as may the threaded events. + var ires []roomserver.InputRoomEvent + for _, outlier := range append(eventsInOrder, messageEvents...) { + ires = append(ires, roomserver.InputRoomEvent{ + Kind: roomserver.KindOutlier, + Event: outlier.Headered(outlier.Version()), + AuthEventIDs: outlier.AuthEventIDs(), + }) + } + // we've got the data by this point so use a background context + err = roomserver.SendInputRoomEvents(context.Background(), rc.rsAPI, ires) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Error("failed to inject MSC2836EventRelationshipsResponse into the roomserver") + } + // update the child count / hash columns for these nodes. We need to do this here because not all events will make it + // through to the KindNewEventPersisted hook because the roomserver will ignore duplicates. Duplicates have meaning though + // as the `unsigned` field may differ (if the number of children changes). + for _, ev := range ires { + err = rc.db.UpdateChildMetadata(context.Background(), ev.Event) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).WithField("event_id", ev.Event.EventID()).Warn("failed to update child metadata for event") + } + } +} + +func (rc *reqCtx) addChildMetadata(ev *gomatrixserverlib.HeaderedEvent) { + count, hash := rc.getChildMetadata(ev.EventID()) + if count == 0 { + return + } + err := ev.SetUnsignedField("children_hash", gomatrixserverlib.Base64Bytes(hash)) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Warn("Failed to set children_hash") + } + err = ev.SetUnsignedField("children", map[string]int{ + constRelType: count, + }) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Warn("Failed to set children count") + } +} + +func (rc *reqCtx) getChildMetadata(eventID string) (count int, hash []byte) { + children, err := rc.db.ChildrenForParent(rc.ctx, eventID, constRelType, false) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).Warn("Failed to get ChildrenForParent for getting child metadata") + return + } + if len(children) == 0 { + return + } + // sort it lexiographically + sort.Slice(children, func(i, j int) bool { + return children[i].EventID < children[j].EventID + }) + // hash it + var eventIDs strings.Builder + for _, c := range children { + _, _ = eventIDs.WriteString(c.EventID) + } + hashValBytes := sha256.Sum256([]byte(eventIDs.String())) + + count = len(children) + hash = hashValBytes[:] + return +} + +// hasUnexploredChildren returns true if this event has unexplored children. +// "An event has unexplored children if the `unsigned` child count on the parent does not match +// how many children the server believes the parent to have. In addition, if the counts match but +// the hashes do not match, then the event is unexplored." +func (rc *reqCtx) hasUnexploredChildren(eventID string) bool { + if rc.isFederatedRequest { + return false // we only explore children for clients, not servers. + } + // extract largest child count from event + eventCount, eventHash, explored, err := rc.db.ChildMetadata(rc.ctx, eventID) + if err != nil { + util.GetLogger(rc.ctx).WithError(err).WithField("event_id", eventID).Warn( + "failed to get ChildMetadata from db", + ) + return false + } + // if there are no recorded children then we know we have >= children. + // if the event has already been explored (read: we hit /event_relationships successfully) + // then don't do it again. We'll only re-do this if we get an even bigger children count, + // see Database.UpdateChildMetadata + if eventCount == 0 || explored { + return false // short-circuit + } + + // calculate child count for event + calcCount, calcHash := rc.getChildMetadata(eventID) + + if eventCount < calcCount { + return false // we have more children + } else if eventCount > calcCount { + return true // the event has more children than we know about + } + // we have the same count, so a mismatched hash means some children are different + return !bytes.Equal(eventHash, calcHash) } type walkInfo struct { @@ -453,9 +758,9 @@ type walker struct { // WalkFrom the event ID given func (w *walker) WalkFrom(eventID string) (limited bool, err error) { - children, err := w.db.ChildrenForParent(w.ctx, eventID, constRelType, w.req.RecentFirst) + children, err := w.childrenForParent(eventID) if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("WalkFrom() ChildrenForParent failed, cannot walk") + util.GetLogger(w.ctx).WithError(err).Error("WalkFrom() childrenForParent failed, cannot walk") return false, err } var next *walkInfo @@ -467,9 +772,9 @@ func (w *walker) WalkFrom(eventID string) (limited bool, err error) { return true, nil } // find the children's children - children, err = w.db.ChildrenForParent(w.ctx, next.EventID, constRelType, w.req.RecentFirst) + children, err = w.childrenForParent(next.EventID) if err != nil { - util.GetLogger(w.ctx).WithError(err).Error("WalkFrom() ChildrenForParent failed, cannot walk") + util.GetLogger(w.ctx).WithError(err).Error("WalkFrom() childrenForParent failed, cannot walk") return false, err } toWalk = w.addChildren(toWalk, children, next.Depth+1) @@ -528,3 +833,20 @@ func (w *walker) nextChild(toWalk []walkInfo) (*walkInfo, []walkInfo) { child, toWalk = toWalk[0], toWalk[1:] return &child, toWalk } + +// childrenForParent returns the children events for this event ID, honouring the direction: up|down flags +// meaning this can actually be returning the parent for the event instead of the children. +func (w *walker) childrenForParent(eventID string) ([]eventInfo, error) { + if w.req.Direction == "down" { + return w.db.ChildrenForParent(w.ctx, eventID, constRelType, w.req.RecentFirst) + } + // find the event to pull out the parent + ei, err := w.db.ParentForChild(w.ctx, eventID, constRelType) + if err != nil { + return nil, err + } + if ei != nil { + return []eventInfo{*ei}, nil + } + return nil, nil +} diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 996cc79f0..4eb5708c1 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -4,10 +4,14 @@ import ( "bytes" "context" "crypto/ed25519" + "crypto/sha256" + "encoding/base64" "encoding/json" "fmt" "io/ioutil" "net/http" + "sort" + "strings" "testing" "time" @@ -43,9 +47,7 @@ func TestMSC2836(t *testing.T) { alice := "@alice:localhost" bob := "@bob:localhost" charlie := "@charlie:localhost" - roomIDA := "!alice:localhost" - roomIDB := "!bob:localhost" - roomIDC := "!charlie:localhost" + roomID := "!alice:localhost" // give access tokens to all three users nopUserAPI := &testUserAPI{ accessTokens: make(map[string]userapi.Device), @@ -66,7 +68,7 @@ func TestMSC2836(t *testing.T) { UserID: charlie, } eventA := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDA, + RoomID: roomID, Sender: alice, Type: "m.room.message", Content: map[string]interface{}{ @@ -74,7 +76,7 @@ func TestMSC2836(t *testing.T) { }, }) eventB := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDB, + RoomID: roomID, Sender: bob, Type: "m.room.message", Content: map[string]interface{}{ @@ -86,7 +88,7 @@ func TestMSC2836(t *testing.T) { }, }) eventC := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDB, + RoomID: roomID, Sender: bob, Type: "m.room.message", Content: map[string]interface{}{ @@ -98,7 +100,7 @@ func TestMSC2836(t *testing.T) { }, }) eventD := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDA, + RoomID: roomID, Sender: alice, Type: "m.room.message", Content: map[string]interface{}{ @@ -110,7 +112,7 @@ func TestMSC2836(t *testing.T) { }, }) eventE := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDB, + RoomID: roomID, Sender: bob, Type: "m.room.message", Content: map[string]interface{}{ @@ -122,7 +124,7 @@ func TestMSC2836(t *testing.T) { }, }) eventF := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDC, + RoomID: roomID, Sender: charlie, Type: "m.room.message", Content: map[string]interface{}{ @@ -134,7 +136,7 @@ func TestMSC2836(t *testing.T) { }, }) eventG := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDA, + RoomID: roomID, Sender: alice, Type: "m.room.message", Content: map[string]interface{}{ @@ -146,7 +148,7 @@ func TestMSC2836(t *testing.T) { }, }) eventH := mustCreateEvent(t, fledglingEvent{ - RoomID: roomIDB, + RoomID: roomID, Sender: bob, Type: "m.room.message", Content: map[string]interface{}{ @@ -160,9 +162,9 @@ func TestMSC2836(t *testing.T) { // make everyone joined to each other's rooms nopRsAPI := &testRoomserverAPI{ userToJoinedRooms: map[string][]string{ - alice: []string{roomIDA, roomIDB, roomIDC}, - bob: []string{roomIDA, roomIDB, roomIDC}, - charlie: []string{roomIDA, roomIDB, roomIDC}, + alice: []string{roomID}, + bob: []string{roomID}, + charlie: []string{roomID}, }, events: map[string]*gomatrixserverlib.HeaderedEvent{ eventA.EventID(): eventA, @@ -198,21 +200,6 @@ func TestMSC2836(t *testing.T) { "include_parent": true, })) }) - t.Run("omits parent if not joined to the room of parent of event", func(t *testing.T) { - nopUserAPI.accessTokens["frank2"] = userapi.Device{ - AccessToken: "frank2", - DisplayName: "Frank2 Not In Room", - UserID: "@frank2:localhost", - } - // Event B is in roomB, Event A is in roomA, so make frank2 joined to roomB - nopRsAPI.userToJoinedRooms["@frank2:localhost"] = []string{roomIDB} - body := postRelationships(t, 200, "frank2", newReq(t, map[string]interface{}{ - "event_id": eventB.EventID(), - "limit": 1, - "include_parent": true, - })) - assertContains(t, body, []string{eventB.EventID()}) - }) t.Run("returns the parent if include_parent is true", func(t *testing.T) { body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{ "event_id": eventB.EventID(), @@ -349,6 +336,39 @@ func TestMSC2836(t *testing.T) { })) assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID(), eventE.EventID(), eventF.EventID(), eventG.EventID(), eventH.EventID()}) }) + t.Run("can navigate up the graph with direction: up", func(t *testing.T) { + // A4 + // | + // B3 + // / \ + // C D2 + // /| \ + // E F1 G + // | + // H + body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{ + "event_id": eventF.EventID(), + "recent_first": false, + "depth_first": true, + "direction": "up", + })) + assertContains(t, body, []string{eventF.EventID(), eventD.EventID(), eventB.EventID(), eventA.EventID()}) + }) + t.Run("includes children and children_hash in unsigned", func(t *testing.T) { + body := postRelationships(t, 200, "alice", newReq(t, map[string]interface{}{ + "event_id": eventB.EventID(), + "recent_first": false, + "depth_first": false, + "limit": 3, + })) + // event B has C,D as children + // event C has no children + // event D has 3 children (not included in response) + assertContains(t, body, []string{eventB.EventID(), eventC.EventID(), eventD.EventID()}) + assertUnsignedChildren(t, body.Events[0], "m.reference", 2, []string{eventC.EventID(), eventD.EventID()}) + assertUnsignedChildren(t, body.Events[1], "", 0, nil) + assertUnsignedChildren(t, body.Events[2], "m.reference", 3, []string{eventE.EventID(), eventF.EventID(), eventG.EventID()}) + }) } // TODO: TestMSC2836TerminatesLoops (short and long) @@ -411,8 +431,12 @@ func postRelationships(t *testing.T, expectCode int, accessToken string, req *ms } if res.StatusCode == 200 { var result msc2836.EventRelationshipResponse - if err := json.NewDecoder(res.Body).Decode(&result); err != nil { - t.Fatalf("response 200 OK but failed to deserialise JSON : %s", err) + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("response 200 OK but failed to read response body: %s", err) + } + if err := json.Unmarshal(body, &result); err != nil { + t.Fatalf("response 200 OK but failed to deserialise JSON : %s\nbody: %s", err, string(body)) } return &result } @@ -435,6 +459,43 @@ func assertContains(t *testing.T, result *msc2836.EventRelationshipResponse, wan } } +func assertUnsignedChildren(t *testing.T, ev gomatrixserverlib.ClientEvent, relType string, wantCount int, childrenEventIDs []string) { + t.Helper() + unsigned := struct { + Children map[string]int `json:"children"` + Hash string `json:"children_hash"` + }{} + if err := json.Unmarshal(ev.Unsigned, &unsigned); err != nil { + if wantCount == 0 { + return // no children so possible there is no unsigned field at all + } + t.Fatalf("Failed to unmarshal unsigned field: %s", err) + } + // zero checks + if wantCount == 0 { + if len(unsigned.Children) != 0 || unsigned.Hash != "" { + t.Fatalf("want 0 children but got unsigned fields %+v", unsigned) + } + return + } + gotCount := unsigned.Children[relType] + if gotCount != wantCount { + t.Errorf("Got %d count, want %d count for rel_type %s", gotCount, wantCount, relType) + } + // work out the hash + sort.Strings(childrenEventIDs) + var b strings.Builder + for _, s := range childrenEventIDs { + b.WriteString(s) + } + t.Logf("hashing %s", b.String()) + hashValBytes := sha256.Sum256([]byte(b.String())) + wantHash := base64.RawStdEncoding.EncodeToString(hashValBytes[:]) + if wantHash != unsigned.Hash { + t.Errorf("Got unsigned hash %s want hash %s", unsigned.Hash, wantHash) + } +} + type testUserAPI struct { accessTokens map[string]userapi.Device } diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go index 72ea5195b..72523916b 100644 --- a/setup/mscs/msc2836/storage.go +++ b/setup/mscs/msc2836/storage.go @@ -1,20 +1,22 @@ package msc2836 import ( + "bytes" "context" "database/sql" + "encoding/base64" "encoding/json" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) type eventInfo struct { EventID string OriginServerTS gomatrixserverlib.Timestamp RoomID string - Servers []string } type Database interface { @@ -25,6 +27,21 @@ type Database interface { // provided `relType`. The returned slice is sorted by origin_server_ts according to whether // `recentFirst` is true or false. ChildrenForParent(ctx context.Context, eventID, relType string, recentFirst bool) ([]eventInfo, error) + // ParentForChild returns the parent event for the given child `eventID`. The eventInfo should be nil if + // there is no parent for this child event, with no error. The parent eventInfo can be missing the + // timestamp if the event is not known to the server. + ParentForChild(ctx context.Context, eventID, relType string) (*eventInfo, error) + // UpdateChildMetadata persists the children_count and children_hash from this event if and only if + // the count is greater than what was previously there. If the count is updated, the event will be + // updated to be unexplored. + UpdateChildMetadata(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error + // ChildMetadata returns the children_count and children_hash for the event ID in question. + // Also returns the `explored` flag, which is set to true when MarkChildrenExplored is called and is set + // back to `false` when a larger count is inserted via UpdateChildMetadata. + // Returns nil error if the event ID does not exist. + ChildMetadata(ctx context.Context, eventID string) (count int, hash []byte, explored bool, err error) + // MarkChildrenExplored sets the 'explored' flag on this event to `true`. + MarkChildrenExplored(ctx context.Context, eventID string) error } type DB struct { @@ -34,6 +51,10 @@ type DB struct { insertNodeStmt *sql.Stmt selectChildrenForParentOldestFirstStmt *sql.Stmt selectChildrenForParentRecentFirstStmt *sql.Stmt + selectParentForChildStmt *sql.Stmt + updateChildMetadataStmt *sql.Stmt + selectChildMetadataStmt *sql.Stmt + updateChildMetadataExploredStmt *sql.Stmt } // NewDatabase loads the database for msc2836 @@ -65,19 +86,26 @@ func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) { CREATE TABLE IF NOT EXISTS msc2836_nodes ( event_id TEXT PRIMARY KEY NOT NULL, origin_server_ts BIGINT NOT NULL, - room_id TEXT NOT NULL + room_id TEXT NOT NULL, + unsigned_children_count BIGINT NOT NULL, + unsigned_children_hash TEXT NOT NULL, + explored SMALLINT NOT NULL ); `) if err != nil { return nil, err } if d.insertEdgeStmt, err = d.db.Prepare(` - INSERT INTO msc2836_edges(parent_event_id, child_event_id, rel_type, parent_room_id, parent_servers) VALUES($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING + INSERT INTO msc2836_edges(parent_event_id, child_event_id, rel_type, parent_room_id, parent_servers) + VALUES($1, $2, $3, $4, $5) + ON CONFLICT DO NOTHING `); err != nil { return nil, err } if d.insertNodeStmt, err = d.db.Prepare(` - INSERT INTO msc2836_nodes(event_id, origin_server_ts, room_id) VALUES($1, $2, $3) ON CONFLICT DO NOTHING + INSERT INTO msc2836_nodes(event_id, origin_server_ts, room_id, unsigned_children_count, unsigned_children_hash, explored) + VALUES($1, $2, $3, $4, $5, $6) + ON CONFLICT DO NOTHING `); err != nil { return nil, err } @@ -93,6 +121,27 @@ func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) { if d.selectChildrenForParentRecentFirstStmt, err = d.db.Prepare(selectChildrenQuery + "DESC"); err != nil { return nil, err } + if d.selectParentForChildStmt, err = d.db.Prepare(` + SELECT parent_event_id, parent_room_id FROM msc2836_edges + WHERE child_event_id = $1 AND rel_type = $2 + `); err != nil { + return nil, err + } + if d.updateChildMetadataStmt, err = d.db.Prepare(` + UPDATE msc2836_nodes SET unsigned_children_count=$1, unsigned_children_hash=$2, explored=$3 WHERE event_id=$4 + `); err != nil { + return nil, err + } + if d.selectChildMetadataStmt, err = d.db.Prepare(` + SELECT unsigned_children_count, unsigned_children_hash, explored FROM msc2836_nodes WHERE event_id=$1 + `); err != nil { + return nil, err + } + if d.updateChildMetadataExploredStmt, err = d.db.Prepare(` + UPDATE msc2836_nodes SET explored=$1 WHERE event_id=$2 + `); err != nil { + return nil, err + } return &d, err } @@ -117,19 +166,26 @@ func newSQLiteDatabase(dbOpts *config.DatabaseOptions) (Database, error) { CREATE TABLE IF NOT EXISTS msc2836_nodes ( event_id TEXT PRIMARY KEY NOT NULL, origin_server_ts BIGINT NOT NULL, - room_id TEXT NOT NULL + room_id TEXT NOT NULL, + unsigned_children_count BIGINT NOT NULL, + unsigned_children_hash TEXT NOT NULL, + explored SMALLINT NOT NULL ); `) if err != nil { return nil, err } if d.insertEdgeStmt, err = d.db.Prepare(` - INSERT INTO msc2836_edges(parent_event_id, child_event_id, rel_type, parent_room_id, parent_servers) VALUES($1, $2, $3, $4, $5) ON CONFLICT (parent_event_id, child_event_id, rel_type) DO NOTHING + INSERT INTO msc2836_edges(parent_event_id, child_event_id, rel_type, parent_room_id, parent_servers) + VALUES($1, $2, $3, $4, $5) + ON CONFLICT (parent_event_id, child_event_id, rel_type) DO NOTHING `); err != nil { return nil, err } if d.insertNodeStmt, err = d.db.Prepare(` - INSERT INTO msc2836_nodes(event_id, origin_server_ts, room_id) VALUES($1, $2, $3) ON CONFLICT DO NOTHING + INSERT INTO msc2836_nodes(event_id, origin_server_ts, room_id, unsigned_children_count, unsigned_children_hash, explored) + VALUES($1, $2, $3, $4, $5, $6) + ON CONFLICT DO NOTHING `); err != nil { return nil, err } @@ -145,6 +201,27 @@ func newSQLiteDatabase(dbOpts *config.DatabaseOptions) (Database, error) { if d.selectChildrenForParentRecentFirstStmt, err = d.db.Prepare(selectChildrenQuery + "DESC"); err != nil { return nil, err } + if d.selectParentForChildStmt, err = d.db.Prepare(` + SELECT parent_event_id, parent_room_id FROM msc2836_edges + WHERE child_event_id = $1 AND rel_type = $2 + `); err != nil { + return nil, err + } + if d.updateChildMetadataStmt, err = d.db.Prepare(` + UPDATE msc2836_nodes SET unsigned_children_count=$1, unsigned_children_hash=$2, explored=$3 WHERE event_id=$4 + `); err != nil { + return nil, err + } + if d.selectChildMetadataStmt, err = d.db.Prepare(` + SELECT unsigned_children_count, unsigned_children_hash, explored FROM msc2836_nodes WHERE event_id=$1 + `); err != nil { + return nil, err + } + if d.updateChildMetadataExploredStmt, err = d.db.Prepare(` + UPDATE msc2836_nodes SET explored=$1 WHERE event_id=$2 + `); err != nil { + return nil, err + } return &d, nil } @@ -158,16 +235,55 @@ func (p *DB) StoreRelation(ctx context.Context, ev *gomatrixserverlib.HeaderedEv if err != nil { return err } + count, hash := extractChildMetadata(ev) return p.writer.Do(p.db, nil, func(txn *sql.Tx) error { _, err := txn.Stmt(p.insertEdgeStmt).ExecContext(ctx, parent, child, relType, relationRoomID, string(relationServersJSON)) if err != nil { return err } - _, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID()) + util.GetLogger(ctx).Infof("StoreRelation child=%s parent=%s rel_type=%s", child, parent, relType) + _, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID(), count, base64.RawStdEncoding.EncodeToString(hash), 0) return err }) } +func (p *DB) UpdateChildMetadata(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error { + eventCount, eventHash := extractChildMetadata(ev) + if eventCount == 0 { + return nil // nothing to update with + } + + // extract current children count/hash, if they are less than the current event then update the columns and set to unexplored + count, hash, _, err := p.ChildMetadata(ctx, ev.EventID()) + if err != nil { + return err + } + if eventCount > count || (eventCount == count && !bytes.Equal(hash, eventHash)) { + _, err = p.updateChildMetadataStmt.ExecContext(ctx, eventCount, base64.RawStdEncoding.EncodeToString(eventHash), 0, ev.EventID()) + return err + } + return nil +} + +func (p *DB) ChildMetadata(ctx context.Context, eventID string) (count int, hash []byte, explored bool, err error) { + var b64hash string + var exploredInt int + if err = p.selectChildMetadataStmt.QueryRowContext(ctx, eventID).Scan(&count, &b64hash, &exploredInt); err != nil { + if err == sql.ErrNoRows { + err = nil + } + return + } + hash, err = base64.RawStdEncoding.DecodeString(b64hash) + explored = exploredInt > 0 + return +} + +func (p *DB) MarkChildrenExplored(ctx context.Context, eventID string) error { + _, err := p.updateChildMetadataExploredStmt.ExecContext(ctx, 1, eventID) + return err +} + func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, recentFirst bool) ([]eventInfo, error) { var rows *sql.Rows var err error @@ -191,6 +307,17 @@ func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, rec return children, nil } +func (p *DB) ParentForChild(ctx context.Context, eventID, relType string) (*eventInfo, error) { + var ei eventInfo + err := p.selectParentForChildStmt.QueryRowContext(ctx, eventID, relType).Scan(&ei.EventID, &ei.RoomID) + if err == sql.ErrNoRows { + return nil, nil + } else if err != nil { + return nil, err + } + return &ei, nil +} + func parentChildEventIDs(ev *gomatrixserverlib.HeaderedEvent) (parent, child, relType string) { if ev == nil { return @@ -224,3 +351,19 @@ func roomIDAndServers(ev *gomatrixserverlib.HeaderedEvent) (roomID string, serve } return body.RoomID, body.Servers } + +func extractChildMetadata(ev *gomatrixserverlib.HeaderedEvent) (count int, hash []byte) { + unsigned := struct { + Counts map[string]int `json:"children"` + Hash gomatrixserverlib.Base64Bytes `json:"children_hash"` + }{} + if err := json.Unmarshal(ev.Unsigned(), &unsigned); err != nil { + // expected if there is no unsigned field at all + return + } + for _, c := range unsigned.Counts { + count += c + } + hash = unsigned.Hash + return +} diff --git a/setup/mscs/mscs.go b/setup/mscs/mscs.go index 8b0498ced..a8e5668ea 100644 --- a/setup/mscs/mscs.go +++ b/setup/mscs/mscs.go @@ -16,15 +16,18 @@ package mscs import ( + "context" "fmt" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/mscs/msc2836" + "github.com/matrix-org/util" ) // Enable MSCs - returns an error on unknown MSCs func Enable(base *setup.BaseDendrite, monolith *setup.Monolith) error { for _, msc := range base.Cfg.MSCs.MSCs { + util.GetLogger(context.Background()).WithField("msc", msc).Info("Enabling MSC") if err := EnableMSC(base, monolith, msc); err != nil { return err } From 5d65a879a55e216e047574f54f69e55724546ec4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 4 Dec 2020 14:52:10 +0000 Subject: [PATCH 06/13] Federation sender event cache (#1614) * Cache federation sender events * Store in the correct cache * Update federation event cache * Fix Unset * Give EDUs same caching treatment as PDUs * Make federationsender_cache_size configurable * Default caches configuration * Fix unit tests * Revert "Fix unit tests" This reverts commit 24eb5d22524f20e1024b1475debe61ae20538a5a. * Revert "Default caches configuration" This reverts commit 464ecd1e64b9d2983f6fd5430e9607519d543cb3. * Revert "Make federationsender_cache_size configurable" This reverts commit 4631f5324151e006a15d6f19008f06361b994607. --- federationsender/federationsender.go | 2 +- federationsender/storage/postgres/storage.go | 4 +- federationsender/storage/shared/storage.go | 2 + .../storage/shared/storage_edus.go | 12 +++- .../storage/shared/storage_pdus.go | 15 ++++- federationsender/storage/sqlite3/storage.go | 4 +- federationsender/storage/storage.go | 7 +- federationsender/storage/storage_wasm.go | 5 +- internal/caching/cache_federationevents.go | 67 +++++++++++++++++++ internal/caching/caches.go | 1 + internal/caching/impl_inmemorylru.go | 10 +++ 11 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 internal/caching/cache_federationevents.go diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index fc0ba6d5f..a24e0f488 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -46,7 +46,7 @@ func NewInternalAPI( ) api.FederationSenderInternalAPI { cfg := &base.Cfg.FederationSender - federationSenderDB, err := storage.NewDatabase(&cfg.Database) + federationSenderDB, err := storage.NewDatabase(&cfg.Database, base.Caches) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index f314f8493..75b54bbcb 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationsender/storage/shared" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" ) @@ -32,7 +33,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open(dbProperties); err != nil { @@ -65,6 +66,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { } d.Database = shared.Database{ DB: d.db, + Cache: cache, Writer: d.writer, FederationSenderJoinedHosts: joinedHosts, FederationSenderQueuePDUs: queuePDUs, diff --git a/federationsender/storage/shared/storage.go b/federationsender/storage/shared/storage.go index d5731f31c..af9d0d6a3 100644 --- a/federationsender/storage/shared/storage.go +++ b/federationsender/storage/shared/storage.go @@ -22,12 +22,14 @@ import ( "github.com/matrix-org/dendrite/federationsender/storage/tables" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) type Database struct { DB *sql.DB + Cache caching.FederationSenderCache Writer sqlutil.Writer FederationSenderQueuePDUs tables.FederationSenderQueuePDUs FederationSenderQueueEDUs tables.FederationSenderQueueEDUs diff --git a/federationsender/storage/shared/storage_edus.go b/federationsender/storage/shared/storage_edus.go index 529b46aa9..ae1d15118 100644 --- a/federationsender/storage/shared/storage_edus.go +++ b/federationsender/storage/shared/storage_edus.go @@ -69,7 +69,16 @@ func (d *Database) GetNextTransactionEDUs( nids: nids, } - blobs, err := d.FederationSenderQueueJSON.SelectQueueJSON(ctx, txn, nids) + retrieve := make([]int64, 0, len(nids)) + for _, nid := range nids { + if edu, ok := d.Cache.GetFederationSenderQueuedEDU(nid); ok { + edus = append(edus, edu) + } else { + retrieve = append(retrieve, nid) + } + } + + blobs, err := d.FederationSenderQueueJSON.SelectQueueJSON(ctx, txn, retrieve) if err != nil { return fmt.Errorf("SelectQueueJSON: %w", err) } @@ -111,6 +120,7 @@ func (d *Database) CleanEDUs( } if count == 0 { deleteNIDs = append(deleteNIDs, nid) + d.Cache.EvictFederationSenderQueuedEDU(nid) } } diff --git a/federationsender/storage/shared/storage_pdus.go b/federationsender/storage/shared/storage_pdus.go index 9ab0b094c..09235a5ec 100644 --- a/federationsender/storage/shared/storage_pdus.go +++ b/federationsender/storage/shared/storage_pdus.go @@ -85,17 +85,27 @@ func (d *Database) GetNextTransactionPDUs( nids: nids, } - blobs, err := d.FederationSenderQueueJSON.SelectQueueJSON(ctx, txn, nids) + retrieve := make([]int64, 0, len(nids)) + for _, nid := range nids { + if event, ok := d.Cache.GetFederationSenderQueuedPDU(nid); ok { + events = append(events, event) + } else { + retrieve = append(retrieve, nid) + } + } + + blobs, err := d.FederationSenderQueueJSON.SelectQueueJSON(ctx, txn, retrieve) if err != nil { return fmt.Errorf("SelectQueueJSON: %w", err) } - for _, blob := range blobs { + for nid, blob := range blobs { var event gomatrixserverlib.HeaderedEvent if err := json.Unmarshal(blob, &event); err != nil { return fmt.Errorf("json.Unmarshal: %w", err) } events = append(events, &event) + d.Cache.StoreFederationSenderQueuedPDU(nid, &event) } return nil @@ -128,6 +138,7 @@ func (d *Database) CleanPDUs( } if count == 0 { deleteNIDs = append(deleteNIDs, nid) + d.Cache.EvictFederationSenderQueuedPDU(nid) } } diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 4f663f645..e66d76909 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -21,6 +21,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/federationsender/storage/shared" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" ) @@ -34,7 +35,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open(dbProperties); err != nil { @@ -67,6 +68,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { } d.Database = shared.Database{ DB: d.db, + Cache: cache, Writer: d.writer, FederationSenderJoinedHosts: joinedHosts, FederationSenderQueuePDUs: queuePDUs, diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index f36138229..5462c3523 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -21,16 +21,17 @@ import ( "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup/config" ) // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(dbProperties) + return sqlite3.NewDatabase(dbProperties, cache) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(dbProperties) + return postgres.NewDatabase(dbProperties, cache) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index c35b4828c..bc52bd9bb 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -18,14 +18,15 @@ import ( "fmt" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup/config" ) // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(dbProperties) + return sqlite3.NewDatabase(dbProperties, cache) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/internal/caching/cache_federationevents.go b/internal/caching/cache_federationevents.go new file mode 100644 index 000000000..a48c11fd2 --- /dev/null +++ b/internal/caching/cache_federationevents.go @@ -0,0 +1,67 @@ +package caching + +import ( + "fmt" + + "github.com/matrix-org/gomatrixserverlib" +) + +const ( + FederationEventCacheName = "federation_event" + FederationEventCacheMaxEntries = 256 + FederationEventCacheMutable = true // to allow use of Unset only +) + +// FederationSenderCache contains the subset of functions needed for +// a federation event cache. +type FederationSenderCache interface { + GetFederationSenderQueuedPDU(eventNID int64) (event *gomatrixserverlib.HeaderedEvent, ok bool) + StoreFederationSenderQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) + EvictFederationSenderQueuedPDU(eventNID int64) + + GetFederationSenderQueuedEDU(eventNID int64) (event *gomatrixserverlib.EDU, ok bool) + StoreFederationSenderQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) + EvictFederationSenderQueuedEDU(eventNID int64) +} + +func (c Caches) GetFederationSenderQueuedPDU(eventNID int64) (*gomatrixserverlib.HeaderedEvent, bool) { + key := fmt.Sprintf("%d", eventNID) + val, found := c.FederationEvents.Get(key) + if found && val != nil { + if event, ok := val.(*gomatrixserverlib.HeaderedEvent); ok { + return event, true + } + } + return nil, false +} + +func (c Caches) StoreFederationSenderQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) { + key := fmt.Sprintf("%d", eventNID) + c.FederationEvents.Set(key, event) +} + +func (c Caches) EvictFederationSenderQueuedPDU(eventNID int64) { + key := fmt.Sprintf("%d", eventNID) + c.FederationEvents.Unset(key) +} + +func (c Caches) GetFederationSenderQueuedEDU(eventNID int64) (*gomatrixserverlib.EDU, bool) { + key := fmt.Sprintf("%d", eventNID) + val, found := c.FederationEvents.Get(key) + if found && val != nil { + if event, ok := val.(*gomatrixserverlib.EDU); ok { + return event, true + } + } + return nil, false +} + +func (c Caches) StoreFederationSenderQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) { + key := fmt.Sprintf("%d", eventNID) + c.FederationEvents.Set(key, event) +} + +func (c Caches) EvictFederationSenderQueuedEDU(eventNID int64) { + key := fmt.Sprintf("%d", eventNID) + c.FederationEvents.Unset(key) +} diff --git a/internal/caching/caches.go b/internal/caching/caches.go index 655cc037c..e7b7f550d 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -10,6 +10,7 @@ type Caches struct { RoomServerEventTypeNIDs Cache // RoomServerNIDsCache RoomServerRoomNIDs Cache // RoomServerNIDsCache RoomServerRoomIDs Cache // RoomServerNIDsCache + FederationEvents Cache // FederationEventsCache } // Cache is the interface that an implementation must satisfy. diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go index e99c18d74..f05e8f3c6 100644 --- a/internal/caching/impl_inmemorylru.go +++ b/internal/caching/impl_inmemorylru.go @@ -63,6 +63,15 @@ func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) { if err != nil { return nil, err } + federationEvents, err := NewInMemoryLRUCachePartition( + FederationEventCacheName, + FederationEventCacheMutable, + FederationEventCacheMaxEntries, + enablePrometheus, + ) + if err != nil { + return nil, err + } return &Caches{ RoomVersions: roomVersions, ServerKeys: serverKeys, @@ -70,6 +79,7 @@ func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) { RoomServerEventTypeNIDs: roomServerEventTypeNIDs, RoomServerRoomNIDs: roomServerRoomNIDs, RoomServerRoomIDs: roomServerRoomIDs, + FederationEvents: federationEvents, }, nil } From 8f5393db5f4d615452dcc807ddafd4841d9abd5b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 7 Dec 2020 10:20:19 +0000 Subject: [PATCH 07/13] Update FAQ --- README.md | 3 +-- docs/FAQ.md | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40fd69ead..7d79bbff0 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ a [brand new Go test suite](https://github.com/matrix-org/complement). - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments. - As of October 2020, Dendrite has now entered **beta** which means: - Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database. - Dendrite has periodic semver releases. We intend to release new versions as we land significant features. @@ -24,7 +23,7 @@ This does not mean: Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode. -Join us in: +If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in: - **[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org)** - General chat about the Dendrite project, for users and server admins alike - **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens diff --git a/docs/FAQ.md b/docs/FAQ.md index 9267aed32..37c6b34c5 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -12,6 +12,10 @@ No, although a good portion of the Matrix specification has been implemented. Mo No, not at present. There will be in the future when Dendrite reaches version 1.0. +### Should I run a monolith or a polylith deployment? + +Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines. + ### I've installed Dendrite but federation isn't working Check the [Federation Tester](https://federationtester.matrix.org). You need at least: From 45dec2059383a7f2f90b75f3b327df3b0309f59e Mon Sep 17 00:00:00 2001 From: bodqhrohro Date: Mon, 7 Dec 2020 13:23:01 +0200 Subject: [PATCH 08/13] Update links to golangci-lint documentation (#1617) Signed-off-by: Bohdan Horbeshko --- build/scripts/find-lint.sh | 2 +- docs/CODE_STYLE.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/scripts/find-lint.sh b/build/scripts/find-lint.sh index 7e37e1548..eb1b28d84 100755 --- a/build/scripts/find-lint.sh +++ b/build/scripts/find-lint.sh @@ -8,7 +8,7 @@ # - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run, # golangci-lint defaults this to NumCPU # - `GOGC` - how often to perform garbage collection during golangci-lint runs. -# Essentially a ratio of memory/speed. See https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint +# Essentially a ratio of memory/speed. See https://golangci-lint.run/usage/performance/#memory-usage # for more info. diff --git a/docs/CODE_STYLE.md b/docs/CODE_STYLE.md index 8f1c1cb58..8096ae27c 100644 --- a/docs/CODE_STYLE.md +++ b/docs/CODE_STYLE.md @@ -2,13 +2,13 @@ In addition to standard Go code style (`gofmt`, `goimports`), we use `golangci-lint` to run a number of linters, the exact list can be found under linters in [.golangci.yml](.golangci.yml). -[Installation](https://github.com/golangci/golangci-lint#install) and [Editor -Integration](https://github.com/golangci/golangci-lint#editor-integration) for +[Installation](https://github.com/golangci/golangci-lint#install-golangci-lint) and [Editor +Integration](https://golangci-lint.run/usage/integrations/#editor-integration) for it can be found in the readme of golangci-lint. For rare cases where a linter is giving a spurious warning, it can be disabled for that line or statement using a [comment -directive](https://github.com/golangci/golangci-lint#nolint), e.g. `var +directive](https://golangci-lint.run/usage/false-positives/#nolint), e.g. `var bad_name int //nolint:golint,unused`. This should be used sparingly and only when its clear that the lint warning is spurious. From e1d32e2ff1bef346836f6b9d795d33ec49dd3864 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 7 Dec 2020 16:21:55 +0000 Subject: [PATCH 09/13] Update to matrix-org/gomatrixserverlib@232bace --- go.mod | 3 +-- go.sum | 12 ++---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 0d39cf03b..e788f43de 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/golang-lru v0.5.4 - github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/lib/pq v1.8.0 github.com/libp2p/go-libp2p v0.11.0 github.com/libp2p/go-libp2p-circuit v0.3.1 @@ -23,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb + github.com/matrix-org/gomatrixserverlib v0.0.0-20201207162041-232bace0cdd4 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 3bd186ce9..f00f06d6f 100644 --- a/go.sum +++ b/go.sum @@ -301,8 +301,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -569,8 +567,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb h1:cN+v/rDbGg2p5e8xyxfATtwXeW7tI4aUn7slLXvuNyw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201207162041-232bace0cdd4 h1:geIU3lUL+P83W6NdQLUaR7vfVh5EHliIJ0H/RgCqbIU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201207162041-232bace0cdd4/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -779,8 +777,6 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= @@ -828,8 +824,6 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= -github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= -github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= @@ -912,8 +906,6 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzht golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 h1:phUcVbl53swtrUN8kQEXFhUxPlIlWyBfKmidCu7P95o= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= From f64c8822bc5b930d65ef32792df4d7b279143bce Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 9 Dec 2020 10:03:22 +0000 Subject: [PATCH 10/13] Federation sender refactor (#1621) * Refactor federation sender, again * Clean up better * Missing operators * Try to get overflowed events from database * Fix queries * Log less * Comments * nil PDUs/EDUs shouldn't happen but guard against them for safety * Tweak logging * Fix transaction coalescing * Update comments * Check nils more * Remove channels as they add extra complexity and possibly will deadlock * Don't hold lock while sending transaction * Less spam about sleeping queues * Comments * Bug-fixing * Don't try to rehydrate twice * Don't queue in memory for blacklisted destinations * Don't queue in memory for blacklisted destinations * Fix a couple of bugs * Check for duplicates when pulling things out of the database * Durable transactions, some more refactoring * Revert "Durable transactions, some more refactoring" This reverts commit 5daf924eaaefec5e4f7c12c16ca24e898de4adbb. * Fix deadlock --- federationsender/queue/destinationqueue.go | 356 ++++++++++-------- federationsender/queue/queue.go | 22 +- federationsender/storage/interface.go | 10 +- .../storage/postgres/queue_pdus_table.go | 37 +- federationsender/storage/shared/storage.go | 14 +- .../storage/shared/storage_edus.go | 47 ++- .../storage/shared/storage_pdus.go | 57 ++- .../storage/sqlite3/queue_pdus_table.go | 15 +- federationsender/storage/tables/interface.go | 3 +- 9 files changed, 294 insertions(+), 267 deletions(-) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 29fef7059..31eeaebc5 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -35,6 +35,8 @@ import ( const ( maxPDUsPerTransaction = 50 maxEDUsPerTransaction = 50 + maxPDUsInMemory = 128 + maxEDUsInMemory = 128 queueIdleTimeout = time.Second * 30 ) @@ -51,54 +53,56 @@ type destinationQueue struct { destination gomatrixserverlib.ServerName // destination of requests running atomic.Bool // is the queue worker running? backingOff atomic.Bool // true if we're backing off + overflowed atomic.Bool // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more statistics *statistics.ServerStatistics // statistics about this remote server transactionIDMutex sync.Mutex // protects transactionID - transactionID gomatrixserverlib.TransactionID // last transaction ID - transactionCount atomic.Int32 // how many events in this transaction so far - notifyPDUs chan bool // interrupts idle wait for PDUs - notifyEDUs chan bool // interrupts idle wait for EDUs + transactionID gomatrixserverlib.TransactionID // last transaction ID if retrying, or "" if last txn was successful + notify chan struct{} // interrupts idle wait pending PDUs/EDUs + pendingPDUs []*queuedPDU // PDUs waiting to be sent + pendingEDUs []*queuedEDU // EDUs waiting to be sent + pendingMutex sync.RWMutex // protects pendingPDUs and pendingEDUs interruptBackoff chan bool // interrupts backoff } // Send event adds the event to the pending queue for the destination. // If the queue is empty then it starts a background goroutine to // start sending events to that destination. -func (oq *destinationQueue) sendEvent(receipt *shared.Receipt) { - // Create a transaction ID. We'll either do this if we don't have - // one made up yet, or if we've exceeded the number of maximum - // events allowed in a single tranaction. We'll reset the counter - // when we do. - oq.transactionIDMutex.Lock() - if oq.transactionID == "" || oq.transactionCount.Load() >= maxPDUsPerTransaction { - now := gomatrixserverlib.AsTimestamp(time.Now()) - oq.transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount())) - oq.transactionCount.Store(0) +func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, receipt *shared.Receipt) { + if event == nil { + log.Errorf("attempt to send nil PDU with destination %q", oq.destination) + return } - oq.transactionIDMutex.Unlock() // Create a database entry that associates the given PDU NID with // this destination queue. We'll then be able to retrieve the PDU // later. if err := oq.db.AssociatePDUWithDestination( context.TODO(), - oq.transactionID, // the current transaction ID - oq.destination, // the destination server name - receipt, // NIDs from federationsender_queue_json table + "", // TODO: remove this, as we don't need to persist the transaction ID + oq.destination, // the destination server name + receipt, // NIDs from federationsender_queue_json table ); err != nil { - log.WithError(err).Errorf("failed to associate PDU receipt %q with destination %q", receipt.String(), oq.destination) + log.WithError(err).Errorf("failed to associate PDU %q with destination %q", event.EventID(), oq.destination) return } - // We've successfully added a PDU to the transaction so increase - // the counter. - oq.transactionCount.Add(1) // Check if the destination is blacklisted. If it isn't then wake // up the queue. if !oq.statistics.Blacklisted() { + // If there's room in memory to hold the event then add it to the + // list. + oq.pendingMutex.Lock() + if len(oq.pendingPDUs) < maxPDUsInMemory { + oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{ + pdu: event, + receipt: receipt, + }) + } else { + oq.overflowed.Store(true) + } + oq.pendingMutex.Unlock() // Wake up the queue if it's asleep. oq.wakeQueueIfNeeded() - // If we're blocking on waiting PDUs then tell the queue that we - // have work to do. select { - case oq.notifyPDUs <- true: + case oq.notify <- struct{}{}: default: } } @@ -107,7 +111,11 @@ func (oq *destinationQueue) sendEvent(receipt *shared.Receipt) { // sendEDU adds the EDU event to the pending queue for the destination. // If the queue is empty then it starts a background goroutine to // start sending events to that destination. -func (oq *destinationQueue) sendEDU(receipt *shared.Receipt) { +func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *shared.Receipt) { + if event == nil { + log.Errorf("attempt to send nil EDU with destination %q", oq.destination) + return + } // Create a database entry that associates the given PDU NID with // this destination queue. We'll then be able to retrieve the PDU // later. @@ -116,21 +124,28 @@ func (oq *destinationQueue) sendEDU(receipt *shared.Receipt) { oq.destination, // the destination server name receipt, // NIDs from federationsender_queue_json table ); err != nil { - log.WithError(err).Errorf("failed to associate EDU receipt %q with destination %q", receipt.String(), oq.destination) + log.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination) return } - // We've successfully added an EDU to the transaction so increase - // the counter. - oq.transactionCount.Add(1) // Check if the destination is blacklisted. If it isn't then wake // up the queue. if !oq.statistics.Blacklisted() { + // If there's room in memory to hold the event then add it to the + // list. + oq.pendingMutex.Lock() + if len(oq.pendingEDUs) < maxEDUsInMemory { + oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{ + edu: event, + receipt: receipt, + }) + } else { + oq.overflowed.Store(true) + } + oq.pendingMutex.Unlock() // Wake up the queue if it's asleep. oq.wakeQueueIfNeeded() - // If we're blocking on waiting EDUs then tell the queue that we - // have work to do. select { - case oq.notifyEDUs <- true: + case oq.notify <- struct{}{}: default: } } @@ -152,48 +167,71 @@ func (oq *destinationQueue) wakeQueueIfNeeded() { } } -// waitForPDUs returns a channel for pending PDUs, which will be -// used in backgroundSend select. It returns a closed channel if -// there is something pending right now, or an open channel if -// we're waiting for something. -func (oq *destinationQueue) waitForPDUs() chan bool { - pendingPDUs, err := oq.db.GetPendingPDUCount(context.TODO(), oq.destination) - if err != nil { - log.WithError(err).Errorf("Failed to get pending PDU count on queue %q", oq.destination) - } - // If there are PDUs pending right now then we'll return a closed - // channel. This will mean that the backgroundSend will not block. - if pendingPDUs > 0 { - ch := make(chan bool, 1) - close(ch) - return ch - } - // If there are no PDUs pending right now then instead we'll return - // the notify channel, so that backgroundSend can pick up normal - // notifications from sendEvent. - return oq.notifyPDUs -} +// getPendingFromDatabase will look at the database and see if +// there are any persisted events that haven't been sent to this +// destination yet. If so, they will be queued up. +// nolint:gocyclo +func (oq *destinationQueue) getPendingFromDatabase() { + // Check to see if there's anything to do for this server + // in the database. + retrieved := false + ctx := context.Background() + oq.pendingMutex.Lock() + defer oq.pendingMutex.Unlock() -// waitForEDUs returns a channel for pending EDUs, which will be -// used in backgroundSend select. It returns a closed channel if -// there is something pending right now, or an open channel if -// we're waiting for something. -func (oq *destinationQueue) waitForEDUs() chan bool { - pendingEDUs, err := oq.db.GetPendingEDUCount(context.TODO(), oq.destination) - if err != nil { - log.WithError(err).Errorf("Failed to get pending EDU count on queue %q", oq.destination) + // Take a note of all of the PDUs and EDUs that we already + // have cached. We will index them based on the receipt, + // which ultimately just contains the index of the PDU/EDU + // in the database. + gotPDUs := map[string]struct{}{} + gotEDUs := map[string]struct{}{} + for _, pdu := range oq.pendingPDUs { + gotPDUs[pdu.receipt.String()] = struct{}{} } - // If there are EDUs pending right now then we'll return a closed - // channel. This will mean that the backgroundSend will not block. - if pendingEDUs > 0 { - ch := make(chan bool, 1) - close(ch) - return ch + for _, edu := range oq.pendingEDUs { + gotEDUs[edu.receipt.String()] = struct{}{} + } + + if pduCapacity := maxPDUsInMemory - len(oq.pendingPDUs); pduCapacity > 0 { + // We have room in memory for some PDUs - let's request no more than that. + if pdus, err := oq.db.GetPendingPDUs(ctx, oq.destination, pduCapacity); err == nil { + for receipt, pdu := range pdus { + if _, ok := gotPDUs[receipt.String()]; ok { + continue + } + oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{receipt, pdu}) + retrieved = true + } + } else { + logrus.WithError(err).Errorf("Failed to get pending PDUs for %q", oq.destination) + } + } + if eduCapacity := maxEDUsInMemory - len(oq.pendingEDUs); eduCapacity > 0 { + // We have room in memory for some EDUs - let's request no more than that. + if edus, err := oq.db.GetPendingEDUs(ctx, oq.destination, eduCapacity); err == nil { + for receipt, edu := range edus { + if _, ok := gotEDUs[receipt.String()]; ok { + continue + } + oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{receipt, edu}) + retrieved = true + } + } else { + logrus.WithError(err).Errorf("Failed to get pending EDUs for %q", oq.destination) + } + } + // If we've retrieved all of the events from the database with room to spare + // in memory then we'll no longer consider this queue to be overflowed. + if len(oq.pendingPDUs) < maxPDUsInMemory && len(oq.pendingEDUs) < maxEDUsInMemory { + oq.overflowed.Store(false) + } + // If we've retrieved some events then notify the destination queue goroutine. + if retrieved { + select { + case oq.notify <- struct{}{}: + default: + } } - // If there are no EDUs pending right now then instead we'll return - // the notify channel, so that backgroundSend can pick up normal - // notifications from sendEvent. - return oq.notifyEDUs } // backgroundSend is the worker goroutine for sending events. @@ -206,25 +244,28 @@ func (oq *destinationQueue) backgroundSend() { } defer oq.running.Store(false) + // Mark the queue as overflowed, so we will consult the database + // to see if there's anything new to send. + oq.overflowed.Store(true) + for { - pendingPDUs, pendingEDUs := false, false + // If we are overflowing memory and have sent things out to the + // database then we can look up what those things are. + if oq.overflowed.Load() { + oq.getPendingFromDatabase() + } // If we have nothing to do then wait either for incoming events, or // until we hit an idle timeout. select { - case <-oq.waitForPDUs(): - // We were woken up because there are new PDUs waiting in the - // database. - pendingPDUs = true - case <-oq.waitForEDUs(): - // We were woken up because there are new PDUs waiting in the - // database. - pendingEDUs = true + case <-oq.notify: + // There's work to do, either because getPendingFromDatabase + // told us there is, or because a new event has come in via + // sendEvent/sendEDU. case <-time.After(queueIdleTimeout): // The worker is idle so stop the goroutine. It'll get // restarted automatically the next time we have an event to // send. - log.Tracef("Queue %q has been idle for %s, going to sleep", oq.destination, queueIdleTimeout) return } @@ -237,6 +278,16 @@ func (oq *destinationQueue) backgroundSend() { // has exceeded a maximum allowable value. Clean up the in-memory // buffers at this point. The PDU clean-up is already on a defer. log.Warnf("Blacklisting %q due to exceeding backoff threshold", oq.destination) + oq.pendingMutex.Lock() + for i := range oq.pendingPDUs { + oq.pendingPDUs[i] = nil + } + for i := range oq.pendingEDUs { + oq.pendingEDUs[i] = nil + } + oq.pendingPDUs = nil + oq.pendingEDUs = nil + oq.pendingMutex.Unlock() return } if until != nil && until.After(time.Now()) { @@ -250,18 +301,41 @@ func (oq *destinationQueue) backgroundSend() { } } + // Work out which PDUs/EDUs to include in the next transaction. + oq.pendingMutex.RLock() + pduCount := len(oq.pendingPDUs) + eduCount := len(oq.pendingEDUs) + if pduCount > maxPDUsPerTransaction { + pduCount = maxPDUsPerTransaction + } + if eduCount > maxEDUsPerTransaction { + eduCount = maxEDUsPerTransaction + } + toSendPDUs := oq.pendingPDUs[:pduCount] + toSendEDUs := oq.pendingEDUs[:eduCount] + oq.pendingMutex.RUnlock() + // If we have pending PDUs or EDUs then construct a transaction. - if pendingPDUs || pendingEDUs { - // Try sending the next transaction and see what happens. - transaction, terr := oq.nextTransaction() - if terr != nil { - // We failed to send the transaction. Mark it as a failure. - oq.statistics.Failure() - } else if transaction { - // If we successfully sent the transaction then clear out - // the pending events and EDUs, and wipe our transaction ID. - oq.statistics.Success() + // Try sending the next transaction and see what happens. + transaction, pc, ec, terr := oq.nextTransaction(toSendPDUs, toSendEDUs) + if terr != nil { + // We failed to send the transaction. Mark it as a failure. + oq.statistics.Failure() + + } else if transaction { + // If we successfully sent the transaction then clear out + // the pending events and EDUs, and wipe our transaction ID. + oq.statistics.Success() + oq.pendingMutex.Lock() + for i := range oq.pendingPDUs[:pc] { + oq.pendingPDUs[i] = nil } + for i := range oq.pendingEDUs[:ec] { + oq.pendingEDUs[i] = nil + } + oq.pendingPDUs = oq.pendingPDUs[pc:] + oq.pendingEDUs = oq.pendingEDUs[ec:] + oq.pendingMutex.Unlock() } } } @@ -270,16 +344,20 @@ func (oq *destinationQueue) backgroundSend() { // queue and sends it. Returns true if a transaction was sent or // false otherwise. // nolint:gocyclo -func (oq *destinationQueue) nextTransaction() (bool, error) { - // Before we do anything, we need to roll over the transaction - // ID that is being used to coalesce events into the next TX. - // Otherwise it's possible that we'll pick up an incomplete - // transaction and end up nuking the rest of the events at the - // cleanup stage. +func (oq *destinationQueue) nextTransaction( + pdus []*queuedPDU, + edus []*queuedEDU, +) (bool, int, int, error) { + // If there's no projected transaction ID then generate one. If + // the transaction succeeds then we'll set it back to "" so that + // we generate a new one next time. If it fails, we'll preserve + // it so that we retry with the same transaction ID. oq.transactionIDMutex.Lock() - oq.transactionID = "" + if oq.transactionID == "" { + now := gomatrixserverlib.AsTimestamp(time.Now()) + oq.transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount())) + } oq.transactionIDMutex.Unlock() - oq.transactionCount.Store(0) // Create the transaction. t := gomatrixserverlib.Transaction{ @@ -289,58 +367,36 @@ func (oq *destinationQueue) nextTransaction() (bool, error) { t.Origin = oq.origin t.Destination = oq.destination t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now()) - - // Ask the database for any pending PDUs from the next transaction. - // maxPDUsPerTransaction is an upper limit but we probably won't - // actually retrieve that many events. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - txid, pdus, pduReceipt, err := oq.db.GetNextTransactionPDUs( - ctx, // context - oq.destination, // server name - maxPDUsPerTransaction, // max events to retrieve - ) - if err != nil { - log.WithError(err).Errorf("failed to get next transaction PDUs for server %q", oq.destination) - return false, fmt.Errorf("oq.db.GetNextTransactionPDUs: %w", err) - } - - edus, eduReceipt, err := oq.db.GetNextTransactionEDUs( - ctx, // context - oq.destination, // server name - maxEDUsPerTransaction, // max events to retrieve - ) - if err != nil { - log.WithError(err).Errorf("failed to get next transaction EDUs for server %q", oq.destination) - return false, fmt.Errorf("oq.db.GetNextTransactionEDUs: %w", err) - } + t.TransactionID = oq.transactionID // If we didn't get anything from the database and there are no // pending EDUs then there's nothing to do - stop here. if len(pdus) == 0 && len(edus) == 0 { - return false, nil + return false, 0, 0, nil } - // Pick out the transaction ID from the database. If we didn't - // get a transaction ID (i.e. because there are no PDUs but only - // EDUs) then generate a transaction ID. - t.TransactionID = txid - if t.TransactionID == "" { - now := gomatrixserverlib.AsTimestamp(time.Now()) - t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount())) - } + var pduReceipts []*shared.Receipt + var eduReceipts []*shared.Receipt // Go through PDUs that we retrieved from the database, if any, // and add them into the transaction. for _, pdu := range pdus { + if pdu == nil || pdu.pdu == nil { + continue + } // Append the JSON of the event, since this is a json.RawMessage type in the // gomatrixserverlib.Transaction struct - t.PDUs = append(t.PDUs, (*pdu).JSON()) + t.PDUs = append(t.PDUs, pdu.pdu.JSON()) + pduReceipts = append(pduReceipts, pdu.receipt) } // Do the same for pending EDUS in the queue. for _, edu := range edus { - t.EDUs = append(t.EDUs, *edu) + if edu == nil || edu.edu == nil { + continue + } + t.EDUs = append(t.EDUs, *edu.edu) + eduReceipts = append(eduReceipts, edu.receipt) } logrus.WithField("server_name", oq.destination).Debugf("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) @@ -349,34 +405,38 @@ func (oq *destinationQueue) nextTransaction() (bool, error) { // TODO: we should check for 500-ish fails vs 400-ish here, // since we shouldn't queue things indefinitely in response // to a 400-ish error - ctx, cancel = context.WithTimeout(context.Background(), time.Minute*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() - _, err = oq.client.SendTransaction(ctx, t) + _, err := oq.client.SendTransaction(ctx, t) switch err.(type) { case nil: // Clean up the transaction in the database. - if pduReceipt != nil { + if pduReceipts != nil { //logrus.Infof("Cleaning PDUs %q", pduReceipt.String()) - if err = oq.db.CleanPDUs(context.Background(), oq.destination, pduReceipt); err != nil { - log.WithError(err).Errorf("failed to clean PDUs %q for server %q", pduReceipt.String(), t.Destination) + if err = oq.db.CleanPDUs(context.Background(), oq.destination, pduReceipts); err != nil { + log.WithError(err).Errorf("Failed to clean PDUs for server %q", t.Destination) } } - if eduReceipt != nil { + if eduReceipts != nil { //logrus.Infof("Cleaning EDUs %q", eduReceipt.String()) - if err = oq.db.CleanEDUs(context.Background(), oq.destination, eduReceipt); err != nil { - log.WithError(err).Errorf("failed to clean EDUs %q for server %q", eduReceipt.String(), t.Destination) + if err = oq.db.CleanEDUs(context.Background(), oq.destination, eduReceipts); err != nil { + log.WithError(err).Errorf("Failed to clean EDUs for server %q", t.Destination) } } - return true, nil + // Reset the transaction ID. + oq.transactionIDMutex.Lock() + oq.transactionID = "" + oq.transactionIDMutex.Unlock() + return true, len(t.PDUs), len(t.EDUs), nil case gomatrix.HTTPError: // Report that we failed to send the transaction and we // will retry again, subject to backoff. - return false, err + return false, 0, 0, err default: log.WithFields(log.Fields{ "destination": oq.destination, log.ErrorKey: err, - }).Info("problem sending transaction") - return false, err + }).Infof("Failed to send transaction %q", t.TransactionID) + return false, 0, 0, err } } diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index d45af8686..da30e4de1 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/statistics" "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/federationsender/storage/shared" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" @@ -83,8 +84,8 @@ func NewOutgoingQueues( log.WithError(err).Error("Failed to get EDU server names for destination queue hydration") } for serverName := range serverNames { - if !queues.getQueue(serverName).statistics.Blacklisted() { - queues.getQueue(serverName).wakeQueueIfNeeded() + if queue := queues.getQueue(serverName); !queue.statistics.Blacklisted() { + queue.wakeQueueIfNeeded() } } }) @@ -100,6 +101,16 @@ type SigningInfo struct { PrivateKey ed25519.PrivateKey } +type queuedPDU struct { + receipt *shared.Receipt + pdu *gomatrixserverlib.HeaderedEvent +} + +type queuedEDU struct { + receipt *shared.Receipt + edu *gomatrixserverlib.EDU +} + func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue { oqs.queuesMutex.Lock() defer oqs.queuesMutex.Unlock() @@ -112,8 +123,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d destination: destination, client: oqs.client, statistics: oqs.statistics.ForServer(destination), - notifyPDUs: make(chan bool, 1), - notifyEDUs: make(chan bool, 1), + notify: make(chan struct{}, 1), interruptBackoff: make(chan bool), signing: oqs.signing, } @@ -188,7 +198,7 @@ func (oqs *OutgoingQueues) SendEvent( } for destination := range destmap { - oqs.getQueue(destination).sendEvent(nid) + oqs.getQueue(destination).sendEvent(ev, nid) } return nil @@ -258,7 +268,7 @@ func (oqs *OutgoingQueues) SendEDU( } for destination := range destmap { - oqs.getQueue(destination).sendEDU(nid) + oqs.getQueue(destination).sendEDU(e, nid) } return nil diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go index a3f5073f9..03d616f1b 100644 --- a/federationsender/storage/interface.go +++ b/federationsender/storage/interface.go @@ -36,14 +36,14 @@ type Database interface { StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) + GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) + GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) + AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error - GetNextTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (gomatrixserverlib.TransactionID, []*gomatrixserverlib.HeaderedEvent, *shared.Receipt, error) - GetNextTransactionEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) ([]*gomatrixserverlib.EDU, *shared.Receipt, error) - - CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error - CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error + CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error + CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) diff --git a/federationsender/storage/postgres/queue_pdus_table.go b/federationsender/storage/postgres/queue_pdus_table.go index 95a3b9eee..f9a477483 100644 --- a/federationsender/storage/postgres/queue_pdus_table.go +++ b/federationsender/storage/postgres/queue_pdus_table.go @@ -45,16 +45,10 @@ const insertQueuePDUSQL = "" + const deleteQueuePDUSQL = "" + "DELETE FROM federationsender_queue_pdus WHERE server_name = $1 AND json_nid = ANY($2)" -const selectQueuePDUNextTransactionIDSQL = "" + - "SELECT transaction_id FROM federationsender_queue_pdus" + - " WHERE server_name = $1" + - " ORDER BY transaction_id ASC" + - " LIMIT 1" - -const selectQueuePDUsByTransactionSQL = "" + +const selectQueuePDUsSQL = "" + "SELECT json_nid FROM federationsender_queue_pdus" + - " WHERE server_name = $1 AND transaction_id = $2" + - " LIMIT $3" + " WHERE server_name = $1" + + " LIMIT $2" const selectQueuePDUReferenceJSONCountSQL = "" + "SELECT COUNT(*) FROM federationsender_queue_pdus" + @@ -71,8 +65,7 @@ type queuePDUsStatements struct { db *sql.DB insertQueuePDUStmt *sql.Stmt deleteQueuePDUsStmt *sql.Stmt - selectQueuePDUNextTransactionIDStmt *sql.Stmt - selectQueuePDUsByTransactionStmt *sql.Stmt + selectQueuePDUsStmt *sql.Stmt selectQueuePDUReferenceJSONCountStmt *sql.Stmt selectQueuePDUsCountStmt *sql.Stmt selectQueuePDUServerNamesStmt *sql.Stmt @@ -92,10 +85,7 @@ func NewPostgresQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) { if s.deleteQueuePDUsStmt, err = s.db.Prepare(deleteQueuePDUSQL); err != nil { return } - if s.selectQueuePDUNextTransactionIDStmt, err = s.db.Prepare(selectQueuePDUNextTransactionIDSQL); err != nil { - return - } - if s.selectQueuePDUsByTransactionStmt, err = s.db.Prepare(selectQueuePDUsByTransactionSQL); err != nil { + if s.selectQueuePDUsStmt, err = s.db.Prepare(selectQueuePDUsSQL); err != nil { return } if s.selectQueuePDUReferenceJSONCountStmt, err = s.db.Prepare(selectQueuePDUReferenceJSONCountSQL); err != nil { @@ -137,18 +127,6 @@ func (s *queuePDUsStatements) DeleteQueuePDUs( return err } -func (s *queuePDUsStatements) SelectQueuePDUNextTransactionID( - ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, -) (gomatrixserverlib.TransactionID, error) { - var transactionID gomatrixserverlib.TransactionID - stmt := sqlutil.TxStmt(txn, s.selectQueuePDUNextTransactionIDStmt) - err := stmt.QueryRowContext(ctx, serverName).Scan(&transactionID) - if err == sql.ErrNoRows { - return "", nil - } - return transactionID, err -} - func (s *queuePDUsStatements) SelectQueuePDUReferenceJSONCount( ctx context.Context, txn *sql.Tx, jsonNID int64, ) (int64, error) { @@ -182,11 +160,10 @@ func (s *queuePDUsStatements) SelectQueuePDUCount( func (s *queuePDUsStatements) SelectQueuePDUs( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, - transactionID gomatrixserverlib.TransactionID, limit int, ) ([]int64, error) { - stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt) - rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit) + stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsStmt) + rows, err := stmt.QueryContext(ctx, serverName, limit) if err != nil { return nil, err } diff --git a/federationsender/storage/shared/storage.go b/federationsender/storage/shared/storage.go index af9d0d6a3..fbf84c705 100644 --- a/federationsender/storage/shared/storage.go +++ b/federationsender/storage/shared/storage.go @@ -17,7 +17,6 @@ package shared import ( "context" "database/sql" - "encoding/json" "fmt" "github.com/matrix-org/dendrite/federationsender/storage/tables" @@ -44,16 +43,11 @@ type Database struct { // to pass them back so that we can clean up if the transaction sends // successfully. type Receipt struct { - nids []int64 + nid int64 } -func (e *Receipt) Empty() bool { - return len(e.nids) == 0 -} - -func (e *Receipt) String() string { - j, _ := json.Marshal(e.nids) - return string(j) +func (r *Receipt) String() string { + return fmt.Sprintf("%d", r.nid) } // UpdateRoom updates the joined hosts for a room and returns what the joined @@ -146,7 +140,7 @@ func (d *Database) StoreJSON( return nil, fmt.Errorf("d.insertQueueJSON: %w", err) } return &Receipt{ - nids: []int64{nid}, + nid: nid, }, nil } diff --git a/federationsender/storage/shared/storage_edus.go b/federationsender/storage/shared/storage_edus.go index ae1d15118..86fee1a37 100644 --- a/federationsender/storage/shared/storage_edus.go +++ b/federationsender/storage/shared/storage_edus.go @@ -33,16 +33,14 @@ func (d *Database) AssociateEDUWithDestination( receipt *Receipt, ) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - for _, nid := range receipt.nids { - if err := d.FederationSenderQueueEDUs.InsertQueueEDU( - ctx, // context - txn, // SQL transaction - "", // TODO: EDU type for coalescing - serverName, // destination server name - nid, // NID from the federationsender_queue_json table - ); err != nil { - return fmt.Errorf("InsertQueueEDU: %w", err) - } + if err := d.FederationSenderQueueEDUs.InsertQueueEDU( + ctx, // context + txn, // SQL transaction + "", // TODO: EDU type for coalescing + serverName, // destination server name + receipt.nid, // NID from the federationsender_queue_json table + ); err != nil { + return fmt.Errorf("InsertQueueEDU: %w", err) } return nil }) @@ -50,29 +48,25 @@ func (d *Database) AssociateEDUWithDestination( // GetNextTransactionEDUs retrieves events from the database for // the next pending transaction, up to the limit specified. -func (d *Database) GetNextTransactionEDUs( +func (d *Database) GetPendingEDUs( ctx context.Context, serverName gomatrixserverlib.ServerName, limit int, ) ( - edus []*gomatrixserverlib.EDU, - receipt *Receipt, + edus map[*Receipt]*gomatrixserverlib.EDU, err error, ) { + edus = make(map[*Receipt]*gomatrixserverlib.EDU) err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { nids, err := d.FederationSenderQueueEDUs.SelectQueueEDUs(ctx, txn, serverName, limit) if err != nil { return fmt.Errorf("SelectQueueEDUs: %w", err) } - receipt = &Receipt{ - nids: nids, - } - retrieve := make([]int64, 0, len(nids)) for _, nid := range nids { if edu, ok := d.Cache.GetFederationSenderQueuedEDU(nid); ok { - edus = append(edus, edu) + edus[&Receipt{nid}] = edu } else { retrieve = append(retrieve, nid) } @@ -83,12 +77,12 @@ func (d *Database) GetNextTransactionEDUs( return fmt.Errorf("SelectQueueJSON: %w", err) } - for _, blob := range blobs { + for nid, blob := range blobs { var event gomatrixserverlib.EDU if err := json.Unmarshal(blob, &event); err != nil { return fmt.Errorf("json.Unmarshal: %w", err) } - edus = append(edus, &event) + edus[&Receipt{nid}] = &event } return nil @@ -101,19 +95,24 @@ func (d *Database) GetNextTransactionEDUs( func (d *Database) CleanEDUs( ctx context.Context, serverName gomatrixserverlib.ServerName, - receipt *Receipt, + receipts []*Receipt, ) error { - if receipt == nil { + if len(receipts) == 0 { return errors.New("expected receipt") } + nids := make([]int64, len(receipts)) + for i := range receipts { + nids[i] = receipts[i].nid + } + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationSenderQueueEDUs.DeleteQueueEDUs(ctx, txn, serverName, receipt.nids); err != nil { + if err := d.FederationSenderQueueEDUs.DeleteQueueEDUs(ctx, txn, serverName, nids); err != nil { return err } var deleteNIDs []int64 - for _, nid := range receipt.nids { + for _, nid := range nids { count, err := d.FederationSenderQueueEDUs.SelectQueueEDUReferenceJSONCount(ctx, txn, nid) if err != nil { return fmt.Errorf("SelectQueueEDUReferenceJSONCount: %w", err) diff --git a/federationsender/storage/shared/storage_pdus.go b/federationsender/storage/shared/storage_pdus.go index 09235a5ec..bc298a905 100644 --- a/federationsender/storage/shared/storage_pdus.go +++ b/federationsender/storage/shared/storage_pdus.go @@ -34,16 +34,14 @@ func (d *Database) AssociatePDUWithDestination( receipt *Receipt, ) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - for _, nid := range receipt.nids { - if err := d.FederationSenderQueuePDUs.InsertQueuePDU( - ctx, // context - txn, // SQL transaction - transactionID, // transaction ID - serverName, // destination server name - nid, // NID from the federationsender_queue_json table - ); err != nil { - return fmt.Errorf("InsertQueuePDU: %w", err) - } + if err := d.FederationSenderQueuePDUs.InsertQueuePDU( + ctx, // context + txn, // SQL transaction + transactionID, // transaction ID + serverName, // destination server name + receipt.nid, // NID from the federationsender_queue_json table + ); err != nil { + return fmt.Errorf("InsertQueuePDU: %w", err) } return nil }) @@ -51,14 +49,12 @@ func (d *Database) AssociatePDUWithDestination( // GetNextTransactionPDUs retrieves events from the database for // the next pending transaction, up to the limit specified. -func (d *Database) GetNextTransactionPDUs( +func (d *Database) GetPendingPDUs( ctx context.Context, serverName gomatrixserverlib.ServerName, limit int, ) ( - transactionID gomatrixserverlib.TransactionID, - events []*gomatrixserverlib.HeaderedEvent, - receipt *Receipt, + events map[*Receipt]*gomatrixserverlib.HeaderedEvent, err error, ) { // Strictly speaking this doesn't need to be using the writer @@ -66,29 +62,17 @@ func (d *Database) GetNextTransactionPDUs( // a guarantee of transactional isolation, it's actually useful // to know in SQLite mode that nothing else is trying to modify // the database. + events = make(map[*Receipt]*gomatrixserverlib.HeaderedEvent) err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - transactionID, err = d.FederationSenderQueuePDUs.SelectQueuePDUNextTransactionID(ctx, txn, serverName) - if err != nil { - return fmt.Errorf("SelectQueuePDUNextTransactionID: %w", err) - } - - if transactionID == "" { - return nil - } - - nids, err := d.FederationSenderQueuePDUs.SelectQueuePDUs(ctx, txn, serverName, transactionID, limit) + nids, err := d.FederationSenderQueuePDUs.SelectQueuePDUs(ctx, txn, serverName, limit) if err != nil { return fmt.Errorf("SelectQueuePDUs: %w", err) } - receipt = &Receipt{ - nids: nids, - } - retrieve := make([]int64, 0, len(nids)) for _, nid := range nids { if event, ok := d.Cache.GetFederationSenderQueuedPDU(nid); ok { - events = append(events, event) + events[&Receipt{nid}] = event } else { retrieve = append(retrieve, nid) } @@ -104,7 +88,7 @@ func (d *Database) GetNextTransactionPDUs( if err := json.Unmarshal(blob, &event); err != nil { return fmt.Errorf("json.Unmarshal: %w", err) } - events = append(events, &event) + events[&Receipt{nid}] = &event d.Cache.StoreFederationSenderQueuedPDU(nid, &event) } @@ -119,19 +103,24 @@ func (d *Database) GetNextTransactionPDUs( func (d *Database) CleanPDUs( ctx context.Context, serverName gomatrixserverlib.ServerName, - receipt *Receipt, + receipts []*Receipt, ) error { - if receipt == nil { + if len(receipts) == 0 { return errors.New("expected receipt") } + nids := make([]int64, len(receipts)) + for i := range receipts { + nids[i] = receipts[i].nid + } + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationSenderQueuePDUs.DeleteQueuePDUs(ctx, txn, serverName, receipt.nids); err != nil { + if err := d.FederationSenderQueuePDUs.DeleteQueuePDUs(ctx, txn, serverName, nids); err != nil { return err } var deleteNIDs []int64 - for _, nid := range receipt.nids { + for _, nid := range nids { count, err := d.FederationSenderQueuePDUs.SelectQueuePDUReferenceJSONCount(ctx, txn, nid) if err != nil { return fmt.Errorf("SelectQueuePDUReferenceJSONCount: %w", err) diff --git a/federationsender/storage/sqlite3/queue_pdus_table.go b/federationsender/storage/sqlite3/queue_pdus_table.go index 70519c9ef..e0fdbda5f 100644 --- a/federationsender/storage/sqlite3/queue_pdus_table.go +++ b/federationsender/storage/sqlite3/queue_pdus_table.go @@ -53,10 +53,10 @@ const selectQueueNextTransactionIDSQL = "" + " ORDER BY transaction_id ASC" + " LIMIT 1" -const selectQueuePDUsByTransactionSQL = "" + +const selectQueuePDUsSQL = "" + "SELECT json_nid FROM federationsender_queue_pdus" + - " WHERE server_name = $1 AND transaction_id = $2" + - " LIMIT $3" + " WHERE server_name = $1" + + " LIMIT $2" const selectQueuePDUsReferenceJSONCountSQL = "" + "SELECT COUNT(*) FROM federationsender_queue_pdus" + @@ -73,7 +73,7 @@ type queuePDUsStatements struct { db *sql.DB insertQueuePDUStmt *sql.Stmt selectQueueNextTransactionIDStmt *sql.Stmt - selectQueuePDUsByTransactionStmt *sql.Stmt + selectQueuePDUsStmt *sql.Stmt selectQueueReferenceJSONCountStmt *sql.Stmt selectQueuePDUsCountStmt *sql.Stmt selectQueueServerNamesStmt *sql.Stmt @@ -97,7 +97,7 @@ func NewSQLiteQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) { if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil { return } - if s.selectQueuePDUsByTransactionStmt, err = db.Prepare(selectQueuePDUsByTransactionSQL); err != nil { + if s.selectQueuePDUsStmt, err = db.Prepare(selectQueuePDUsSQL); err != nil { return } if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueuePDUsReferenceJSONCountSQL); err != nil { @@ -193,11 +193,10 @@ func (s *queuePDUsStatements) SelectQueuePDUCount( func (s *queuePDUsStatements) SelectQueuePDUs( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, - transactionID gomatrixserverlib.TransactionID, limit int, ) ([]int64, error) { - stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt) - rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit) + stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsStmt) + rows, err := stmt.QueryContext(ctx, serverName, limit) if err != nil { return nil, err } diff --git a/federationsender/storage/tables/interface.go b/federationsender/storage/tables/interface.go index 1167a212a..69e952de2 100644 --- a/federationsender/storage/tables/interface.go +++ b/federationsender/storage/tables/interface.go @@ -25,10 +25,9 @@ import ( type FederationSenderQueuePDUs interface { InsertQueuePDU(ctx context.Context, txn *sql.Tx, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nid int64) error DeleteQueuePDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, jsonNIDs []int64) error - SelectQueuePDUNextTransactionID(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (gomatrixserverlib.TransactionID, error) SelectQueuePDUReferenceJSONCount(ctx context.Context, txn *sql.Tx, jsonNID int64) (int64, error) SelectQueuePDUCount(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (int64, error) - SelectQueuePDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, transactionID gomatrixserverlib.TransactionID, limit int) ([]int64, error) + SelectQueuePDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, limit int) ([]int64, error) SelectQueuePDUServerNames(ctx context.Context, txn *sql.Tx) ([]gomatrixserverlib.ServerName, error) } From d9b3035342fe3b432bfbdf46d9f5a411d9cd9d39 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 9 Dec 2020 13:34:37 +0000 Subject: [PATCH 11/13] Adjust latest events updater (#1623) * Adjust forward elatest events updater * Populate newLatest in all cases * Re-add existingPrevs loop --- roomserver/internal/input/input_latest_events.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 3608ef4b0..9554bf330 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -285,15 +285,15 @@ func (u *latestEventsUpdater) calculateLatest( // then do nothing - it's not a candidate to be a new extremity if // it has been referenced. if _, ok := existingPrevs[newEvent.EventID()]; ok { + u.latest = oldLatest return false, nil } // If the "new" event is already a forward extremity then stop, as // nothing changes. - for _, event := range events { - if event.EventID() == newEvent.EventID() { - return false, nil - } + if _, ok := existingRefs[newEvent.EventID()]; ok { + u.latest = oldLatest + return false, nil } // Include our new event in the extremities. From 851c02659a5c15896aa8d292e38e82c2169fa80a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 9 Dec 2020 14:13:02 +0000 Subject: [PATCH 12/13] Version 0.3.3 --- CHANGES.md | 15 +++++++++++++++ dendrite-config.yaml | 20 ++++++++++---------- internal/version.go | 2 +- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f463f29f1..fbdd3b29d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,20 @@ # Changelog +## Dendrite 0.3.3 (2020-12-09) + +### Features + +* Federation sender should now use considerably less CPU cycles and RAM when sending events into large rooms +* The roomserver now uses considerably less CPU cycles by not calculating event IDs so often +* Experimental support for [MSC2836](https://github.com/matrix-org/matrix-doc/pull/2836) (threading) has been merged +* Dendrite will no longer hold federation HTTP connections open unnecessarily, which should help to reduce ambient CPU/RAM usage and hold fewer long-term file descriptors + +### Fixes + +* A bug in the latest event updater has been fixed, which should prevent the roomserver from losing forward extremities in some rare cases +* A panic has been fixed when federation is disabled (contributed by [kraem](https://github.com/kraem)) +* The response format of the `/joined_members` endpoint has been fixed (contributed by [alexkursell](https://github.com/alexkursell)) + ## Dendrite 0.3.2 (2020-12-02) ### Features diff --git a/dendrite-config.yaml b/dendrite-config.yaml index ccdb32432..a6bf63afd 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -89,7 +89,7 @@ global: # Naffka database options. Not required when using Kafka. naffka_database: connection_string: file:naffka.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -110,7 +110,7 @@ app_service_api: connect: http://localhost:7777 database: connection_string: file:appservice.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -185,7 +185,7 @@ federation_sender: connect: http://localhost:7775 database: connection_string: file:federationsender.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -211,7 +211,7 @@ key_server: connect: http://localhost:7779 database: connection_string: file:keyserver.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -224,7 +224,7 @@ media_api: listen: http://[::]:8074 database: connection_string: file:mediaapi.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -260,7 +260,7 @@ room_server: connect: http://localhost:7770 database: connection_string: file:roomserver.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -271,7 +271,7 @@ signing_key_server: connect: http://localhost:7780 database: connection_string: file:signingkeyserver.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -300,7 +300,7 @@ sync_api: listen: http://[::]:8073 database: connection_string: file:syncapi.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 @@ -316,12 +316,12 @@ user_api: connect: http://localhost:7781 account_database: connection_string: file:userapi_accounts.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 device_database: connection_string: file:userapi_devices.db - max_open_conns: 100 + max_open_conns: 10 max_idle_conns: 2 conn_max_lifetime: -1 diff --git a/internal/version.go b/internal/version.go index bc54163ca..4dd432839 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 3 - VersionPatch = 2 + VersionPatch = 3 VersionTag = "" // example: "rc1" ) From bad81c028f090af0e1005076829db67d1a749a14 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 9 Dec 2020 18:07:17 +0000 Subject: [PATCH 13/13] Don't recalculate event ID so often in sync (#1624) * Don't bail so quickly in fetchMissingStateEvents * Don't recalculate event IDs so often in sync API * Add comments * Fix comments * Update to matrix-org/gomatrixserverlib@eb6a890 --- go.mod | 2 +- go.sum | 4 ++-- .../postgres/current_room_state_table.go | 11 +++++----- .../postgres/output_room_events_table.go | 13 ++++++----- syncapi/storage/shared/syncserver.go | 22 ++++++++++++++----- .../sqlite3/current_room_state_table.go | 11 +++++----- .../sqlite3/output_room_events_table.go | 13 ++++++----- 7 files changed, 45 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index e788f43de..eddf96a7c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201207162041-232bace0cdd4 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201209172200-eb6a8903f9fb github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index f00f06d6f..fc48085a3 100644 --- a/go.sum +++ b/go.sum @@ -567,8 +567,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201207162041-232bace0cdd4 h1:geIU3lUL+P83W6NdQLUaR7vfVh5EHliIJ0H/RgCqbIU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201207162041-232bace0cdd4/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201209172200-eb6a8903f9fb h1:UlhiSebJupQ+qAM93cdVGg4nAJ6bnxwAA5/EBygtYoo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201209172200-eb6a8903f9fb/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 123272782..554163e58 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -76,7 +76,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + @@ -92,10 +92,10 @@ const selectStateEventSQL = "" + const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise - // the rowsToStreamEvents expects there to be exactly five columns. We need to + // the rowsToStreamEvents expects there to be exactly six columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id = ANY($1)" type currentRoomStateStatements struct { @@ -278,13 +278,14 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) { result := []*gomatrixserverlib.HeaderedEvent{} for rows.Next() { + var eventID string var eventBytes []byte - if err := rows.Scan(&eventBytes); err != nil { + if err := rows.Scan(&eventID, &eventBytes); err != nil { return nil, err } // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, err } result = append(result, &ev) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index ce4b63350..f4bbebd26 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -79,20 +79,20 @@ const insertEventSQL = "" + "RETURNING id" const selectEventsSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" const selectRecentEventsSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -413,6 +413,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { var result []types.StreamEvent for rows.Next() { var ( + eventID string streamPos types.StreamPosition eventBytes []byte excludeFromSync bool @@ -420,12 +421,12 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { txnID *string transactionID *api.TransactionID ) - if err := rows.Scan(&streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil { return nil, err } // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, err } diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 6c35a7653..9df049439 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) @@ -514,25 +515,28 @@ func (d *Database) addPDUDeltaToResponse( deltas, joinedRoomIDs, err = d.getStateDeltas( ctx, &device, txn, r, device.UserID, &stateFilter, ) + if err != nil { + return nil, fmt.Errorf("d.getStateDeltas: %w", err) + } } else { deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( ctx, &device, txn, r, device.UserID, &stateFilter, ) - } - if err != nil { - return nil, err + if err != nil { + return nil, fmt.Errorf("d.getStateDeltasForFullStateSync: %w", err) + } } for _, delta := range deltas { err = d.addRoomDeltaToResponse(ctx, &device, txn, r, delta, numRecentEventsPerRoom, res) if err != nil { - return nil, err + return nil, fmt.Errorf("d.addRoomDeltaToResponse: %w", err) } } // TODO: This should be done in getStateDeltas if err = d.addInvitesToResponse(ctx, txn, device.UserID, r, res); err != nil { - return nil, err + return nil, fmt.Errorf("d.addInvitesToResponse: %w", err) } succeeded = true @@ -1126,7 +1130,13 @@ func (d *Database) fetchMissingStateEvents( return nil, err } if len(stateEvents) != len(missing) { - return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) + logrus.WithContext(ctx).Warnf("Failed to map all event IDs to events (got %d, wanted %d)", len(stateEvents), len(missing)) + + // TODO: Why is this happening? It's probably the roomserver. Uncomment + // this error again when we work out what it is and fix it, otherwise we + // just end up returning lots of 500s to the client and that breaks + // pretty much everything, rather than just sending what we have. + //return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) } events = append(events, stateEvents...) return events, nil diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 357d4282e..f16a66127 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -64,7 +64,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2 IS NULL OR sender IN ($2) )" + " AND ( $3 IS NULL OR NOT(sender IN ($3)) )" + " AND ( $4 IS NULL OR type IN ($4) )" + @@ -80,10 +80,10 @@ const selectStateEventSQL = "" + const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise - // the rowsToStreamEvents expects there to be exactly five columns. We need to + // the rowsToStreamEvents expects there to be exactly six columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id IN ($1)" type currentRoomStateStatements struct { @@ -289,13 +289,14 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) { result := []*gomatrixserverlib.HeaderedEvent{} for rows.Next() { + var eventID string var eventBytes []byte - if err := rows.Scan(&eventBytes); err != nil { + if err := rows.Scan(&eventID, &eventBytes); err != nil { return nil, err } // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, err } result = append(result, &ev) diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 064075824..edbd36fb1 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -56,20 +56,20 @@ const insertEventSQL = "" + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $13" const selectEventsSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" const selectRecentEventsSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -428,6 +428,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { var result []types.StreamEvent for rows.Next() { var ( + eventID string streamPos types.StreamPosition eventBytes []byte excludeFromSync bool @@ -435,12 +436,12 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { txnID *string transactionID *api.TransactionID ) - if err := rows.Scan(&streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil { return nil, err } // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, err }