From 1ce9c524420bc6a7b08748ca736284382581ab56 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 4 Dec 2020 10:41:07 +0000 Subject: [PATCH 1/4] Don't recalculate event IDs so often (#1610) * Don't recalculate event IDs so often * Revert invite change * Make sure we're using the right NIDs * Update gomatrixserverlib * Update to NewEventFromTrustedJSONWithEventID * Fix go.mod * Update gomatrixserverlib to matrix-org/gomatrixserverlib#243 * Use BulkSelectEventID --- go.mod | 2 +- go.sum | 4 ++-- roomserver/storage/shared/storage.go | 28 +++++++++++++++++++++++----- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index d01c80860..54a139321 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201202134418-2ba106a5bca3 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 18715f86a..9780a688d 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201202134418-2ba106a5bca3 h1:+45Q/5FybBhHPMr10YdzJNFYO/6RRgkBcZbMzIRq5Ck= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201202134418-2ba106a5bca3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb h1:cN+v/rDbGg2p5e8xyxfATtwXeW7tI4aUn7slLXvuNyw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201204094806-e2d6b1a05ccb/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 2548980d4..83982299a 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -309,6 +309,10 @@ func (d *Database) Events( if err != nil { return nil, err } + eventIDs, _ := d.EventsTable.BulkSelectEventID(ctx, eventNIDs) + if err != nil { + eventIDs = map[types.EventNID]string{} + } results := make([]types.Event, len(eventJSONs)) for i, eventJSON := range eventJSONs { var roomNID types.RoomNID @@ -328,8 +332,8 @@ func (d *Database) Events( return nil, err } } - result.Event, err = gomatrixserverlib.NewEventFromTrustedJSON( - eventJSON.EventJSON, false, roomVersion, + result.Event, err = gomatrixserverlib.NewEventFromTrustedJSONWithEventID( + eventIDs[eventJSON.EventNID], eventJSON.EventJSON, false, roomVersion, ) if err != nil { return nil, err @@ -779,6 +783,7 @@ func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event { // GetStateEvent returns the current state event of a given type for a given room with a given state key // If no event could be found, returns nil // If there was an issue during the retrieval, returns an error +// nolint:gocyclo func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) { roomInfo, err := d.RoomInfo(ctx, roomID) if err != nil { @@ -800,6 +805,16 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s if err != nil { return nil, err } + var eventNIDs []types.EventNID + for _, e := range entries { + if e.EventTypeNID == eventTypeNID && e.EventStateKeyNID == stateKeyNID { + eventNIDs = append(eventNIDs, e.EventNID) + } + } + eventIDs, _ := d.EventsTable.BulkSelectEventID(ctx, eventNIDs) + if err != nil { + eventIDs = map[types.EventNID]string{} + } // return the event requested for _, e := range entries { if e.EventTypeNID == eventTypeNID && e.EventStateKeyNID == stateKeyNID { @@ -810,7 +825,7 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s if len(data) == 0 { return nil, fmt.Errorf("GetStateEvent: no json for event nid %d", e.EventNID) } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(data[0].EventJSON, false, roomInfo.RoomVersion) + ev, err := gomatrixserverlib.NewEventFromTrustedJSONWithEventID(eventIDs[e.EventNID], data[0].EventJSON, false, roomInfo.RoomVersion) if err != nil { return nil, err } @@ -921,7 +936,10 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu } } } - + eventIDs, _ := d.EventsTable.BulkSelectEventID(ctx, eventNIDs) + if err != nil { + eventIDs = map[types.EventNID]string{} + } events, err := d.EventJSONTable.BulkSelectEventJSON(ctx, eventNIDs) if err != nil { return nil, fmt.Errorf("GetBulkStateContent: failed to load event JSON for event nids: %w", err) @@ -929,7 +947,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu result := make([]tables.StrippedEvent, len(events)) for i := range events { roomVer := eventNIDToVer[events[i].EventNID] - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(events[i].EventJSON, false, roomVer) + ev, err := gomatrixserverlib.NewEventFromTrustedJSONWithEventID(eventIDs[events[i].EventNID], events[i].EventJSON, false, roomVer) if err != nil { return nil, fmt.Errorf("GetBulkStateContent: failed to load event JSON for event NID %v : %w", events[i].EventNID, err) } From a677a288bd3344a5d31080be047125043ada75dc Mon Sep 17 00:00:00 2001 From: Ronnie Ebrin Date: Fri, 4 Dec 2020 15:08:17 +0100 Subject: [PATCH 2/4] federationsender/roomserver: don't panic while federation is disabled (#1615) --- federationsender/consumers/roomserver.go | 21 ++++++++++++++------- federationsender/queue/queue.go | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 13c415bc7..b53cb251b 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -94,13 +94,20 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { } if err := s.processMessage(*output.NewRoomEvent); err != nil { - // panic rather than continue with an inconsistent database - log.WithFields(log.Fields{ - "event": string(ev.JSON()), - "add": output.NewRoomEvent.AddsStateEventIDs, - "del": output.NewRoomEvent.RemovesStateEventIDs, - log.ErrorKey: err, - }).Panicf("roomserver output log: write room event failure") + switch err.(type) { + case *queue.ErrorFederationDisabled: + log.WithField("error", output.Type).Info( + err.Error(), + ) + default: + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + "event": string(ev.JSON()), + "add": output.NewRoomEvent.AddsStateEventIDs, + "del": output.NewRoomEvent.RemovesStateEventIDs, + log.ErrorKey: err, + }).Panicf("roomserver output log: write room event failure") + } return nil } default: diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 95d37b2b8..d45af8686 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -122,13 +122,23 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d return oq } +type ErrorFederationDisabled struct { + Message string +} + +func (e *ErrorFederationDisabled) Error() string { + return e.Message +} + // SendEvent sends an event to the destinations func (oqs *OutgoingQueues) SendEvent( ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName, ) error { if oqs.disabled { - return fmt.Errorf("federation is disabled") + return &ErrorFederationDisabled{ + Message: "Federation disabled", + } } if origin != oqs.origin { // TODO: Support virtual hosting; gh issue #577. @@ -190,7 +200,9 @@ func (oqs *OutgoingQueues) SendEDU( destinations []gomatrixserverlib.ServerName, ) error { if oqs.disabled { - return fmt.Errorf("federation is disabled") + return &ErrorFederationDisabled{ + Message: "Federation disabled", + } } if origin != oqs.origin { // TODO: Support virtual hosting; gh issue #577. From 68d92fd3b4bfdc5a8e13ec66905bc7c0a2b951c9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 4 Dec 2020 14:09:21 +0000 Subject: [PATCH 3/4] Update sytest-whitelist/blacklist --- sytest-blacklist | 5 ++++- sytest-whitelist | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sytest-blacklist b/sytest-blacklist index f42d5f822..601a3f705 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -63,4 +63,7 @@ If user leaves room, remote user changes device and rejoins we see update in /sy A prev_batch token from incremental sync can be used in the v1 messages API # Blacklisted due to flakiness -Forgotten room messages cannot be paginated \ No newline at end of file +Forgotten room messages cannot be paginated + +# Blacklisted due to flakiness +Can re-join room if re-invited \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index 17027041d..da4b201ce 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -501,6 +501,5 @@ Can get rooms/{roomId}/state for a departed room (SPEC-216) Users cannot set notifications powerlevel higher than their own Forgetting room does not show up in v2 /sync Can forget room you've been kicked from -Can re-join room if re-invited /whois /joined_members return joined members From c052edafdd765d821f9732add4f5d33962ba5ba4 Mon Sep 17 00:00:00 2001 From: TR_SLimey <37966924+TR-SLimey@users.noreply.github.com> Date: Fri, 4 Dec 2020 14:10:01 +0000 Subject: [PATCH 4/4] Fix broken image in CONTRIBUTING.md (#1612) --- docs/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 1ab885ebc..ea4b2b27d 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -37,7 +37,7 @@ If a job fails, click the "details" button and you should be taken to the job's logs. ![Click the details button on the failing build -step](docs/images/details-button-location.jpg) +step](https://raw.githubusercontent.com/matrix-org/dendrite/master/docs/images/details-button-location.jpg) Scroll down to the failing step and you should see some log output. Scan the logs until you find what it's complaining about, fix it, submit a new commit,