From d1f87e63f1b744825f40e59b8202933608480a70 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 13:16:14 +0100 Subject: [PATCH 1/7] Move `SetLatestEvents` call --- roomserver/internal/input/input_latest_events.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index d6efad79d..ec6626720 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -178,6 +178,10 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { u.newStateNID = u.oldStateNID } + if err = u.updater.SetLatestEvents(u.roomInfo.RoomNID, u.latest, u.stateAtEvent.EventNID, u.newStateNID); err != nil { + return fmt.Errorf("u.updater.SetLatestEvents: %w", err) + } + update, err := u.makeOutputNewRoomEvent() if err != nil { return fmt.Errorf("u.makeOutputNewRoomEvent: %w", err) @@ -196,10 +200,6 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { return fmt.Errorf("u.api.WriteOutputEvents: %w", err) } - if err = u.updater.SetLatestEvents(u.roomInfo.RoomNID, u.latest, u.stateAtEvent.EventNID, u.newStateNID); err != nil { - return fmt.Errorf("u.updater.SetLatestEvents: %w", err) - } - if err = u.updater.MarkEventAsSent(u.stateAtEvent.EventNID); err != nil { return fmt.Errorf("u.updater.MarkEventAsSent: %w", err) } From 1c1d09abd4009e3fcc76b71ad158fb16e5e776cc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 14:17:04 +0100 Subject: [PATCH 2/7] Optimise `resolve-state` tool --- cmd/resolve-state/main.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 6852ab4fb..f8bb130c7 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -76,9 +76,14 @@ func main() { panic(err) } - var eventNIDs []types.EventNID + eventNIDMap := map[types.EventNID]struct{}{} for _, entry := range append(removed, added...) { - eventNIDs = append(eventNIDs, entry.EventNID) + eventNIDMap[entry.EventNID] = struct{}{} + } + + eventNIDs := make([]types.EventNID, 0, len(eventNIDMap)) + for eventNID := range eventNIDMap { + eventNIDs = append(eventNIDs, eventNID) } var eventEntries []types.Event @@ -129,12 +134,17 @@ func main() { stateEntries = append(stateEntries, entries...) } - var eventNIDs []types.EventNID + eventNIDMap := map[types.EventNID]struct{}{} for _, entry := range stateEntries { - eventNIDs = append(eventNIDs, entry.EventNID) + eventNIDMap[entry.EventNID] = struct{}{} } - fmt.Println("Fetching", len(eventNIDs), "state events") + eventNIDs := make([]types.EventNID, 0, len(eventNIDMap)) + for eventNID := range eventNIDMap { + eventNIDs = append(eventNIDs, eventNID) + } + + fmt.Println("Fetching", len(eventNIDMap), "state events") eventEntries, err := roomserverDB.Events(ctx, eventNIDs) if err != nil { panic(err) From bea3dbe77bac078f07927df54c63f145df8a0caf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:00:54 +0100 Subject: [PATCH 3/7] State and auth tweaks (update to matrix-org/gomatrixserverlib#335) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 67a7771ec..b7f1920ca 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index dcfed5eaa..3212c5d70 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 h1:esbNn9hg//tAStA6TogatAJAursw23A+yfVRQsdiv70= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 h1:55lVTCUNuq/WvTH6usVwrliqnxbziOjOCHeQs1tcSeg= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From ecee5f10f49487f0b6628fea065914ea3e685a79 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:08:54 +0100 Subject: [PATCH 4/7] Tweak logging for detected state resets --- roomserver/internal/input/input_latest_events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index ec6626720..7a4105ab8 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -275,7 +275,7 @@ func (u *latestEventsUpdater) latestState() error { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if removed := len(u.removed) - len(u.added); removed > 0 { + if removed := len(u.removed) - len(u.added); !u.rewritesState && removed > 0 { logrus.WithFields(logrus.Fields{ "event_id": u.event.EventID(), "room_id": u.event.RoomID(), @@ -283,7 +283,7 @@ func (u *latestEventsUpdater) latestState() error { "new_state_nid": u.newStateNID, "old_latest": u.oldLatest.EventIDs(), "new_latest": u.latest.EventIDs(), - }).Errorf("Unexpected state deletion (removing %d events)", removed) + }).Warnf("State reset detected (removing %d events)", removed) } // Also work out the state before the event removes and the event From cd22ba22b01fe5197bf8e4630c8414dd413225af Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:25:11 +0100 Subject: [PATCH 5/7] Improve Sentry reporting --- internal/httputil/httpapi.go | 1 + roomserver/internal/input/input.go | 6 ++++++ roomserver/internal/input/input_latest_events.go | 10 ++++++++++ 3 files changed, 17 insertions(+) diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index e0436c60a..ab8319bbb 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -60,6 +60,7 @@ func MakeAuthAPI( // add the user to Sentry, if enabled hub := sentry.GetHubFromContext(req.Context()) if hub != nil { + hub.Scope().SetTag("username", device.UserID) hub.Scope().SetTag("user_id", device.UserID) hub.Scope().SetTag("device_id", device.ID) } diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 429cc4bd2..a8a3e0248 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -188,6 +188,9 @@ func (w *worker) _next() { // Look up what the next event is that's waiting to be processed. ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute) defer cancel() + if scope := sentry.CurrentHub().Scope(); scope != nil { + scope.SetTag("room_id", w.roomID) + } msgs, err := w.subscription.Fetch(1, nats.Context(ctx)) switch err { case nil: @@ -239,6 +242,9 @@ func (w *worker) _next() { return } + if scope := sentry.CurrentHub().Scope(); scope != nil { + scope.SetTag("event_id", inputRoomEvent.Event.EventID()) + } roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Inc() defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec() diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 7a4105ab8..7e42db1b7 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/opentracing/opentracing-go" @@ -284,6 +285,15 @@ func (u *latestEventsUpdater) latestState() error { "old_latest": u.oldLatest.EventIDs(), "new_latest": u.latest.EventIDs(), }).Warnf("State reset detected (removing %d events)", removed) + sentry.WithScope(func(scope *sentry.Scope) { + scope.SetLevel("warning") + scope.SetTag("event_id", u.event.EventID()) + scope.SetTag("old_state_nid", fmt.Sprintf("%d", u.oldStateNID)) + scope.SetTag("new_state_nid", fmt.Sprintf("%d", u.newStateNID)) + scope.SetTag("old_latest", u.oldLatest.EventIDs()) + scope.SetTag("new_latest", u.latest.EventIDs()) + sentry.CaptureMessage("State reset detected") + }) } // Also work out the state before the event removes and the event From 5992b4c7ed2779e12c8b9da7bc96b2b6398cf04c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:35:43 +0100 Subject: [PATCH 6/7] Set Sentry user where known --- internal/httputil/httpapi.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index ab8319bbb..5612d78c1 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -60,7 +60,10 @@ func MakeAuthAPI( // add the user to Sentry, if enabled hub := sentry.GetHubFromContext(req.Context()) if hub != nil { - hub.Scope().SetTag("username", device.UserID) + hub.Scope().SetUser(sentry.User{ + Username: device.UserID, + IPAddress: req.RemoteAddr, + }) hub.Scope().SetTag("user_id", device.UserID) hub.Scope().SetTag("device_id", device.ID) } From fd7661f69cfecb426f0170a27a4e414b178a1cdf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:37:17 +0100 Subject: [PATCH 7/7] Tweak Sentry again --- internal/httputil/httpapi.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 5612d78c1..36dcaf453 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -61,8 +61,7 @@ func MakeAuthAPI( hub := sentry.GetHubFromContext(req.Context()) if hub != nil { hub.Scope().SetUser(sentry.User{ - Username: device.UserID, - IPAddress: req.RemoteAddr, + Username: device.UserID, }) hub.Scope().SetTag("user_id", device.UserID) hub.Scope().SetTag("device_id", device.ID)