Try to fix state reset sentry messages, maybe?

This commit is contained in:
Till Faelligen 2023-12-29 22:19:45 +01:00
parent 2c81b060d6
commit 925843d05b
No known key found for this signature in database
GPG key ID: 3DF82D8AB9211D4E
2 changed files with 24 additions and 5 deletions

View file

@ -108,12 +108,16 @@ type worker struct {
r *Inputer r *Inputer
roomID string roomID string
subscription *nats.Subscription subscription *nats.Subscription
sentryHub *sentry.Hub
} }
func (r *Inputer) startWorkerForRoom(roomID string) { func (r *Inputer) startWorkerForRoom(roomID string) {
v, loaded := r.workers.LoadOrStore(roomID, &worker{ v, loaded := r.workers.LoadOrStore(roomID, &worker{
r: r, r: r,
roomID: roomID, roomID: roomID,
// We're cloning the CurrentHub, as we otherwise get total garbage
// in sentry, with i.e. mismatching rooms and event_ids.
sentryHub: sentry.CurrentHub().Clone(),
}) })
w := v.(*worker) w := v.(*worker)
w.Lock() w.Lock()
@ -265,7 +269,7 @@ func (w *worker) _next() {
// Look up what the next event is that's waiting to be processed. // Look up what the next event is that's waiting to be processed.
ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute) ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute)
defer cancel() defer cancel()
if scope := sentry.CurrentHub().Scope(); scope != nil { if scope := w.sentryHub.Scope(); scope != nil {
scope.SetTag("room_id", w.roomID) scope.SetTag("room_id", w.roomID)
} }
msgs, err := w.subscription.Fetch(1, nats.Context(ctx)) msgs, err := w.subscription.Fetch(1, nats.Context(ctx))
@ -323,7 +327,7 @@ func (w *worker) _next() {
return return
} }
if scope := sentry.CurrentHub().Scope(); scope != nil { if scope := w.sentryHub.Scope(); scope != nil {
scope.SetTag("event_id", inputRoomEvent.Event.EventID()) scope.SetTag("event_id", inputRoomEvent.Event.EventID())
} }
@ -347,7 +351,7 @@ func (w *worker) _next() {
}).Warn("Roomserver rejected event") }).Warn("Roomserver rejected event")
default: default:
if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
sentry.CaptureException(err) w.sentryHub.CaptureException(err)
} }
logrus.WithError(err).WithFields(logrus.Fields{ logrus.WithError(err).WithFields(logrus.Fields{
"room_id": w.roomID, "room_id": w.roomID,

View file

@ -79,12 +79,26 @@ func (r *Inputer) updateLatestEvents(
return nil return nil
} }
// Attempt to get Sentry hub
w, loaded := r.workers.Load(event.RoomID())
if !loaded {
// this _should_ never happen...
logrus.Panicf("failed to load worker on existing room")
}
s, ok := w.(*worker)
if !ok {
// this _should_ never happen as well...
logrus.Panicf("failed to get sentry hub from worker")
}
u := latestEventsUpdater{ u := latestEventsUpdater{
api: r, api: r,
updater: updater, updater: updater,
stateAtEvent: stateAtEvent, stateAtEvent: stateAtEvent,
event: event, event: event,
rewritesState: rewritesState, rewritesState: rewritesState,
sentryHub: s.sentryHub,
} }
var updates []api.OutputEvent var updates []api.OutputEvent
@ -149,6 +163,7 @@ type latestEventsUpdater struct {
// The snapshots of current state before and after processing this event // The snapshots of current state before and after processing this event
oldStateNID types.StateSnapshotNID oldStateNID types.StateSnapshotNID
newStateNID types.StateSnapshotNID newStateNID types.StateSnapshotNID
sentryHub *sentry.Hub
} }
func (u *latestEventsUpdater) doUpdateLatestEvents(ctx context.Context, roomInfo *types.RoomInfo) ([]api.OutputEvent, error) { func (u *latestEventsUpdater) doUpdateLatestEvents(ctx context.Context, roomInfo *types.RoomInfo) ([]api.OutputEvent, error) {
@ -288,7 +303,7 @@ func (u *latestEventsUpdater) latestState(ctx context.Context, roomInfo *types.R
"rewrites_state": u.rewritesState, "rewrites_state": u.rewritesState,
"state_at_event": fmt.Sprintf("%#v", u.stateAtEvent), "state_at_event": fmt.Sprintf("%#v", u.stateAtEvent),
}).Warnf("State reset detected (removing %d events)", removed) }).Warnf("State reset detected (removing %d events)", removed)
sentry.WithScope(func(scope *sentry.Scope) { u.sentryHub.WithScope(func(scope *sentry.Scope) {
scope.SetLevel("warning") scope.SetLevel("warning")
scope.SetContext("State reset", map[string]interface{}{ scope.SetContext("State reset", map[string]interface{}{
"Event ID": u.event.EventID(), "Event ID": u.event.EventID(),
@ -300,7 +315,7 @@ func (u *latestEventsUpdater) latestState(ctx context.Context, roomInfo *types.R
"State rewritten": fmt.Sprintf("%v", u.rewritesState), "State rewritten": fmt.Sprintf("%v", u.rewritesState),
"State at event": fmt.Sprintf("%#v", u.stateAtEvent), "State at event": fmt.Sprintf("%#v", u.stateAtEvent),
}) })
sentry.CaptureMessage("State reset detected") u.sentryHub.CaptureMessage("State reset detected")
}) })
} }