From 7899f47e71631170bf2578bf04060c9f5abee461 Mon Sep 17 00:00:00 2001
From: George Antoniadis <george@noodles.gr>
Date: Wed, 26 Jul 2023 07:16:43 +0100
Subject: [PATCH 01/57] add deployment strategy option to helm chart (re #3021)
 (#3155)

@S7evinK sorry for the spam but any chance we get get this merged into
main at some point? It was previously merged in
https://github.com/matrix-org/dendrite/pull/3021 into a temp branch that
never made it into main. If there is an issue with this being merged let
me know.

---

Minor update to the helm chart to allow setting the update strategy as
the default `RollingUpdate` one is a bit annoying if using
`ReadWriteOnce` volumes for media. Hope this makes sense.

---

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] ~~I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests~~ Haven't touched any go files.
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `George Antoniadis <george@noodles.gr>` [skip ci]
---
 helm/dendrite/templates/deployment.yaml |  7 +++++++
 helm/dendrite/values.yaml               | 10 ++++++++++
 2 files changed, 17 insertions(+)

diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml
index df7dbbdc3..3a0bd68d8 100644
--- a/helm/dendrite/templates/deployment.yaml
+++ b/helm/dendrite/templates/deployment.yaml
@@ -26,6 +26,13 @@ spec:
       annotations:
         confighash: secret-{{ .Values.dendrite_config | toYaml | sha256sum | trunc 32 }}
     spec:
+      strategy:
+        type: {{ $.Values.strategy.type }}
+        {{- if eq $.Values.strategy.type "RollingUpdate" }}
+        rollingUpdate:
+          maxSurge: {{ $.Values.strategy.rollingUpdate.maxSurge }}
+          maxUnavailable: {{ $.Values.strategy.rollingUpdate.maxUnavailable }}
+        {{- end }}
       volumes:
       - name: {{ include "dendrite.fullname" . }}-conf-vol
         secret:
diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml
index 8a72f6693..396e70319 100644
--- a/helm/dendrite/values.yaml
+++ b/helm/dendrite/values.yaml
@@ -65,6 +65,16 @@ extraVolumeMounts: []
 # - mountPath: /etc/dendrite/extra-config
 #   name: extra-config
 
+strategy:
+  # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
+  # If you are using ReadWriteOnce volumes, you should probably use Recreate
+  type: RollingUpdate
+  rollingUpdate:
+    # -- Maximum number of pods that can be unavailable during the update process
+    maxUnavailable: 25%
+    # -- Maximum number of pods that can be scheduled above the desired number of pods
+    maxSurge: 25%
+
 strategy:
   # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
   # If you are using ReadWriteOnce volumes, you should probably use Recreate

From 79d4a0e399bb68920b81bc877744108095c06f1a Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 26 Jul 2023 09:09:04 +0200
Subject: [PATCH 02/57] Restore old behaviour of PurgeRoom

---
 docs/administration/4_adminapi.md            |  2 +-
 roomserver/internal/perform/perform_admin.go | 12 ------------
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index 6f6458997..40d02622b 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -75,7 +75,7 @@ This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a f
 
 ## POST `/_dendrite/admin/purgeRoom/{roomID}`
 
-This endpoint instructs Dendrite to remove the given room from its database. Before doing so, it will evacuate all local users from the room. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room.
+This endpoint instructs Dendrite to remove the given room from its database. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room.
 
 ## POST `/_synapse/admin/v1/send_server_notice`
 
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index 12b557f51..dd7132624 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -204,18 +204,6 @@ func (r *Admin) PerformAdminPurgeRoom(
 		return err
 	}
 
-	// Evacuate the room before purging it from the database
-	evacAffected, err := r.PerformAdminEvacuateRoom(ctx, roomID)
-	if err != nil {
-		logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to evacuate room before purging")
-		return err
-	}
-
-	logrus.WithFields(logrus.Fields{
-		"room_id":         roomID,
-		"evacuated_users": len(evacAffected),
-	}).Warn("Evacuated room, purging room from roomserver now")
-
 	logrus.WithField("room_id", roomID).Warn("Purging room from roomserver")
 	if err := r.DB.PurgeRoom(ctx, roomID); err != nil {
 		logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to purge room from roomserver")

From 3f727485d6e21a603e4df1cb31c3795cc1023caa Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 28 Jul 2023 08:40:05 +0200
Subject: [PATCH 03/57] Send a more generic error message to clients if the
 file can't be found (#3161)

Fixes #3160
---
 mediaapi/routing/download.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go
index 8fb1b6534..51afa1f9f 100644
--- a/mediaapi/routing/download.go
+++ b/mediaapi/routing/download.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/fs"
 	"mime"
 	"net/http"
 	"net/url"
@@ -126,6 +127,17 @@ func Download(
 		activeRemoteRequests, activeThumbnailGeneration,
 	)
 	if err != nil {
+		// If we bubbled up a os.PathError, e.g. no such file or directory, don't send
+		// it to the client, be more generic.
+		var perr *fs.PathError
+		if errors.As(err, &perr) {
+			dReq.Logger.WithError(err).Error("failed to open file")
+			dReq.jsonErrorResponse(w, util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: spec.NotFound("File not found"),
+			})
+			return
+		}
 		// TODO: Handle the fact we might have started writing the response
 		dReq.jsonErrorResponse(w, util.JSONResponse{
 			Code: http.StatusNotFound,

From af13fa1c7554fbed802d51421163f81b5b3fbe0d Mon Sep 17 00:00:00 2001
From: Sam Wedgwood <28223854+swedgwood@users.noreply.github.com>
Date: Mon, 31 Jul 2023 14:39:41 +0100
Subject: [PATCH 04/57] [pseudoIDs] Fixes for room alias tests (#3159)

Some (deceptively) simple fixes for some bugs that caused room alias
tests to fail (sytext `tests/30rooms/05aliases.pl`). Each commit has
details about what it fixes.

Sytest results:

- Sytest before (79d4a0e):
https://gist.github.com/swedgwood/972ac4ef93edd130d3db0930703d6c82
- Sytest after (4b09bed):
https://gist.github.com/swedgwood/504b00ac4ee892acb757b7fac55fa28a

Room aliases go from `8/15` to `15/15`, but looks like these fixes also
managed to fix about `4` other tests, which is a nice bonus :)

Signed-off-by: `Sam Wedgwood <sam@wedgwood.dev>`
---
 clientapi/routing/directory.go                |  84 +++++++++++----
 roomserver/api/alias.go                       |  34 ------
 roomserver/api/api.go                         |  15 ++-
 roomserver/internal/alias.go                  | 102 ++++++++----------
 .../internal/perform/perform_create_room.go   |  15 +--
 .../internal/perform/perform_upgrade.go       |  26 +++--
 roomserver/roomserver_test.go                 |  20 ++--
 syncapi/streams/stream_pdu.go                 |   2 +-
 8 files changed, 160 insertions(+), 138 deletions(-)

diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go
index d9129d1bd..3ec959b4b 100644
--- a/clientapi/routing/directory.go
+++ b/clientapi/routing/directory.go
@@ -181,13 +181,33 @@ func SetLocalAlias(
 		return *resErr
 	}
 
-	queryReq := roomserverAPI.SetRoomAliasRequest{
-		UserID: device.UserID,
-		RoomID: r.RoomID,
-		Alias:  alias,
+	roomID, err := spec.NewRoomID(r.RoomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid room ID"),
+		}
 	}
-	var queryRes roomserverAPI.SetRoomAliasResponse
-	if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *roomID, *userID)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("QuerySenderIDForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), senderID, *roomID, alias)
+	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -195,7 +215,7 @@ func SetLocalAlias(
 		}
 	}
 
-	if queryRes.AliasExists {
+	if aliasAlreadyExists {
 		return util.JSONResponse{
 			Code: http.StatusConflict,
 			JSON: spec.Unknown("The alias " + alias + " already exists."),
@@ -240,6 +260,31 @@ func RemoveLocalAlias(
 			JSON: spec.NotFound("The alias does not exist."),
 		}
 	}
+
+	// This seems like the kind of auth check that should be done in the roomserver, but
+	// if this check fails (user is not in the room), then there will be no SenderID for the user
+	// for pseudo-ID rooms - it will just return "". However, we can't use lack of a sender ID
+	// as meaning they are not in the room, since lacking a sender ID could be caused by other bugs.
+	// TODO: maybe have QuerySenderIDForUser return richer errors?
+	var queryResp roomserverAPI.QueryMembershipForUserResponse
+	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}, &queryResp)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.QueryMembershipForUser failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+	if !queryResp.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You do not have permission to remove this alias."),
+		}
+	}
+
 	deviceSenderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *userID)
 	if err != nil {
 		return util.JSONResponse{
@@ -247,28 +292,31 @@ func RemoveLocalAlias(
 			JSON: spec.NotFound("The alias does not exist."),
 		}
 	}
-
-	queryReq := roomserverAPI.RemoveRoomAliasRequest{
-		Alias:    alias,
-		SenderID: deviceSenderID,
-	}
-	var queryRes roomserverAPI.RemoveRoomAliasResponse
-	if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
+	// TODO: how to handle this case? missing user/room keys seem to be a whole new class of errors
+	if deviceSenderID == "" {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
+			JSON: spec.Unknown("internal server error"),
 		}
 	}
 
-	if !queryRes.Found {
+	aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), deviceSenderID, alias)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	if !aliasFound {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: spec.NotFound("The alias does not exist."),
 		}
 	}
 
-	if !queryRes.Removed {
+	if !aliasRemoved {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: spec.Forbidden("You do not have permission to remove this alias."),
diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go
index c091cf6a3..6269d0b04 100644
--- a/roomserver/api/alias.go
+++ b/roomserver/api/alias.go
@@ -16,26 +16,8 @@ package api
 
 import (
 	"regexp"
-
-	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
-// SetRoomAliasRequest is a request to SetRoomAlias
-type SetRoomAliasRequest struct {
-	// ID of the user setting the alias
-	UserID string `json:"user_id"`
-	// New alias for the room
-	Alias string `json:"alias"`
-	// The room ID the alias is referring to
-	RoomID string `json:"room_id"`
-}
-
-// SetRoomAliasResponse is a response to SetRoomAlias
-type SetRoomAliasResponse struct {
-	// Does the alias already refer to a room?
-	AliasExists bool `json:"alias_exists"`
-}
-
 // GetRoomIDForAliasRequest is a request to GetRoomIDForAlias
 type GetRoomIDForAliasRequest struct {
 	// Alias we want to lookup
@@ -63,22 +45,6 @@ type GetAliasesForRoomIDResponse struct {
 	Aliases []string `json:"aliases"`
 }
 
-// RemoveRoomAliasRequest is a request to RemoveRoomAlias
-type RemoveRoomAliasRequest struct {
-	// ID of the user removing the alias
-	SenderID spec.SenderID `json:"user_id"`
-	// The room alias to remove
-	Alias string `json:"alias"`
-}
-
-// RemoveRoomAliasResponse is a response to RemoveRoomAlias
-type RemoveRoomAliasResponse struct {
-	// Did the alias exist before?
-	Found bool `json:"found"`
-	// Did we remove it?
-	Removed bool `json:"removed"`
-}
-
 type AliasEvent struct {
 	Alias      string   `json:"alias"`
 	AltAliases []string `json:"alt_aliases"`
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 28b381d35..ed87ce93a 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -237,8 +237,19 @@ type ClientRoomserverAPI interface {
 	PerformPublish(ctx context.Context, req *PerformPublishRequest) error
 	// PerformForget forgets a rooms history for a specific user
 	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
-	SetRoomAlias(ctx context.Context, req *SetRoomAliasRequest, res *SetRoomAliasResponse) error
-	RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error
+
+	// Sets a room alias, as provided sender, pointing to the provided room ID.
+	//
+	// If err is nil, then the returned boolean indicates if the alias is already in use.
+	// If true, then the alias has not been set to the provided room, as it already in use.
+	SetRoomAlias(ctx context.Context, senderID spec.SenderID, roomID spec.RoomID, alias string) (aliasAlreadyExists bool, err error)
+
+	//RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error
+	// Removes a room alias, as provided sender.
+	//
+	// Returns whether the alias was found, whether it was removed, and an error (if any occurred)
+	RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error)
+
 	SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error)
 }
 
diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go
index b04a56fe8..a7f0aab9c 100644
--- a/roomserver/internal/alias.go
+++ b/roomserver/internal/alias.go
@@ -35,27 +35,27 @@ import (
 // SetRoomAlias implements alias.RoomserverInternalAPI
 func (r *RoomserverInternalAPI) SetRoomAlias(
 	ctx context.Context,
-	request *api.SetRoomAliasRequest,
-	response *api.SetRoomAliasResponse,
-) error {
+	senderID spec.SenderID,
+	roomID spec.RoomID,
+	alias string,
+) (aliasAlreadyUsed bool, err error) {
 	// Check if the alias isn't already referring to a room
-	roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias)
+	existingRoomID, err := r.DB.GetRoomIDForAlias(ctx, alias)
 	if err != nil {
-		return err
+		return false, err
 	}
-	if len(roomID) > 0 {
+
+	if len(existingRoomID) > 0 {
 		// If the alias already exists, stop the process
-		response.AliasExists = true
-		return nil
+		return true, nil
 	}
-	response.AliasExists = false
 
 	// Save the new alias
-	if err := r.DB.SetRoomAlias(ctx, request.Alias, request.RoomID, request.UserID); err != nil {
-		return err
+	if err := r.DB.SetRoomAlias(ctx, alias, roomID.String(), string(senderID)); err != nil {
+		return false, err
 	}
 
-	return nil
+	return false, nil
 }
 
 // GetRoomIDForAlias implements alias.RoomserverInternalAPI
@@ -116,90 +116,79 @@ func (r *RoomserverInternalAPI) GetAliasesForRoomID(
 // nolint:gocyclo
 // RemoveRoomAlias implements alias.RoomserverInternalAPI
 // nolint: gocyclo
-func (r *RoomserverInternalAPI) RemoveRoomAlias(
-	ctx context.Context,
-	request *api.RemoveRoomAliasRequest,
-	response *api.RemoveRoomAliasResponse,
-) error {
-	roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias)
+func (r *RoomserverInternalAPI) RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error) {
+	roomID, err := r.DB.GetRoomIDForAlias(ctx, alias)
 	if err != nil {
-		return fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err)
+		return false, false, fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err)
 	}
 	if roomID == "" {
-		response.Found = false
-		response.Removed = false
-		return nil
+		return false, false, nil
 	}
 
 	validRoomID, err := spec.NewRoomID(roomID)
 	if err != nil {
-		return err
+		return true, false, err
 	}
 
-	sender, err := r.QueryUserIDForSender(ctx, *validRoomID, request.SenderID)
+	sender, err := r.QueryUserIDForSender(ctx, *validRoomID, senderID)
 	if err != nil || sender == nil {
-		return fmt.Errorf("r.QueryUserIDForSender: %w", err)
+		return true, false, fmt.Errorf("r.QueryUserIDForSender: %w", err)
 	}
 	virtualHost := sender.Domain()
 
-	response.Found = true
-	creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias)
+	creatorID, err := r.DB.GetCreatorIDForAlias(ctx, alias)
 	if err != nil {
-		return fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err)
+		return true, false, fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err)
 	}
 
-	if spec.SenderID(creatorID) != request.SenderID {
+	if spec.SenderID(creatorID) != senderID {
 		var plEvent *types.HeaderedEvent
 		var pls *gomatrixserverlib.PowerLevelContent
 
 		plEvent, err = r.DB.GetStateEvent(ctx, roomID, spec.MRoomPowerLevels, "")
 		if err != nil {
-			return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+			return true, false, fmt.Errorf("r.DB.GetStateEvent: %w", err)
 		}
 
 		pls, err = plEvent.PowerLevels()
 		if err != nil {
-			return fmt.Errorf("plEvent.PowerLevels: %w", err)
+			return true, false, fmt.Errorf("plEvent.PowerLevels: %w", err)
 		}
 
-		if pls.UserLevel(request.SenderID) < pls.EventLevel(spec.MRoomCanonicalAlias, true) {
-			response.Removed = false
-			return nil
+		if pls.UserLevel(senderID) < pls.EventLevel(spec.MRoomCanonicalAlias, true) {
+			return true, false, nil
 		}
 	}
 
 	ev, err := r.DB.GetStateEvent(ctx, roomID, spec.MRoomCanonicalAlias, "")
 	if err != nil && err != sql.ErrNoRows {
-		return err
+		return true, false, err
 	} else if ev != nil {
 		stateAlias := gjson.GetBytes(ev.Content(), "alias").Str
 		// the alias to remove is currently set as the canonical alias, remove it
-		if stateAlias == request.Alias {
+		if stateAlias == alias {
 			res, err := sjson.DeleteBytes(ev.Content(), "alias")
 			if err != nil {
-				return err
+				return true, false, err
 			}
 
-			senderID := request.SenderID
-			if request.SenderID != ev.SenderID() {
-				senderID = ev.SenderID()
-			}
-			sender, err := r.QueryUserIDForSender(ctx, *validRoomID, senderID)
-			if err != nil || sender == nil {
-				return err
+			canonicalSenderID := ev.SenderID()
+			canonicalSender, err := r.QueryUserIDForSender(ctx, *validRoomID, canonicalSenderID)
+			if err != nil || canonicalSender == nil {
+				return true, false, err
 			}
 
 			validRoomID, err := spec.NewRoomID(roomID)
 			if err != nil {
-				return err
+				return true, false, err
 			}
-			identity, err := r.SigningIdentityFor(ctx, *validRoomID, *sender)
+			identity, err := r.SigningIdentityFor(ctx, *validRoomID, *canonicalSender)
 			if err != nil {
-				return err
+				return true, false, err
 			}
 
 			proto := &gomatrixserverlib.ProtoEvent{
-				SenderID: string(senderID),
+				SenderID: string(canonicalSenderID),
 				RoomID:   ev.RoomID(),
 				Type:     ev.Type(),
 				StateKey: ev.StateKey(),
@@ -208,34 +197,33 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias(
 
 			eventsNeeded, err := gomatrixserverlib.StateNeededForProtoEvent(proto)
 			if err != nil {
-				return fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err)
+				return true, false, fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err)
 			}
 			if len(eventsNeeded.Tuples()) == 0 {
-				return errors.New("expecting state tuples for event builder, got none")
+				return true, false, errors.New("expecting state tuples for event builder, got none")
 			}
 
 			stateRes := &api.QueryLatestEventsAndStateResponse{}
 			if err = helpers.QueryLatestEventsAndState(ctx, r.DB, r, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil {
-				return err
+				return true, false, err
 			}
 
 			newEvent, err := eventutil.BuildEvent(ctx, proto, &identity, time.Now(), &eventsNeeded, stateRes)
 			if err != nil {
-				return err
+				return true, false, err
 			}
 
 			err = api.SendEvents(ctx, r, api.KindNew, []*types.HeaderedEvent{newEvent}, virtualHost, r.ServerName, r.ServerName, nil, false)
 			if err != nil {
-				return err
+				return true, false, err
 			}
 		}
 	}
 
 	// Remove the alias from the database
-	if err := r.DB.RemoveRoomAlias(ctx, request.Alias); err != nil {
-		return err
+	if err := r.DB.RemoveRoomAlias(ctx, alias); err != nil {
+		return true, false, err
 	}
 
-	response.Removed = true
-	return nil
+	return true, true, nil
 }
diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go
index 12e756c2e..cd6629d28 100644
--- a/roomserver/internal/perform/perform_create_room.go
+++ b/roomserver/internal/perform/perform_create_room.go
@@ -433,23 +433,16 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo
 	// from creating the room but still failing due to the alias having already
 	// been taken.
 	if roomAlias != "" {
-		aliasReq := api.SetRoomAliasRequest{
-			Alias:  roomAlias,
-			RoomID: roomID.String(),
-			UserID: userID.String(),
-		}
-
-		var aliasResp api.SetRoomAliasResponse
-		err = c.RSAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed")
+		aliasAlreadyExists, aliasErr := c.RSAPI.SetRoomAlias(ctx, senderID, roomID, roomAlias)
+		if aliasErr != nil {
+			util.GetLogger(ctx).WithError(aliasErr).Error("aliasAPI.SetRoomAlias failed")
 			return "", &util.JSONResponse{
 				Code: http.StatusInternalServerError,
 				JSON: spec.InternalServerError{},
 			}
 		}
 
-		if aliasResp.AliasExists {
+		if aliasAlreadyExists {
 			return "", &util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.RoomInUse("Room alias already exists."),
diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go
index 32f547dc1..74c62cd9e 100644
--- a/roomserver/internal/perform/perform_upgrade.go
+++ b/roomserver/internal/perform/perform_upgrade.go
@@ -116,7 +116,7 @@ func (r *Upgrader) performRoomUpgrade(
 	}
 
 	// 4. Move local aliases to the new room
-	if pErr = moveLocalAliases(ctx, roomID, newRoomID, senderID, userID, r.URSAPI); pErr != nil {
+	if pErr = moveLocalAliases(ctx, roomID, newRoomID, senderID, r.URSAPI); pErr != nil {
 		return "", pErr
 	}
 
@@ -171,7 +171,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T
 }
 
 func moveLocalAliases(ctx context.Context,
-	roomID, newRoomID string, senderID spec.SenderID, userID spec.UserID,
+	roomID, newRoomID string, senderID spec.SenderID,
 	URSAPI api.RoomserverInternalAPI,
 ) (err error) {
 
@@ -181,17 +181,27 @@ func moveLocalAliases(ctx context.Context,
 		return fmt.Errorf("Failed to get old room aliases: %w", err)
 	}
 
+	// TODO: this should be spec.RoomID further up the call stack
+	parsedNewRoomID, err := spec.NewRoomID(newRoomID)
+	if err != nil {
+		return err
+	}
+
 	for _, alias := range aliasRes.Aliases {
-		removeAliasReq := api.RemoveRoomAliasRequest{SenderID: senderID, Alias: alias}
-		removeAliasRes := api.RemoveRoomAliasResponse{}
-		if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil {
+		aliasFound, aliasRemoved, err := URSAPI.RemoveRoomAlias(ctx, senderID, alias)
+		if err != nil {
 			return fmt.Errorf("Failed to remove old room alias: %w", err)
+		} else if !aliasFound {
+			return fmt.Errorf("Failed to remove old room alias: alias not found, possible race")
+		} else if !aliasRemoved {
+			return fmt.Errorf("Failed to remove old alias")
 		}
 
-		setAliasReq := api.SetRoomAliasRequest{UserID: userID.String(), Alias: alias, RoomID: newRoomID}
-		setAliasRes := api.SetRoomAliasResponse{}
-		if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil {
+		aliasAlreadyExists, err := URSAPI.SetRoomAlias(ctx, senderID, *parsedNewRoomID, alias)
+		if err != nil {
 			return fmt.Errorf("Failed to set new room alias: %w", err)
+		} else if aliasAlreadyExists {
+			return fmt.Errorf("Failed to set new room alias: alias exists when it should have just been removed")
 		}
 	}
 	return nil
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index ce0721bea..1626bf831 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -227,6 +227,11 @@ func TestPurgeRoom(t *testing.T) {
 	bob := test.NewUser(t)
 	room := test.NewRoom(t, alice, test.RoomPreset(test.PresetTrustedPrivateChat))
 
+	roomID, err := spec.NewRoomID(room.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Invite Bob
 	inviteEvent := room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
 		"membership": "invite",
@@ -274,9 +279,7 @@ func TestPurgeRoom(t *testing.T) {
 		if !isPublished {
 			t.Fatalf("room should be published before purging")
 		}
-
-		aliasResp := &api.SetRoomAliasResponse{}
-		if err = rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{RoomID: room.ID, Alias: "myalias", UserID: alice.ID}, aliasResp); err != nil {
+		if _, err = rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID), *roomID, "myalias"); err != nil {
 			t.Fatal(err)
 		}
 		// check the alias is actually there
@@ -930,14 +933,17 @@ func TestUpgrade(t *testing.T) {
 			upgradeUser: alice.ID,
 			roomFunc: func(rsAPI api.RoomserverInternalAPI) string {
 				r := test.NewRoom(t, alice)
+				roomID, err := spec.NewRoomID(r.ID)
+				if err != nil {
+					t.Fatal(err)
+				}
 				if err := api.SendEvents(ctx, rsAPI, api.KindNew, r.Events(), "test", "test", "test", nil, false); err != nil {
 					t.Errorf("failed to send events: %v", err)
 				}
 
-				if err := rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{
-					RoomID: r.ID,
-					Alias:  "#myroomalias:test",
-				}, &api.SetRoomAliasResponse{}); err != nil {
+				if _, err := rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID),
+					*roomID,
+					"#myroomalias:test"); err != nil {
 					t.Fatal(err)
 				}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 3f6888804..48daf857d 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -519,7 +519,7 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 	}
 
 	var evNew gomatrixserverlib.PDU
-	evNew, err = gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON(newEv, false)
+	evNew, err = gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSONWithEventID(ev.EventID(), newEv, false)
 	if err != nil {
 		return nil, err
 	}

From c7193e24d06a549b2e4a3bfca2d6e0f6c62d5f80 Mon Sep 17 00:00:00 2001
From: Sam Wedgwood <28223854+swedgwood@users.noreply.github.com>
Date: Wed, 2 Aug 2023 11:12:14 +0100
Subject: [PATCH 05/57] Use `*spec.SenderID` for `QuerySenderIDForUser` (#3164)

There are cases where a dendrite instance is unaware of a pseudo ID for
a user, the user is not a member of that room. To represent this case,
we currently use the 'zero' value, which is often not checked and so
causes errors later down the line. To make this case more explict, and
to be consistent with `QueryUserIDForSender`, this PR changes this to
use a pointer (and `nil` to mean no sender ID).

Signed-off-by: `Sam Wedgwood <sam@wedgwood.dev>`
---
 clientapi/routing/directory.go                 | 16 +++++++++++-----
 clientapi/routing/membership.go                | 16 ++++++++++------
 clientapi/routing/profile.go                   |  5 ++++-
 clientapi/routing/redaction.go                 | 16 +++++++++++++---
 clientapi/routing/sendevent.go                 |  8 +++++---
 clientapi/threepid/invites.go                  |  4 +++-
 federationapi/federationapi_test.go            |  5 +++--
 federationapi/internal/perform.go              |  4 +++-
 federationapi/routing/join.go                  |  7 +++++--
 federationapi/routing/leave.go                 |  8 +++++++-
 go.mod                                         |  4 ++--
 go.sum                                         | 14 ++++++++++----
 roomserver/api/api.go                          |  2 +-
 roomserver/internal/perform/perform_admin.go   |  4 +++-
 roomserver/internal/perform/perform_invite.go  |  6 ++++--
 roomserver/internal/perform/perform_join.go    |  8 +++++---
 roomserver/internal/perform/perform_leave.go   | 11 ++++++-----
 roomserver/internal/perform/perform_upgrade.go | 17 ++++++++++-------
 roomserver/internal/query/query.go             | 17 +++++++++++------
 setup/mscs/msc2836/msc2836_test.go             |  5 +++--
 syncapi/internal/history_visibility.go         |  4 ++--
 syncapi/storage/shared/storage_consumer.go     |  8 ++++----
 userapi/consumers/roomserver.go                |  5 ++++-
 23 files changed, 129 insertions(+), 65 deletions(-)

diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go
index 3ec959b4b..907727662 100644
--- a/clientapi/routing/directory.go
+++ b/clientapi/routing/directory.go
@@ -204,9 +204,15 @@ func SetLocalAlias(
 			Code: http.StatusInternalServerError,
 			JSON: spec.Unknown("internal server error"),
 		}
+	} else if senderID == nil {
+		util.GetLogger(req.Context()).WithField("roomID", *roomID).WithField("userID", *userID).Error("Sender ID not found")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
 	}
 
-	aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), senderID, *roomID, alias)
+	aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), *senderID, *roomID, alias)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 		return util.JSONResponse{
@@ -293,14 +299,14 @@ func RemoveLocalAlias(
 		}
 	}
 	// TODO: how to handle this case? missing user/room keys seem to be a whole new class of errors
-	if deviceSenderID == "" {
+	if deviceSenderID == nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: spec.Unknown("internal server error"),
 		}
 	}
 
-	aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), deviceSenderID, alias)
+	aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), *deviceSenderID, alias)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
 		return util.JSONResponse{
@@ -385,7 +391,7 @@ func SetVisibility(
 		}
 	}
 	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
-	if err != nil {
+	if err != nil || senderID == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: spec.Unknown("failed to find senderID for this user"),
@@ -416,7 +422,7 @@ func SetVisibility(
 
 	// NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event
 	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].PDU)
-	if power.UserLevel(senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
+	if power.UserLevel(*senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index def6f0617..8b8cc47bc 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -71,7 +71,7 @@ func SendBan(
 		}
 	}
 	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
-	if err != nil {
+	if err != nil || senderID == nil {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: spec.Forbidden("You don't have permission to ban this user, unknown senderID"),
@@ -87,7 +87,7 @@ func SendBan(
 	if errRes != nil {
 		return *errRes
 	}
-	allowedToBan := pl.UserLevel(senderID) >= pl.Ban
+	allowedToBan := pl.UserLevel(*senderID) >= pl.Ban
 	if !allowedToBan {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
@@ -169,7 +169,7 @@ func SendKick(
 		}
 	}
 	senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID)
-	if err != nil {
+	if err != nil || senderID == nil {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"),
@@ -185,7 +185,7 @@ func SendKick(
 	if errRes != nil {
 		return *errRes
 	}
-	allowedToKick := pl.UserLevel(senderID) >= pl.Kick
+	allowedToKick := pl.UserLevel(*senderID) >= pl.Kick
 	if !allowedToKick {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
@@ -476,6 +476,8 @@ func buildMembershipEvent(
 	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID)
 	if err != nil {
 		return nil, err
+	} else if senderID == nil {
+		return nil, fmt.Errorf("no sender ID for %s in %s", *userID, *validRoomID)
 	}
 
 	targetID, err := spec.NewUserID(targetUserID, true)
@@ -485,6 +487,8 @@ func buildMembershipEvent(
 	targetSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *targetID)
 	if err != nil {
 		return nil, err
+	} else if targetSenderID == nil {
+		return nil, fmt.Errorf("no sender ID for %s in %s", *targetID, *validRoomID)
 	}
 
 	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *userID)
@@ -492,8 +496,8 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	return buildMembershipEventDirect(ctx, targetSenderID, reason, profile.DisplayName, profile.AvatarURL,
-		senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI)
+	return buildMembershipEventDirect(ctx, *targetSenderID, reason, profile.DisplayName, profile.AvatarURL,
+		*senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go
index 35da15e0e..66b58507e 100644
--- a/clientapi/routing/profile.go
+++ b/clientapi/routing/profile.go
@@ -16,6 +16,7 @@ package routing
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
@@ -362,8 +363,10 @@ func buildMembershipEvents(
 		senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
 		if err != nil {
 			return nil, err
+		} else if senderID == nil {
+			return nil, fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID)
 		}
-		senderIDString := string(senderID)
+		senderIDString := string(*senderID)
 		proto := gomatrixserverlib.ProtoEvent{
 			SenderID: senderIDString,
 			RoomID:   roomID,
diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go
index 1b9a5a818..230c96d28 100644
--- a/clientapi/routing/redaction.go
+++ b/clientapi/routing/redaction.go
@@ -74,6 +74,16 @@ func SendRedaction(
 		return *resErr
 	}
 
+	// if user is member of room, and sender ID is nil, then this user doesn't have a pseudo ID for some reason,
+	// which is unexpected.
+	if senderID == nil {
+		util.GetLogger(req.Context()).WithField("userID", *deviceUserID).WithField("roomID", roomID).Error("missing sender ID for user, despite having membership")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
 	if txnID != nil {
 		// Try to fetch response from transactionsCache
 		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
@@ -98,7 +108,7 @@ func SendRedaction(
 	// "Users may redact their own events, and any user with a power level greater than or equal
 	// to the redact power level of the room may redact events there"
 	// https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-rooms-roomid-redact-eventid-txnid
-	allowedToRedact := ev.SenderID() == senderID
+	allowedToRedact := ev.SenderID() == *senderID
 	if !allowedToRedact {
 		plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{
 			EventType: spec.MRoomPowerLevels,
@@ -119,7 +129,7 @@ func SendRedaction(
 				),
 			}
 		}
-		allowedToRedact = pl.UserLevel(senderID) >= pl.Redact
+		allowedToRedact = pl.UserLevel(*senderID) >= pl.Redact
 	}
 	if !allowedToRedact {
 		return util.JSONResponse{
@@ -136,7 +146,7 @@ func SendRedaction(
 
 	// create the new event and set all the fields we can
 	proto := gomatrixserverlib.ProtoEvent{
-		SenderID: string(senderID),
+		SenderID: string(*senderID),
 		RoomID:   roomID,
 		Type:     spec.MRoomRedaction,
 		Redacts:  eventID,
diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index 41a3793ae..172001714 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -251,8 +251,10 @@ func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID strin
 		senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *uID)
 		if err != nil {
 			return err
+		} else if senderID == nil {
+			return fmt.Errorf("sender ID not found for %s in %s", uID, *validRoomID)
 		}
-		userMap[string(senderID)] = level
+		userMap[string(*senderID)] = level
 		delete(userMap, user)
 	}
 	r["users"] = userMap
@@ -314,7 +316,7 @@ func generateSendEvent(
 		}
 	}
 	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
-	if err != nil {
+	if err != nil || senderID == nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: spec.NotFound("Unable to find senderID for user"),
@@ -323,7 +325,7 @@ func generateSendEvent(
 
 	// create the new event and set all the fields we can
 	proto := gomatrixserverlib.ProtoEvent{
-		SenderID: string(senderID),
+		SenderID: string(*senderID),
 		RoomID:   roomID,
 		Type:     eventType,
 		StateKey: stateKey,
diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go
index d15cc6d46..365e9f869 100644
--- a/clientapi/threepid/invites.go
+++ b/clientapi/threepid/invites.go
@@ -366,9 +366,11 @@ func emit3PIDInviteEvent(
 	sender, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID)
 	if err != nil {
 		return err
+	} else if sender == nil {
+		return fmt.Errorf("sender ID not found for %s in %s", *userID, *validRoomID)
 	}
 	proto := &gomatrixserverlib.ProtoEvent{
-		SenderID: string(sender),
+		SenderID: string(*sender),
 		RoomID:   roomID,
 		Type:     "m.room.third_party_invite",
 		StateKey: &res.Token,
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index 5d167c0ee..c426eb67d 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -40,8 +40,9 @@ func (f *fedRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec
 	return spec.NewUserID(string(senderID), true)
 }
 
-func (f *fedRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) {
-	return spec.SenderID(userID.String()), nil
+func (f *fedRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	senderID := spec.SenderID(userID.String())
+	return &senderID, nil
 }
 
 // PerformJoin will call this function
diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index ff00305bf..3bba3ea0d 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -481,8 +481,10 @@ func (r *FederationInternalAPI) PerformLeave(
 		senderID, err := r.rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID)
 		if err != nil {
 			return err
+		} else if senderID == nil {
+			return fmt.Errorf("sender ID not found for %s in %s", *userID, *roomID)
 		}
-		senderIDString := string(senderID)
+		senderIDString := string(*senderID)
 		respMakeLeave.LeaveEvent.Type = spec.MRoomMember
 		respMakeLeave.LeaveEvent.SenderID = senderIDString
 		respMakeLeave.LeaveEvent.StateKey = &senderIDString
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index a090dbc8d..ce7ad30ff 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -99,7 +99,7 @@ func MakeJoin(
 		Roomserver: rsAPI,
 	}
 
-	senderID, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID)
+	senderIDPtr, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID)
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QuerySenderIDForUser failed")
 		return util.JSONResponse{
@@ -108,8 +108,11 @@ func MakeJoin(
 		}
 	}
 
-	if senderID == "" {
+	var senderID spec.SenderID
+	if senderIDPtr == nil {
 		senderID = spec.SenderID(userID.String())
+	} else {
+		senderID = *senderIDPtr
 	}
 
 	input := gomatrixserverlib.HandleMakeJoinInput{
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index 5c8dd00f3..f28c82115 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -94,11 +94,17 @@ func MakeLeave(
 			Code: http.StatusInternalServerError,
 			JSON: spec.InternalServerError{},
 		}
+	} else if senderID == nil {
+		util.GetLogger(httpReq.Context()).WithField("roomID", roomID).WithField("userID", userID).Error("rsAPI.QuerySenderIDForUser returned nil sender ID")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
 	}
 
 	input := gomatrixserverlib.HandleMakeLeaveInput{
 		UserID:             userID,
-		SenderID:           senderID,
+		SenderID:           *senderID,
 		RoomID:             roomID,
 		RoomVersion:        roomVersion,
 		RequestOrigin:      request.Origin(),
diff --git a/go.mod b/go.mod
index 0e77f903e..74b17166c 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230721154317-b5b0448aa378
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230802090652-1b697d109d87
 	github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
@@ -36,7 +36,7 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.2
-	github.com/tidwall/gjson v1.14.4
+	github.com/tidwall/gjson v1.15.0
 	github.com/tidwall/sjson v1.2.5
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
diff --git a/go.sum b/go.sum
index 28c60df43..779c28618 100644
--- a/go.sum
+++ b/go.sum
@@ -113,6 +113,7 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -175,12 +176,14 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM=
 github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0=
 github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
@@ -207,8 +210,10 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230721154317-b5b0448aa378 h1:a6sfiJiNZWVbPRHvEB/YlpqSg+Dh7El+824mzccSk68=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230721154317-b5b0448aa378/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230801102756-b66c2627dc08 h1:uy1mlUraKrEbUzZ3KrSQp/nLxMccVhIJM8mZSIbQzeA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230801102756-b66c2627dc08/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230802090652-1b697d109d87 h1:z0RFUknidOShRxkvjT3ovGCWnusyplu6OLjFHcbDYaE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230802090652-1b697d109d87/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -241,6 +246,7 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
 github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
 github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA=
@@ -322,8 +328,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
-github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.15.0 h1:5n/pM+v3r5ujuNl4YLZLsQ+UE5jlkLVm7jMzT5Mpolw=
+github.com/tidwall/gjson v1.15.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index ed87ce93a..69997fc41 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -97,7 +97,7 @@ type InputRoomEventsAPI interface {
 }
 
 type QuerySenderIDAPI interface {
-	QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error)
+	QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error)
 	QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error)
 }
 
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index dd7132624..2888067b4 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -292,10 +292,12 @@ func (r *Admin) PerformAdminDownloadState(
 	senderID, err := r.Queryer.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
 	if err != nil {
 		return err
+	} else if senderID == nil {
+		return fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID)
 	}
 	proto := &gomatrixserverlib.ProtoEvent{
 		Type:     "org.matrix.dendrite.state_download",
-		SenderID: string(senderID),
+		SenderID: string(*senderID),
 		RoomID:   roomID,
 		Content:  spec.RawJSON("{}"),
 	}
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index 278ddd7d8..e07780d68 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -133,6 +133,8 @@ func (r *Inviter) PerformInvite(
 	senderID, err := r.RSAPI.QuerySenderIDForUser(ctx, req.InviteInput.RoomID, req.InviteInput.Inviter)
 	if err != nil {
 		return err
+	} else if senderID == nil {
+		return fmt.Errorf("sender ID not found for %s in %s", req.InviteInput.Inviter, req.InviteInput.RoomID)
 	}
 	info, err := r.DB.RoomInfo(ctx, req.InviteInput.RoomID.String())
 	if err != nil {
@@ -140,7 +142,7 @@ func (r *Inviter) PerformInvite(
 	}
 
 	proto := gomatrixserverlib.ProtoEvent{
-		SenderID: string(senderID),
+		SenderID: string(*senderID),
 		RoomID:   req.InviteInput.RoomID.String(),
 		Type:     "m.room.member",
 	}
@@ -187,7 +189,7 @@ func (r *Inviter) PerformInvite(
 		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return r.RSAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		},
-		SenderIDQuerier: func(roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) {
+		SenderIDQuerier: func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
 			return r.RSAPI.QuerySenderIDForUser(ctx, roomID, userID)
 		},
 		SenderIDCreator: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) {
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index dfce9cc77..23988e467 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -201,11 +201,11 @@ func (r *Joiner) performJoinRoomByID(
 	if err == nil && info != nil {
 		switch info.RoomVersion {
 		case gomatrixserverlib.RoomVersionPseudoIDs:
-			senderID, err = r.Queryer.QuerySenderIDForUser(ctx, *roomID, *userID)
-			if err == nil {
+			senderIDPtr, queryErr := r.Queryer.QuerySenderIDForUser(ctx, *roomID, *userID)
+			if queryErr == nil {
 				checkInvitePending = true
 			}
-			if senderID == "" {
+			if senderIDPtr == nil {
 				// create user room key if needed
 				key, keyErr := r.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, *userID, *roomID)
 				if keyErr != nil {
@@ -213,6 +213,8 @@ func (r *Joiner) performJoinRoomByID(
 					return "", "", fmt.Errorf("GetOrCreateUserRoomPrivateKey failed: %w", keyErr)
 				}
 				senderID = spec.SenderIDFromPseudoIDKey(key)
+			} else {
+				senderID = *senderIDPtr
 			}
 		default:
 			checkInvitePending = true
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index a20896cf7..5c63a6684 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -73,6 +73,7 @@ func (r *Leaver) PerformLeave(
 	return nil, fmt.Errorf("room ID %q is invalid", req.RoomID)
 }
 
+// nolint:gocyclo
 func (r *Leaver) performLeaveRoomByID(
 	ctx context.Context,
 	req *api.PerformLeaveRequest,
@@ -83,20 +84,20 @@ func (r *Leaver) performLeaveRoomByID(
 		return nil, err
 	}
 	leaver, err := r.RSAPI.QuerySenderIDForUser(ctx, *roomID, req.Leaver)
-	if err != nil {
+	if err != nil || leaver == nil {
 		return nil, fmt.Errorf("leaver %s has no matching senderID in this room", req.Leaver.String())
 	}
 
 	// If there's an invite outstanding for the room then respond to
 	// that.
-	isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, leaver)
+	isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, *leaver)
 	if err == nil && isInvitePending {
 		sender, serr := r.RSAPI.QueryUserIDForSender(ctx, *roomID, senderUser)
 		if serr != nil || sender == nil {
 			return nil, fmt.Errorf("sender %q has no matching userID", senderUser)
 		}
 		if !r.Cfg.Matrix.IsLocalServerName(sender.Domain()) {
-			return r.performFederatedRejectInvite(ctx, req, res, *sender, eventID, leaver)
+			return r.performFederatedRejectInvite(ctx, req, res, *sender, eventID, *leaver)
 		}
 		// check that this is not a "server notice room"
 		accData := &userapi.QueryAccountDataResponse{}
@@ -132,7 +133,7 @@ func (r *Leaver) performLeaveRoomByID(
 		StateToFetch: []gomatrixserverlib.StateKeyTuple{
 			{
 				EventType: spec.MRoomMember,
-				StateKey:  string(leaver),
+				StateKey:  string(*leaver),
 			},
 		},
 	}
@@ -157,7 +158,7 @@ func (r *Leaver) performLeaveRoomByID(
 	}
 
 	// Prepare the template for the leave event.
-	senderIDString := string(leaver)
+	senderIDString := string(*leaver)
 	proto := gomatrixserverlib.ProtoEvent{
 		Type:     spec.MRoomMember,
 		SenderID: senderIDString,
diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go
index 74c62cd9e..c32e10d53 100644
--- a/roomserver/internal/perform/perform_upgrade.go
+++ b/roomserver/internal/perform/perform_upgrade.go
@@ -62,10 +62,13 @@ func (r *Upgrader) performRoomUpgrade(
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("Failed getting senderID for user")
 		return "", err
+	} else if senderID == nil {
+		util.GetLogger(ctx).WithField("userID", userID).WithField("roomID", *fullRoomID).Error("No senderID for user")
+		return "", fmt.Errorf("No sender ID for %s in %s", userID, *fullRoomID)
 	}
 
 	// 1. Check if the user is authorized to actually perform the upgrade (can send m.room.tombstone)
-	if !r.userIsAuthorized(ctx, senderID, roomID) {
+	if !r.userIsAuthorized(ctx, *senderID, roomID) {
 		return "", api.ErrNotAllowed{Err: fmt.Errorf("You don't have permission to upgrade the room, power level too low.")}
 	}
 
@@ -83,20 +86,20 @@ func (r *Upgrader) performRoomUpgrade(
 	}
 
 	// Make the tombstone event
-	tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, senderID, userID.Domain(), roomID, newRoomID)
+	tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, *senderID, userID.Domain(), roomID, newRoomID)
 	if pErr != nil {
 		return "", pErr
 	}
 
 	// Generate the initial events we need to send into the new room. This includes copied state events and bans
 	// as well as the power level events needed to set up the room
-	eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, senderID, roomID, roomVersion, tombstoneEvent)
+	eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, *senderID, roomID, roomVersion, tombstoneEvent)
 	if pErr != nil {
 		return "", pErr
 	}
 
 	// Send the setup events to the new room
-	if pErr = r.sendInitialEvents(ctx, evTime, senderID, userID.Domain(), newRoomID, roomVersion, eventsToMake); pErr != nil {
+	if pErr = r.sendInitialEvents(ctx, evTime, *senderID, userID.Domain(), newRoomID, roomVersion, eventsToMake); pErr != nil {
 		return "", pErr
 	}
 
@@ -111,17 +114,17 @@ func (r *Upgrader) performRoomUpgrade(
 	}
 
 	// If the old room had a canonical alias event, it should be deleted in the old room
-	if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, senderID, userID.Domain(), roomID); pErr != nil {
+	if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, *senderID, userID.Domain(), roomID); pErr != nil {
 		return "", pErr
 	}
 
 	// 4. Move local aliases to the new room
-	if pErr = moveLocalAliases(ctx, roomID, newRoomID, senderID, r.URSAPI); pErr != nil {
+	if pErr = moveLocalAliases(ctx, roomID, newRoomID, *senderID, r.URSAPI); pErr != nil {
 		return "", pErr
 	}
 
 	// 6. Restrict power levels in the old room
-	if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, senderID, userID.Domain(), roomID); pErr != nil {
+	if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, *senderID, userID.Domain(), roomID); pErr != nil {
 		return "", pErr
 	}
 
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 11e5564dc..0fe0f4f27 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -283,7 +283,7 @@ func (r *Queryer) QueryMembershipForUser(
 		return err
 	}
 
-	return r.QueryMembershipForSenderID(ctx, *roomID, senderID, response)
+	return r.QueryMembershipForSenderID(ctx, *roomID, *senderID, response)
 }
 
 // QueryMembershipAtEvent returns the known memberships at a given event.
@@ -1009,21 +1009,26 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, roomID spec.Ro
 	return verImpl.CheckRestrictedJoin(ctx, r.Cfg.Global.ServerName, &api.JoinRoomQuerier{Roomserver: r}, roomID, senderID)
 }
 
-func (r *Queryer) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) {
+func (r *Queryer) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
 	version, err := r.DB.GetRoomVersion(ctx, roomID.String())
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	switch version {
 	case gomatrixserverlib.RoomVersionPseudoIDs:
 		key, err := r.DB.SelectUserRoomPublicKey(ctx, userID, roomID)
 		if err != nil {
-			return "", err
+			return nil, err
+		} else if key == nil {
+			return nil, nil
+		} else {
+			senderID := spec.SenderID(spec.Base64Bytes(key).Encode())
+			return &senderID, nil
 		}
-		return spec.SenderID(spec.Base64Bytes(key).Encode()), nil
 	default:
-		return spec.SenderID(userID.String()), nil
+		senderID := spec.SenderID(userID.String())
+		return &senderID, nil
 	}
 }
 
diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go
index 16fb3efe1..ecbab706f 100644
--- a/setup/mscs/msc2836/msc2836_test.go
+++ b/setup/mscs/msc2836/msc2836_test.go
@@ -529,8 +529,9 @@ func (r *testRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spe
 	return spec.NewUserID(string(senderID), true)
 }
 
-func (r *testRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) {
-	return spec.SenderID(userID.String()), nil
+func (r *testRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	senderID := spec.SenderID(userID.String())
+	return &senderID, nil
 }
 
 func (r *testRoomserverAPI) QueryEventsByID(ctx context.Context, req *roomserver.QueryEventsByIDRequest, res *roomserver.QueryEventsByIDResponse) error {
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
index ce6846ca4..3c2308954 100644
--- a/syncapi/internal/history_visibility.go
+++ b/syncapi/internal/history_visibility.go
@@ -144,8 +144,8 @@ func ApplyHistoryVisibilityFilter(
 			return nil, err
 		}
 		senderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *user)
-		if err == nil {
-			if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(senderID)) {
+		if err == nil && senderID != nil {
+			if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(*senderID)) {
 				eventsFiltered = append(eventsFiltered, ev)
 				continue
 			}
diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go
index 746a324fa..69e64cc79 100644
--- a/syncapi/storage/shared/storage_consumer.go
+++ b/syncapi/storage/shared/storage_consumer.go
@@ -122,13 +122,13 @@ func (d *Database) StreamEventsToEvents(ctx context.Context, device *userapi.Dev
 				continue
 			}
 			deviceSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID)
-			if err != nil {
+			if err != nil || deviceSenderID == nil {
 				logrus.WithFields(logrus.Fields{
 					"event_id": out[i].EventID(),
 				}).WithError(err).Warnf("Failed to add transaction ID to event")
 				continue
 			}
-			if deviceSenderID == in[i].SenderID() && device.SessionID == in[i].TransactionID.SessionID {
+			if *deviceSenderID == in[i].SenderID() && device.SessionID == in[i].TransactionID.SessionID {
 				err := out[i].SetUnsignedField(
 					"transaction_id", in[i].TransactionID.TransactionID,
 				)
@@ -527,11 +527,11 @@ func getMembershipFromEvent(ctx context.Context, ev gomatrixserverlib.PDU, userI
 		return "", ""
 	}
 	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *fullUser)
-	if err != nil {
+	if err != nil || senderID == nil {
 		return "", ""
 	}
 
-	if ev.Type() != "m.room.member" || !ev.StateKeyEquals(string(senderID)) {
+	if ev.Type() != "m.room.member" || !ev.StateKeyEquals(string(*senderID)) {
 		return "", ""
 	}
 	membership, err := ev.Membership()
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index 1f866ef4d..a88b2129d 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -840,8 +840,11 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes
 		if err != nil {
 			logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID())
 			return nil, err
+		} else if localSender == nil {
+			logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID())
+			return nil, fmt.Errorf("no sender ID for user %s in %s", userID.String(), roomID.String())
 		}
-		if event.StateKey() != nil && *event.StateKey() == string(localSender) {
+		if event.StateKey() != nil && *event.StateKey() == string(*localSender) {
 			req.Notification.UserIsTarget = true
 		}
 	}

From 294eff8a7f42f11b3559ca941468c766358fcae1 Mon Sep 17 00:00:00 2001
From: maxberger <max@berger.name>
Date: Thu, 3 Aug 2023 09:26:42 +0200
Subject: [PATCH 06/57] Add ID in error messages for ApplicationServices
 (#3162)

This is to easier identify which service caused the error.

Feature is just improving logging, thus no tests added.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [X]  I have justified why this PR doesn't need tests
* [X] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `Maximilian Berger <max@berger.name>`

Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>
---
 appservice/query/query.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/appservice/query/query.go b/appservice/query/query.go
index ca8d7b3a3..5c736f379 100644
--- a/appservice/query/query.go
+++ b/appservice/query/query.go
@@ -217,7 +217,7 @@ func (a *AppServiceQueryAPI) Locations(
 		}
 
 		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
-			log.WithError(err).Error("unable to get 'locations' from application service")
+			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
 			continue
 		}
 
@@ -252,7 +252,7 @@ func (a *AppServiceQueryAPI) User(
 		}
 
 		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
-			log.WithError(err).Error("unable to get 'user' from application service")
+			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
 			continue
 		}
 
@@ -290,7 +290,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
 			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
-				log.WithError(err).Error("unable to get 'protocol' from application service")
+				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
 
@@ -320,7 +320,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
 			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
-				log.WithError(err).Error("unable to get 'protocol' from application service")
+				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
 			existing, ok := response[p]

From 35804f8493a7a51542b27ff98bc60814685d5020 Mon Sep 17 00:00:00 2001
From: Sam Wedgwood <28223854+swedgwood@users.noreply.github.com>
Date: Tue, 8 Aug 2023 14:20:05 +0100
Subject: [PATCH 07/57] Add config key for default room version (#3171)

This PR adds a config key `room_server.default_config_key` to set the
default room version for the room server.

Signed-off-by: `Sam Wedgwood <sam@wedgwood.dev>`
---
 clientapi/clientapi_test.go         |  6 +++++-
 clientapi/routing/capabilities.go   |  5 +++--
 clientapi/routing/createroom.go     |  2 +-
 clientapi/routing/routing.go        |  2 +-
 clientapi/routing/server_notices.go |  3 +--
 go.mod                              | 10 +++++-----
 go.sum                              | 26 ++++++++++----------------
 roomserver/api/api.go               |  7 +++++++
 roomserver/internal/api.go          |  6 ++++++
 roomserver/roomserver_test.go       |  3 +--
 roomserver/version/version.go       |  6 ------
 setup/config/config_roomserver.go   | 16 ++++++++++++++++
 12 files changed, 56 insertions(+), 36 deletions(-)

diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index ae14d271d..82ec9fea2 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -923,13 +923,17 @@ func TestCapabilities(t *testing.T) {
 		}
 	}
 
+	var tempRoomServerCfg config.RoomServer
+	tempRoomServerCfg.Defaults(config.DefaultOpts{})
+	defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion
+
 	expectedMap := map[string]interface{}{
 		"capabilities": map[string]interface{}{
 			"m.change_password": map[string]bool{
 				"enabled": true,
 			},
 			"m.room_versions": map[string]interface{}{
-				"default":   version.DefaultRoomVersion(),
+				"default":   defaultRoomVersion,
 				"available": versionsMap,
 			},
 		},
diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go
index fa50fa1aa..38e5bd467 100644
--- a/clientapi/routing/capabilities.go
+++ b/clientapi/routing/capabilities.go
@@ -17,6 +17,7 @@ package routing
 import (
 	"net/http"
 
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -24,7 +25,7 @@ import (
 
 // GetCapabilities returns information about the server's supported feature set
 // and other relevant capabilities to an authenticated user.
-func GetCapabilities() util.JSONResponse {
+func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	versionsMap := map[gomatrixserverlib.RoomVersion]string{}
 	for v, desc := range version.SupportedRoomVersions() {
 		if desc.Stable() {
@@ -40,7 +41,7 @@ func GetCapabilities() util.JSONResponse {
 				"enabled": true,
 			},
 			"m.room_versions": map[string]interface{}{
-				"default":   version.DefaultRoomVersion(),
+				"default":   rsAPI.DefaultRoomVersion(),
 				"available": versionsMap,
 			},
 		},
diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 320f236cb..47e3ba1c3 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -171,7 +171,7 @@ func createRoom(
 
 	// Clobber keys: creator, room_version
 
-	roomVersion := roomserverVersion.DefaultRoomVersion()
+	roomVersion := rsAPI.DefaultRoomVersion()
 	if createRequest.RoomVersion != "" {
 		candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion)
 		_, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion)
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 8cd207b7a..8b3ae5e1e 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -1256,7 +1256,7 @@ func Setup(
 			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
-			return GetCapabilities()
+			return GetCapabilities(rsAPI)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
 
diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go
index 66258a68a..1c5d693ca 100644
--- a/clientapi/routing/server_notices.go
+++ b/clientapi/routing/server_notices.go
@@ -28,7 +28,6 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/dendrite/roomserver/version"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -135,7 +134,7 @@ func SendServerNotice(
 
 	var (
 		roomID      string
-		roomVersion = version.DefaultRoomVersion()
+		roomVersion = rsAPI.DefaultRoomVersion()
 	)
 
 	// create a new room for the user
diff --git a/go.mod b/go.mod
index 74b17166c..ae37b2e7e 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230802090652-1b697d109d87
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac
 	github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
@@ -42,12 +42,12 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
 	go.uber.org/atomic v1.10.0
-	golang.org/x/crypto v0.11.0
+	golang.org/x/crypto v0.12.0
 	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db
 	golang.org/x/image v0.5.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
 	golang.org/x/sync v0.2.0
-	golang.org/x/term v0.10.0
+	golang.org/x/term v0.11.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
@@ -127,8 +127,8 @@ require (
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.6.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
diff --git a/go.sum b/go.sum
index 779c28618..e744d2d3f 100644
--- a/go.sum
+++ b/go.sum
@@ -113,7 +113,6 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -176,14 +175,12 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM=
 github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0=
 github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
@@ -210,10 +207,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230801102756-b66c2627dc08 h1:uy1mlUraKrEbUzZ3KrSQp/nLxMccVhIJM8mZSIbQzeA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230801102756-b66c2627dc08/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230802090652-1b697d109d87 h1:z0RFUknidOShRxkvjT3ovGCWnusyplu6OLjFHcbDYaE=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230802090652-1b697d109d87/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac h1:s4EZRNT6/TtGAzcO6yzL+UTv96vEeeaH6y2RrIOfsWw=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -246,7 +241,6 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
 github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
 github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA=
@@ -364,8 +358,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -428,19 +422,19 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 69997fc41..ad6a7122c 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -55,6 +55,11 @@ type RestrictedJoinAPI interface {
 	LocallyJoinedUsers(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomNID types.RoomNID) ([]gomatrixserverlib.PDU, error)
 }
 
+type DefaultRoomVersionAPI interface {
+	// Returns the default room version used.
+	DefaultRoomVersion() gomatrixserverlib.RoomVersion
+}
+
 // RoomserverInputAPI is used to write events to the room server.
 type RoomserverInternalAPI interface {
 	SyncRoomserverAPI
@@ -64,6 +69,7 @@ type RoomserverInternalAPI interface {
 	FederationRoomserverAPI
 	QuerySenderIDAPI
 	UserRoomPrivateKeyCreator
+	DefaultRoomVersionAPI
 
 	// needed to avoid chicken and egg scenario when setting up the
 	// interdependencies between the roomserver and other input APIs
@@ -210,6 +216,7 @@ type ClientRoomserverAPI interface {
 	QuerySenderIDAPI
 	UserRoomPrivateKeyCreator
 	QueryRoomHierarchyAPI
+	DefaultRoomVersionAPI
 	QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error
 	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
 	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index e8899a210..530147daa 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -61,6 +61,7 @@ type RoomserverInternalAPI struct {
 	OutputProducer         *producers.RoomEventProducer
 	PerspectiveServerNames []spec.ServerName
 	enableMetrics          bool
+	defaultRoomVersion     gomatrixserverlib.RoomVersion
 }
 
 func NewRoomserverAPI(
@@ -92,6 +93,7 @@ func NewRoomserverAPI(
 		Durable:                dendriteCfg.Global.JetStream.Durable("RoomserverInputConsumer"),
 		ServerACLs:             serverACLs,
 		enableMetrics:          enableMetrics,
+		defaultRoomVersion:     dendriteCfg.RoomServer.DefaultRoomVersion,
 		// perform-er structs + queryer struct get initialised when we have a federation sender to use
 	}
 	return a
@@ -218,6 +220,10 @@ func (r *RoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalA
 	r.asAPI = asAPI
 }
 
+func (r *RoomserverInternalAPI) DefaultRoomVersion() gomatrixserverlib.RoomVersion {
+	return r.defaultRoomVersion
+}
+
 func (r *RoomserverInternalAPI) IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) {
 	return r.Inviter.IsKnownRoom(ctx, roomID)
 }
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index 1626bf831..47626b30a 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -11,7 +11,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/stretchr/testify/assert"
 	"github.com/tidwall/gjson"
@@ -1060,7 +1059,7 @@ func TestUpgrade(t *testing.T) {
 				if err != nil {
 					t.Fatalf("upgrade userID is invalid")
 				}
-				newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, *userID, version.DefaultRoomVersion())
+				newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, *userID, rsAPI.DefaultRoomVersion())
 				if err != nil && tc.wantNewRoom {
 					t.Fatal(err)
 				}
diff --git a/roomserver/version/version.go b/roomserver/version/version.go
index 270d42897..e8bd890a0 100644
--- a/roomserver/version/version.go
+++ b/roomserver/version/version.go
@@ -20,12 +20,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// DefaultRoomVersion contains the room version that will, by
-// default, be used to create new rooms on this server.
-func DefaultRoomVersion() gomatrixserverlib.RoomVersion {
-	return gomatrixserverlib.RoomVersionV10
-}
-
 // RoomVersions returns a map of all known room versions to this
 // server.
 func RoomVersions() map[gomatrixserverlib.RoomVersion]gomatrixserverlib.IRoomVersion {
diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go
index 319c2419c..06e7757fb 100644
--- a/setup/config/config_roomserver.go
+++ b/setup/config/config_roomserver.go
@@ -1,12 +1,22 @@
 package config
 
+import (
+	"fmt"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	log "github.com/sirupsen/logrus"
+)
+
 type RoomServer struct {
 	Matrix *Global `yaml:"-"`
 
+	DefaultRoomVersion gomatrixserverlib.RoomVersion `yaml:"default_room_version,omitempty"`
+
 	Database DatabaseOptions `yaml:"database,omitempty"`
 }
 
 func (c *RoomServer) Defaults(opts DefaultOpts) {
+	c.DefaultRoomVersion = gomatrixserverlib.RoomVersionV10
 	if opts.Generate {
 		if !opts.SingleDatabase {
 			c.Database.ConnectionString = "file:roomserver.db"
@@ -18,4 +28,10 @@ func (c *RoomServer) Verify(configErrs *ConfigErrors) {
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
 	}
+
+	if !gomatrixserverlib.KnownRoomVersion(c.DefaultRoomVersion) {
+		configErrs.Add(fmt.Sprintf("invalid value for config key 'room_server.default_room_version': unsupported room version: %q", c.DefaultRoomVersion))
+	} else if !gomatrixserverlib.StableRoomVersion(c.DefaultRoomVersion) {
+		log.Warnf("WARNING: Provided default room version %q is unstable", c.DefaultRoomVersion)
+	}
 }

From fa6c7ba45671c8fbf13cb7ba456355a04941b535 Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Fri, 11 Aug 2023 14:29:48 +0000
Subject: [PATCH 08/57] Update pinecone to use new quic version (#3174)

---
 .../monolith/monolith.go                      |  2 +-
 go.mod                                        | 25 ++++-----
 go.sum                                        | 54 +++++++++----------
 3 files changed, 37 insertions(+), 44 deletions(-)

diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go
index abeea19d4..41af568a6 100644
--- a/cmd/dendrite-demo-pinecone/monolith/monolith.go
+++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go
@@ -221,8 +221,8 @@ func (p *P2PMonolith) closeAllResources() {
 	p.httpServerMu.Lock()
 	if p.httpServer != nil {
 		_ = p.httpServer.Shutdown(context.Background())
-		p.httpServerMu.Unlock()
 	}
+	p.httpServerMu.Unlock()
 
 	select {
 	case p.stopHandlingEvents <- true:
diff --git a/go.mod b/go.mod
index ae37b2e7e..915e813a6 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac
-	github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a
+	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/nats-io/nats-server/v2 v2.9.19
@@ -43,10 +43,10 @@ require (
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
 	go.uber.org/atomic v1.10.0
 	golang.org/x/crypto v0.12.0
-	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db
+	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
 	golang.org/x/image v0.5.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
-	golang.org/x/sync v0.2.0
+	golang.org/x/sync v0.3.0
 	golang.org/x/term v0.11.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
@@ -82,14 +82,14 @@ require (
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
+	github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
@@ -107,30 +107,27 @@ require (
 	github.com/nats-io/jwt/v2 v2.4.1 // indirect
 	github.com/nats-io/nkeys v0.4.4 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/onsi/ginkgo/v2 v2.3.0 // indirect
-	github.com/onsi/gomega v1.22.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/quic-go/qtls-go1-18 v0.2.0 // indirect
-	github.com/quic-go/qtls-go1-19 v0.2.0 // indirect
-	github.com/quic-go/qtls-go1-20 v0.1.0 // indirect
-	github.com/quic-go/quic-go v0.32.0 // indirect
+	github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
+	github.com/quic-go/quic-go v0.37.4 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/zerolog v1.29.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/tools v0.12.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index e744d2d3f..6c7e48dbf 100644
--- a/go.sum
+++ b/go.sum
@@ -113,6 +113,7 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -120,8 +121,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
@@ -160,8 +161,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c=
+github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -209,8 +210,8 @@ github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac h1:s4EZRNT6/TtGAzcO6yzL+UTv96vEeeaH6y2RrIOfsWw=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
-github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A=
-github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ=
+github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
+github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66/go.mod h1:iBI1foelCqA09JJgPV0FYz4qA5dUXYOxMi57FxKBdd4=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
@@ -256,10 +257,9 @@ github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHG
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/onsi/ginkgo/v2 v2.3.0 h1:kUMoxMoQG3ogk/QWyKh3zibV7BKZ+xBpWil1cTylVqc=
-github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
-github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI=
-github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
+github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
+github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
+github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec=
@@ -284,14 +284,10 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
-github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U=
-github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc=
-github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk=
-github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
-github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI=
-github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
-github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA=
-github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo=
+github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
+github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
+github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4=
+github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
@@ -315,7 +311,7 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ -365,8 +361,8 @@ golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o=
-golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U=
+golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -380,8 +376,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -390,16 +386,16 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
-golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -449,8 +445,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
+golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From 9a12420428f1832c76fc0c84ad85db200e261ecb Mon Sep 17 00:00:00 2001
From: Sam Wedgwood <28223854+swedgwood@users.noreply.github.com>
Date: Tue, 15 Aug 2023 12:37:04 +0100
Subject: [PATCH 09/57] [pseudoID] More pseudo ID fixes (#3167)

Signed-off-by: `Sam Wedgwood <sam@wedgwood.dev>`
---
 clientapi/routing/joined_rooms.go             |  31 +++--
 clientapi/routing/profile.go                  |  21 ++-
 clientapi/routing/sendevent.go                |  13 +-
 clientapi/routing/server_notices.go           |  42 +++---
 federationapi/consumers/keychange.go          |  41 ++++--
 federationapi/consumers/presence.go           |  20 ++-
 federationapi/federationapi_test.go           |  22 +--
 roomserver/api/api.go                         |  40 +++---
 roomserver/api/query.go                       |  28 +---
 roomserver/internal/perform/perform_admin.go  |   4 +-
 roomserver/internal/query/query.go            | 128 +++++++++++-------
 roomserver/storage/interface.go               |   2 +-
 .../storage/postgres/user_room_keys_table.go  |  35 ++++-
 roomserver/storage/shared/storage.go          |  66 ++++++++-
 .../storage/sqlite3/user_room_keys_table.go   |  35 ++++-
 roomserver/storage/tables/interface.go        |   2 +
 syncapi/internal/history_visibility.go        |  59 ++++----
 syncapi/internal/keychange_test.go            |   4 +-
 syncapi/routing/context.go                    |   6 +-
 syncapi/routing/getevent.go                   |  43 ++++--
 syncapi/routing/messages.go                   |  13 +-
 syncapi/routing/relations.go                  |  33 +++--
 syncapi/streams/stream_pdu.go                 |   7 +-
 syncapi/syncapi_test.go                       |  14 +-
 24 files changed, 472 insertions(+), 237 deletions(-)

diff --git a/clientapi/routing/joined_rooms.go b/clientapi/routing/joined_rooms.go
index f664183f8..3fe0d8b4d 100644
--- a/clientapi/routing/joined_rooms.go
+++ b/clientapi/routing/joined_rooms.go
@@ -33,23 +33,36 @@ func GetJoinedRooms(
 	device *userapi.Device,
 	rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
-	var res api.QueryRoomsForUserResponse
-	err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
-		UserID:         device.UserID,
-		WantMembership: "join",
-	}, &res)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("Invalid device user ID")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
+	rooms, err := rsAPI.QueryRoomsForUser(req.Context(), *deviceUserID, "join")
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
+			JSON: spec.Unknown("internal server error"),
 		}
 	}
-	if res.RoomIDs == nil {
-		res.RoomIDs = []string{}
+
+	var roomIDStrs []string
+	if rooms == nil {
+		roomIDStrs = []string{}
+	} else {
+		roomIDStrs = make([]string, len(rooms))
+		for i, roomID := range rooms {
+			roomIDStrs[i] = roomID.String()
+		}
 	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: getJoinedRoomsResponse{res.RoomIDs},
+		JSON: getJoinedRoomsResponse{roomIDStrs},
 	}
 }
diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go
index 66b58507e..564cd588a 100644
--- a/clientapi/routing/profile.go
+++ b/clientapi/routing/profile.go
@@ -251,11 +251,15 @@ func updateProfile(
 	profile *authtypes.Profile,
 	userID string, evTime time.Time,
 ) (util.JSONResponse, error) {
-	var res api.QueryRoomsForUserResponse
-	err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-		UserID:         device.UserID,
-		WantMembership: "join",
-	}, &res)
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}, err
+	}
+
+	rooms, err := rsAPI.QueryRoomsForUser(ctx, *deviceUserID, "join")
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("QueryRoomsForUser failed")
 		return util.JSONResponse{
@@ -264,6 +268,11 @@ func updateProfile(
 		}, err
 	}
 
+	roomIDStrs := make([]string, len(rooms))
+	for i, room := range rooms {
+		roomIDStrs[i] = room.String()
+	}
+
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
@@ -274,7 +283,7 @@ func updateProfile(
 	}
 
 	events, err := buildMembershipEvents(
-		ctx, res.RoomIDs, *profile, userID, evTime, rsAPI,
+		ctx, roomIDStrs, *profile, userID, evTime, rsAPI,
 	)
 	switch e := err.(type) {
 	case nil:
diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index 172001714..a167a5a77 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -316,10 +316,17 @@ func generateSendEvent(
 		}
 	}
 	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID)
-	if err != nil || senderID == nil {
+	if err != nil {
 		return nil, &util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: spec.NotFound("Unable to find senderID for user"),
+			Code: http.StatusInternalServerError,
+			JSON: spec.NotFound("internal server error"),
+		}
+	} else if senderID == nil {
+		// TODO: is it always the case that lack of a sender ID means they're not joined?
+		//       And should this logic be deferred to the roomserver somehow?
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("not joined to room"),
 		}
 	}
 
diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go
index 1c5d693ca..5deb559df 100644
--- a/clientapi/routing/server_notices.go
+++ b/clientapi/routing/server_notices.go
@@ -94,34 +94,42 @@ func SendServerNotice(
 		}
 	}
 
+	userID, err := spec.NewUserID(r.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid user ID"),
+		}
+	}
+
 	// get rooms for specified user
-	allUserRooms := []string{}
-	userRooms := api.QueryRoomsForUserResponse{}
+	allUserRooms := []spec.RoomID{}
 	// Get rooms the user is either joined, invited or has left.
 	for _, membership := range []string{"join", "invite", "leave"} {
-		if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-			UserID:         r.UserID,
-			WantMembership: membership,
-		}, &userRooms); err != nil {
+		userRooms, queryErr := rsAPI.QueryRoomsForUser(ctx, *userID, membership)
+		if queryErr != nil {
 			return util.ErrorResponse(err)
 		}
-		allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
+		allUserRooms = append(allUserRooms, userRooms...)
 	}
 
 	// get rooms of the sender
-	senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName)
-	senderRooms := api.QueryRoomsForUserResponse{}
-	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-		UserID:         senderUserID,
-		WantMembership: "join",
-	}, &senderRooms); err != nil {
+	senderUserID, err := spec.NewUserID(fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName), true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+	senderRooms, err := rsAPI.QueryRoomsForUser(ctx, *senderUserID, "join")
+	if err != nil {
 		return util.ErrorResponse(err)
 	}
 
 	// check if we have rooms in common
-	commonRooms := []string{}
+	commonRooms := []spec.RoomID{}
 	for _, userRoomID := range allUserRooms {
-		for _, senderRoomID := range senderRooms.RoomIDs {
+		for _, senderRoomID := range senderRooms {
 			if userRoomID == senderRoomID {
 				commonRooms = append(commonRooms, senderRoomID)
 			}
@@ -139,7 +147,7 @@ func SendServerNotice(
 
 	// create a new room for the user
 	if len(commonRooms) == 0 {
-		powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID)
+		powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID.String())
 		powerLevelContent.Users[r.UserID] = -10 // taken from Synapse
 		pl, err := json.Marshal(powerLevelContent)
 		if err != nil {
@@ -195,7 +203,7 @@ func SendServerNotice(
 			}
 		}
 
-		roomID = commonRooms[0]
+		roomID = commonRooms[0].String()
 		membershipRes := api.QueryMembershipForUserResponse{}
 		err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: *deviceUserID, RoomID: roomID}, &membershipRes)
 		if err != nil {
diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go
index 3fdc835bb..6210bddb6 100644
--- a/federationapi/consumers/keychange.go
+++ b/federationapi/consumers/keychange.go
@@ -117,19 +117,27 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		return true
 	}
 
-	var queryRes roomserverAPI.QueryRoomsForUserResponse
-	err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{
-		UserID:         m.UserID,
-		WantMembership: "join",
-	}, &queryRes)
+	userID, err := spec.NewUserID(m.UserID, true)
+	if err != nil {
+		sentry.CaptureException(err)
+		logger.WithError(err).Error("invalid user ID")
+		return true
+	}
+
+	roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *userID, "join")
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
 
+	roomIDStrs := make([]string, len(roomIDs))
+	for i, room := range roomIDs {
+		roomIDStrs[i] = room.String()
+	}
+
 	// send this key change to all servers who share rooms with this user.
-	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
+	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true)
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in")
@@ -179,18 +187,27 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 	}
 	logger := logrus.WithField("user_id", output.UserID)
 
-	var queryRes roomserverAPI.QueryRoomsForUserResponse
-	err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{
-		UserID:         output.UserID,
-		WantMembership: "join",
-	}, &queryRes)
+	outputUserID, err := spec.NewUserID(output.UserID, true)
+	if err != nil {
+		sentry.CaptureException(err)
+		logrus.WithError(err).Errorf("invalid user ID")
+		return true
+	}
+
+	rooms, err := t.rsAPI.QueryRoomsForUser(t.ctx, *outputUserID, "join")
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user")
 		return true
 	}
+
+	roomIDStrs := make([]string, len(rooms))
+	for i, room := range rooms {
+		roomIDStrs[i] = room.String()
+	}
+
 	// send this key change to all servers who share rooms with this user.
-	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
+	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true)
 	if err != nil {
 		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in")
diff --git a/federationapi/consumers/presence.go b/federationapi/consumers/presence.go
index e751b65d4..dd100bc08 100644
--- a/federationapi/consumers/presence.go
+++ b/federationapi/consumers/presence.go
@@ -29,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 )
@@ -94,16 +95,23 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
 		return true
 	}
 
-	var queryRes roomserverAPI.QueryRoomsForUserResponse
-	err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{
-		UserID:         userID,
-		WantMembership: "join",
-	}, &queryRes)
+	parsedUserID, err := spec.NewUserID(userID, true)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).WithField("user_id", userID).Error("invalid user ID")
+		return true
+	}
+
+	roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *parsedUserID, "join")
 	if err != nil {
 		log.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
 
+	roomIDStrs := make([]string, len(roomIDs))
+	for i, roomID := range roomIDs {
+		roomIDStrs[i] = roomID.String()
+	}
+
 	presence := msg.Header.Get("presence")
 
 	ts, err := strconv.Atoi(msg.Header.Get("last_active_ts"))
@@ -112,7 +120,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
 	}
 
 	// send this presence to all servers who share rooms with this user.
-	joined, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
+	joined, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true)
 	if err != nil {
 		log.WithError(err).Error("failed to get joined hosts")
 		return true
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index c426eb67d..4c2a99bbc 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -33,7 +33,7 @@ import (
 type fedRoomserverAPI struct {
 	rsapi.FederationRoomserverAPI
 	inputRoomEvents   func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse)
-	queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error
+	queryRoomsForUser func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error)
 }
 
 func (f *fedRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
@@ -54,11 +54,11 @@ func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.Input
 }
 
 // keychange consumer calls this
-func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
+func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
 	if f.queryRoomsForUser == nil {
-		return nil
+		return nil, nil
 	}
-	return f.queryRoomsForUser(ctx, req, res)
+	return f.queryRoomsForUser(ctx, userID, desiredMembership)
 }
 
 // TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
@@ -199,18 +199,22 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 	fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID)
 	room := test.NewRoom(t, creator)
 
+	roomID, err := spec.NewRoomID(room.ID)
+	if err != nil {
+		t.Fatalf("Invalid room ID: %q", roomID)
+	}
+
 	rsapi := &fedRoomserverAPI{
 		inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
 			if req.Asynchronous {
 				t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous")
 			}
 		},
-		queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
-			if req.UserID == joiningUser.ID && req.WantMembership == "join" {
-				res.RoomIDs = []string{room.ID}
-				return nil
+		queryRoomsForUser: func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
+			if userID.String() == joiningUser.ID && desiredMembership == "join" {
+				return []spec.RoomID{*roomID}, nil
 			}
-			return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req)
+			return nil, fmt.Errorf("unexpected queryRoomsForUser: %v, %v", userID, desiredMembership)
 		},
 	}
 	fc := &fedClient{
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index ad6a7122c..ef5bc3d17 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -141,11 +141,28 @@ type QueryRoomHierarchyAPI interface {
 	QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *RoomHierarchyWalker, error)
 }
 
+type QueryMembershipAPI interface {
+	QueryMembershipForSenderID(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, res *QueryMembershipForUserResponse) error
+	QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error
+	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
+	QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error)
+
+	// QueryMembershipAtEvent queries the memberships at the given events.
+	// Returns a map from eventID to *types.HeaderedEvent of membership events.
+	QueryMembershipAtEvent(
+		ctx context.Context,
+		roomID spec.RoomID,
+		eventIDs []string,
+		senderID spec.SenderID,
+	) (map[string]*types.HeaderedEvent, error)
+}
+
 // API functions required by the syncapi
 type SyncRoomserverAPI interface {
 	QueryLatestEventsAndStateAPI
 	QueryBulkStateContentAPI
 	QuerySenderIDAPI
+	QueryMembershipAPI
 	// QuerySharedUsers returns a list of users who share at least 1 room in common with the given user.
 	QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error
 	// QueryEventsByID queries a list of events by event ID for one room. If no room is specified, it will try to determine
@@ -155,12 +172,6 @@ type SyncRoomserverAPI interface {
 		req *QueryEventsByIDRequest,
 		res *QueryEventsByIDResponse,
 	) error
-	// Query the membership event for an user for a room.
-	QueryMembershipForUser(
-		ctx context.Context,
-		req *QueryMembershipForUserRequest,
-		res *QueryMembershipForUserResponse,
-	) error
 
 	// Query the state after a list of events in a room from the room server.
 	QueryStateAfterEvents(
@@ -175,14 +186,6 @@ type SyncRoomserverAPI interface {
 		req *PerformBackfillRequest,
 		res *PerformBackfillResponse,
 	) error
-
-	// QueryMembershipAtEvent queries the memberships at the given events.
-	// Returns a map from eventID to a slice of types.HeaderedEvent.
-	QueryMembershipAtEvent(
-		ctx context.Context,
-		request *QueryMembershipAtEventRequest,
-		response *QueryMembershipAtEventResponse,
-	) error
 }
 
 type AppserviceRoomserverAPI interface {
@@ -219,7 +222,7 @@ type ClientRoomserverAPI interface {
 	DefaultRoomVersionAPI
 	QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error
 	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
-	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+	QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error)
 	QueryStateAfterEvents(ctx context.Context, req *QueryStateAfterEventsRequest, res *QueryStateAfterEventsResponse) error
 	// QueryKnownUsers returns a list of users that we know about from our joined rooms.
 	QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error
@@ -278,15 +281,12 @@ type FederationRoomserverAPI interface {
 	QueryBulkStateContentAPI
 	QuerySenderIDAPI
 	QueryRoomHierarchyAPI
+	QueryMembershipAPI
 	UserRoomPrivateKeyCreator
 	AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error)
 	SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error)
 	// QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs.
 	QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error
-	QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error
-	QueryMembershipForSenderID(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, res *QueryMembershipForUserResponse) error
-	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
-	QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error)
 	GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error
 	// QueryEventsByID queries a list of events by event ID for one room. If no room is specified, it will try to determine
 	// which room to use by querying the first events roomID.
@@ -300,7 +300,7 @@ type FederationRoomserverAPI interface {
 	QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error
 	// Query whether a server is allowed to see an event
 	QueryServerAllowedToSeeEvent(ctx context.Context, serverName spec.ServerName, eventID string, roomID string) (allowed bool, err error)
-	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+	QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error)
 	QueryRestrictedJoinAllowed(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (string, error)
 	PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error
 	HandleInvite(ctx context.Context, event *types.HeaderedEvent) error
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index 57bac2df9..893d5dccf 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -132,6 +132,8 @@ type QueryMembershipForUserResponse struct {
 	// True if the user asked to forget this room.
 	IsRoomForgotten bool `json:"is_room_forgotten"`
 	RoomExists      bool `json:"room_exists"`
+	// The sender ID of the user in the room, if it exists
+	SenderID *spec.SenderID
 }
 
 // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom
@@ -289,16 +291,6 @@ type QuerySharedUsersResponse struct {
 	UserIDsToCount map[string]int
 }
 
-type QueryRoomsForUserRequest struct {
-	UserID string
-	// The desired membership of the user. If this is the empty string then no rooms are returned.
-	WantMembership string
-}
-
-type QueryRoomsForUserResponse struct {
-	RoomIDs []string
-}
-
 type QueryBulkStateContentRequest struct {
 	// Returns state events in these rooms
 	RoomIDs []string
@@ -414,22 +406,6 @@ func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-// QueryMembershipAtEventRequest requests the membership event for a user
-// for a list of eventIDs.
-type QueryMembershipAtEventRequest struct {
-	RoomID   string
-	EventIDs []string
-	UserID   string
-}
-
-// QueryMembershipAtEventResponse is the response to QueryMembershipAtEventRequest.
-type QueryMembershipAtEventResponse struct {
-	// Membership is a map from eventID to membership event. Events that
-	// do not have known state will return a nil event, resulting in a "leave" membership
-	// when calculating history visibility.
-	Membership map[string]*types.HeaderedEvent `json:"membership"`
-}
-
 // QueryLeftUsersRequest is a request to calculate users that we (the server) don't share a
 // a room with anymore. This is used to cleanup stale device list entries, where we would
 // otherwise keep on trying to get device lists.
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index 2888067b4..ae203854b 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -161,12 +161,12 @@ func (r *Admin) PerformAdminEvacuateUser(
 		return nil, fmt.Errorf("can only evacuate local users using this endpoint")
 	}
 
-	roomIDs, err := r.DB.GetRoomsByMembership(ctx, userID, spec.Join)
+	roomIDs, err := r.DB.GetRoomsByMembership(ctx, *fullUserID, spec.Join)
 	if err != nil {
 		return nil, err
 	}
 
-	inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, userID, spec.Invite)
+	inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, *fullUserID, spec.Invite)
 	if err != nil && err != sql.ErrNoRows {
 		return nil, err
 	}
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 0fe0f4f27..f87a3f7ed 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -230,6 +230,33 @@ func (r *Queryer) QueryMembershipForSenderID(
 	senderID spec.SenderID,
 	response *api.QueryMembershipForUserResponse,
 ) error {
+	return r.queryMembershipForOptionalSenderID(ctx, roomID, &senderID, response)
+}
+
+// QueryMembershipForUser implements api.RoomserverInternalAPI
+func (r *Queryer) QueryMembershipForUser(
+	ctx context.Context,
+	request *api.QueryMembershipForUserRequest,
+	response *api.QueryMembershipForUserResponse,
+) error {
+	roomID, err := spec.NewRoomID(request.RoomID)
+	if err != nil {
+		return err
+	}
+	senderID, err := r.QuerySenderIDForUser(ctx, *roomID, request.UserID)
+	if err != nil {
+		return err
+	}
+
+	return r.queryMembershipForOptionalSenderID(ctx, *roomID, senderID, response)
+}
+
+// Query membership information for provided sender ID and room ID
+//
+// If sender ID is nil, then act as if the provided sender is not a member of the room.
+func (r *Queryer) queryMembershipForOptionalSenderID(ctx context.Context, roomID spec.RoomID, senderID *spec.SenderID, response *api.QueryMembershipForUserResponse) error {
+	response.SenderID = senderID
+
 	info, err := r.DB.RoomInfo(ctx, roomID.String())
 	if err != nil {
 		return err
@@ -240,7 +267,11 @@ func (r *Queryer) QueryMembershipForSenderID(
 	}
 	response.RoomExists = true
 
-	membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, senderID)
+	if senderID == nil {
+		return nil
+	}
+
+	membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, *senderID)
 	if err != nil {
 		return err
 	}
@@ -268,70 +299,55 @@ func (r *Queryer) QueryMembershipForSenderID(
 	return err
 }
 
-// QueryMembershipForUser implements api.RoomserverInternalAPI
-func (r *Queryer) QueryMembershipForUser(
-	ctx context.Context,
-	request *api.QueryMembershipForUserRequest,
-	response *api.QueryMembershipForUserResponse,
-) error {
-	roomID, err := spec.NewRoomID(request.RoomID)
-	if err != nil {
-		return err
-	}
-	senderID, err := r.QuerySenderIDForUser(ctx, *roomID, request.UserID)
-	if err != nil {
-		return err
-	}
-
-	return r.QueryMembershipForSenderID(ctx, *roomID, *senderID, response)
-}
-
 // QueryMembershipAtEvent returns the known memberships at a given event.
 // If the state before an event is not known, an empty list will be returned
 // for that event instead.
+//
+// Returned map from eventID to membership event. Events that
+// do not have known state will return a nil event, resulting in a "leave" membership
+// when calculating history visibility.
 func (r *Queryer) QueryMembershipAtEvent(
 	ctx context.Context,
-	request *api.QueryMembershipAtEventRequest,
-	response *api.QueryMembershipAtEventResponse,
-) error {
-	response.Membership = make(map[string]*types.HeaderedEvent)
-
-	info, err := r.DB.RoomInfo(ctx, request.RoomID)
+	roomID spec.RoomID,
+	eventIDs []string,
+	senderID spec.SenderID,
+) (map[string]*types.HeaderedEvent, error) {
+	info, err := r.DB.RoomInfo(ctx, roomID.String())
 	if err != nil {
-		return fmt.Errorf("unable to get roomInfo: %w", err)
+		return nil, fmt.Errorf("unable to get roomInfo: %w", err)
 	}
 	if info == nil {
-		return fmt.Errorf("no roomInfo found")
+		return nil, fmt.Errorf("no roomInfo found")
 	}
 
 	// get the users stateKeyNID
-	stateKeyNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{request.UserID})
+	stateKeyNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{string(senderID)})
 	if err != nil {
-		return fmt.Errorf("unable to get stateKeyNIDs for %s: %w", request.UserID, err)
+		return nil, fmt.Errorf("unable to get stateKeyNIDs for %s: %w", senderID, err)
 	}
-	if _, ok := stateKeyNIDs[request.UserID]; !ok {
-		return fmt.Errorf("requested stateKeyNID for %s was not found", request.UserID)
+	if _, ok := stateKeyNIDs[string(senderID)]; !ok {
+		return nil, fmt.Errorf("requested stateKeyNID for %s was not found", senderID)
 	}
 
-	response.Membership, err = r.DB.GetMembershipForHistoryVisibility(ctx, stateKeyNIDs[request.UserID], info, request.EventIDs...)
+	eventIDMembershipMap, err := r.DB.GetMembershipForHistoryVisibility(ctx, stateKeyNIDs[string(senderID)], info, eventIDs...)
 	switch err {
 	case nil:
-		return nil
+		return eventIDMembershipMap, nil
 	case tables.OptimisationNotSupportedError: // fallthrough, slow way of getting the membership events for each event
 	default:
-		return err
+		return eventIDMembershipMap, err
 	}
 
-	response.Membership = make(map[string]*types.HeaderedEvent)
-	stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, nil, request.EventIDs, stateKeyNIDs[request.UserID], r)
+	eventIDMembershipMap = make(map[string]*types.HeaderedEvent)
+	stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, nil, eventIDs, stateKeyNIDs[string(senderID)], r)
 	if err != nil {
-		return fmt.Errorf("unable to get state before event: %w", err)
+		return eventIDMembershipMap, fmt.Errorf("unable to get state before event: %w", err)
 	}
 
 	// If we only have one or less state entries, we can short circuit the below
 	// loop and avoid hitting the database
 	allStateEventNIDs := make(map[types.EventNID]types.StateEntry)
-	for _, eventID := range request.EventIDs {
+	for _, eventID := range eventIDs {
 		stateEntry := stateEntries[eventID]
 		for _, s := range stateEntry {
 			allStateEventNIDs[s.EventNID] = s
@@ -344,10 +360,10 @@ func (r *Queryer) QueryMembershipAtEvent(
 	}
 
 	var memberships []types.Event
-	for _, eventID := range request.EventIDs {
+	for _, eventID := range eventIDs {
 		stateEntry, ok := stateEntries[eventID]
 		if !ok || len(stateEntry) == 0 {
-			response.Membership[eventID] = nil
+			eventIDMembershipMap[eventID] = nil
 			continue
 		}
 
@@ -361,7 +377,7 @@ func (r *Queryer) QueryMembershipAtEvent(
 			memberships, err = helpers.GetMembershipsAtState(ctx, r.DB, info, stateEntry, false)
 		}
 		if err != nil {
-			return fmt.Errorf("unable to get memberships at state: %w", err)
+			return eventIDMembershipMap, fmt.Errorf("unable to get memberships at state: %w", err)
 		}
 
 		// Iterate over all membership events we got. Given we only query the membership for
@@ -369,13 +385,13 @@ func (r *Queryer) QueryMembershipAtEvent(
 		// a given event, overwrite any other existing membership events.
 		for i := range memberships {
 			ev := memberships[i]
-			if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(request.UserID) {
-				response.Membership[eventID] = &types.HeaderedEvent{PDU: ev.PDU}
+			if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(senderID)) {
+				eventIDMembershipMap[eventID] = &types.HeaderedEvent{PDU: ev.PDU}
 			}
 		}
 	}
 
-	return nil
+	return eventIDMembershipMap, nil
 }
 
 // QueryMembershipsForRoom implements api.RoomserverInternalAPI
@@ -830,13 +846,20 @@ func (r *Queryer) QueryCurrentState(ctx context.Context, req *api.QueryCurrentSt
 	return nil
 }
 
-func (r *Queryer) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error {
-	roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, req.WantMembership)
+func (r *Queryer) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
+	roomIDStrs, err := r.DB.GetRoomsByMembership(ctx, userID, desiredMembership)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	res.RoomIDs = roomIDs
-	return nil
+	roomIDs := make([]spec.RoomID, len(roomIDStrs))
+	for i, roomIDStr := range roomIDStrs {
+		roomID, err := spec.NewRoomID(roomIDStr)
+		if err != nil {
+			return nil, err
+		}
+		roomIDs[i] = *roomID
+	}
+	return roomIDs, nil
 }
 
 func (r *Queryer) QueryKnownUsers(ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse) error {
@@ -879,7 +902,12 @@ func (r *Queryer) QueryLeftUsers(ctx context.Context, req *api.QueryLeftUsersReq
 }
 
 func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error {
-	roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, "join")
+	parsedUserID, err := spec.NewUserID(req.UserID, true)
+	if err != nil {
+		return err
+	}
+
+	roomIDs, err := r.DB.GetRoomsByMembership(ctx, *parsedUserID, "join")
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index e9b4609ec..0638252b2 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -158,7 +158,7 @@ type Database interface {
 	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*types.HeaderedEvent, error)
 	GetStateEventsWithEventType(ctx context.Context, roomID, evType string) ([]*types.HeaderedEvent, error)
 	// GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key).
-	GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error)
+	GetRoomsByMembership(ctx context.Context, userID spec.UserID, membership string) ([]string, error)
 	// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
 	// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
 	GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go
index 202b0abc1..217ee957f 100644
--- a/roomserver/storage/postgres/user_room_keys_table.go
+++ b/roomserver/storage/postgres/user_room_keys_table.go
@@ -56,12 +56,15 @@ const selectUserRoomPublicKeySQL = `SELECT pseudo_id_pub_key FROM roomserver_use
 
 const selectUserNIDsSQL = `SELECT user_nid, room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE room_nid = ANY($1) AND pseudo_id_pub_key = ANY($2)`
 
+const selectAllUserRoomPublicKeyForUserSQL = `SELECT room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1`
+
 type userRoomKeysStatements struct {
-	insertUserRoomPrivateKeyStmt *sql.Stmt
-	insertUserRoomPublicKeyStmt  *sql.Stmt
-	selectUserRoomKeyStmt        *sql.Stmt
-	selectUserRoomPublicKeyStmt  *sql.Stmt
-	selectUserNIDsStmt           *sql.Stmt
+	insertUserRoomPrivateKeyStmt       *sql.Stmt
+	insertUserRoomPublicKeyStmt        *sql.Stmt
+	selectUserRoomKeyStmt              *sql.Stmt
+	selectUserRoomPublicKeyStmt        *sql.Stmt
+	selectUserNIDsStmt                 *sql.Stmt
+	selectAllUserRoomPublicKeysForUser *sql.Stmt
 }
 
 func CreateUserRoomKeysTable(db *sql.DB) error {
@@ -77,6 +80,7 @@ func PrepareUserRoomKeysTable(db *sql.DB) (tables.UserRoomKeys, error) {
 		{&s.selectUserRoomKeyStmt, selectUserRoomKeySQL},
 		{&s.selectUserRoomPublicKeyStmt, selectUserRoomPublicKeySQL},
 		{&s.selectUserNIDsStmt, selectUserNIDsSQL},
+		{&s.selectAllUserRoomPublicKeysForUser, selectAllUserRoomPublicKeyForUserSQL},
 	}.Prepare(db)
 }
 
@@ -150,3 +154,24 @@ func (s *userRoomKeysStatements) BulkSelectUserNIDs(ctx context.Context, txn *sq
 	}
 	return result, rows.Err()
 }
+
+func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) {
+	stmt := sqlutil.TxStmtContext(ctx, txn, s.selectAllUserRoomPublicKeysForUser)
+
+	rows, err := stmt.QueryContext(ctx, userNID)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+
+	resultMap := make(map[types.RoomNID]ed25519.PublicKey)
+
+	var roomNID types.RoomNID
+	var pubkey ed25519.PublicKey
+	for rows.Next() {
+		if err = rows.Scan(&roomNID, &pubkey); err != nil {
+			return nil, err
+		}
+		resultMap[roomNID] = pubkey
+	}
+	return resultMap, err
+}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 3c8b69c32..b09c5afbd 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1347,7 +1347,7 @@ func (d *Database) GetStateEventsWithEventType(ctx context.Context, roomID, evTy
 }
 
 // GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key).
-func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) {
+func (d *Database) GetRoomsByMembership(ctx context.Context, userID spec.UserID, membership string) ([]string, error) {
 	var membershipState tables.MembershipState
 	switch membership {
 	case "join":
@@ -1361,17 +1361,73 @@ func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership
 	default:
 		return nil, fmt.Errorf("GetRoomsByMembership: invalid membership %s", membership)
 	}
-	stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID)
+
+	// Convert provided user ID to NID
+	userNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID.String())
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, nil
+		} else {
+			return nil, fmt.Errorf("SelectEventStateKeyNID: cannot map user ID to state key NIDs: %w", err)
 		}
-		return nil, fmt.Errorf("GetRoomsByMembership: cannot map user ID to state key NID: %w", err)
 	}
-	roomNIDs, err := d.MembershipTable.SelectRoomsWithMembership(ctx, nil, stateKeyNID, membershipState)
+
+	// Use this NID to fetch all associated room keys (for pseudo ID rooms)
+	roomKeyMap, err := d.UserRoomKeyTable.SelectAllPublicKeysForUser(ctx, nil, userNID)
 	if err != nil {
-		return nil, fmt.Errorf("GetRoomsByMembership: failed to SelectRoomsWithMembership: %w", err)
+		if err == sql.ErrNoRows {
+			roomKeyMap = map[types.RoomNID]ed25519.PublicKey{}
+		} else {
+			return nil, fmt.Errorf("SelectAllPublicKeysForUser: could not select user room public keys for user: %w", err)
+		}
 	}
+
+	var eventStateKeyNIDs []types.EventStateKeyNID
+
+	// If there are room keys (i.e. this user is in pseudo ID rooms), then gather the appropriate NIDs
+	if len(roomKeyMap) != 0 {
+		// Convert keys to string representation
+		userRoomKeys := make([]string, len(roomKeyMap))
+		i := 0
+		for _, key := range roomKeyMap {
+			userRoomKeys[i] = spec.Base64Bytes(key).Encode()
+			i += 1
+		}
+
+		// Convert the string representation to its NID
+		pseudoIDStateKeys, sqlErr := d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, nil, userRoomKeys)
+		if sqlErr != nil {
+			if sqlErr == sql.ErrNoRows {
+				pseudoIDStateKeys = map[string]types.EventStateKeyNID{}
+			} else {
+				return nil, fmt.Errorf("BulkSelectEventStateKeyNID: could not select state keys for public room keys: %w", err)
+			}
+		}
+
+		// Collect all NIDs together
+		eventStateKeyNIDs = make([]types.EventStateKeyNID, len(pseudoIDStateKeys)+1)
+		eventStateKeyNIDs[0] = userNID
+		i = 1
+		for _, nid := range pseudoIDStateKeys {
+			eventStateKeyNIDs[i] = nid
+			i += 1
+		}
+	} else {
+		// If there are no room keys (so no pseudo ID rooms), we only need to care about the user ID NID.
+		eventStateKeyNIDs = []types.EventStateKeyNID{userNID}
+	}
+
+	// Fetch rooms that match membership for each NID
+	roomNIDs := []types.RoomNID{}
+	for _, nid := range eventStateKeyNIDs {
+		var roomNIDsChunk []types.RoomNID
+		roomNIDsChunk, err = d.MembershipTable.SelectRoomsWithMembership(ctx, nil, nid, membershipState)
+		if err != nil {
+			return nil, fmt.Errorf("GetRoomsByMembership: failed to SelectRoomsWithMembership: %w", err)
+		}
+		roomNIDs = append(roomNIDs, roomNIDsChunk...)
+	}
+
 	roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, roomNIDs)
 	if err != nil {
 		return nil, fmt.Errorf("GetRoomsByMembership: failed to lookup room nids: %w", err)
diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go
index 5d6ddc9a8..434bad295 100644
--- a/roomserver/storage/sqlite3/user_room_keys_table.go
+++ b/roomserver/storage/sqlite3/user_room_keys_table.go
@@ -56,12 +56,15 @@ const selectUserRoomPublicKeySQL = `SELECT pseudo_id_pub_key FROM roomserver_use
 
 const selectUserNIDsSQL = `SELECT user_nid, room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE room_nid IN ($1) AND pseudo_id_pub_key IN ($2)`
 
+const selectAllUserRoomPublicKeyForUserSQL = `SELECT room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1`
+
 type userRoomKeysStatements struct {
-	db                           *sql.DB
-	insertUserRoomPrivateKeyStmt *sql.Stmt
-	insertUserRoomPublicKeyStmt  *sql.Stmt
-	selectUserRoomKeyStmt        *sql.Stmt
-	selectUserRoomPublicKeyStmt  *sql.Stmt
+	db                                 *sql.DB
+	insertUserRoomPrivateKeyStmt       *sql.Stmt
+	insertUserRoomPublicKeyStmt        *sql.Stmt
+	selectUserRoomKeyStmt              *sql.Stmt
+	selectUserRoomPublicKeyStmt        *sql.Stmt
+	selectAllUserRoomPublicKeysForUser *sql.Stmt
 	//selectUserNIDsStmt           *sql.Stmt //prepared at runtime
 }
 
@@ -77,6 +80,7 @@ func PrepareUserRoomKeysTable(db *sql.DB) (tables.UserRoomKeys, error) {
 		{&s.insertUserRoomPublicKeyStmt, insertUserRoomPublicKeySQL},
 		{&s.selectUserRoomKeyStmt, selectUserRoomKeySQL},
 		{&s.selectUserRoomPublicKeyStmt, selectUserRoomPublicKeySQL},
+		{&s.selectAllUserRoomPublicKeysForUser, selectAllUserRoomPublicKeyForUserSQL},
 		//{&s.selectUserNIDsStmt, selectUserNIDsSQL}, //prepared at runtime
 	}.Prepare(db)
 }
@@ -165,3 +169,24 @@ func (s *userRoomKeysStatements) BulkSelectUserNIDs(ctx context.Context, txn *sq
 	}
 	return result, rows.Err()
 }
+
+func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) {
+	stmt := sqlutil.TxStmtContext(ctx, txn, s.selectAllUserRoomPublicKeysForUser)
+
+	rows, err := stmt.QueryContext(ctx, userNID)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+
+	resultMap := make(map[types.RoomNID]ed25519.PublicKey)
+
+	var roomNID types.RoomNID
+	var pubkey ed25519.PublicKey
+	for rows.Next() {
+		if err = rows.Scan(&roomNID, &pubkey); err != nil {
+			return nil, err
+		}
+		resultMap[roomNID] = pubkey
+	}
+	return resultMap, err
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 445c1223f..0ae064e6b 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -198,6 +198,8 @@ type UserRoomKeys interface {
 	// BulkSelectUserNIDs selects all userIDs for the requested senderKeys. Returns a map from publicKey -> types.UserRoomKeyPair.
 	// If a senderKey can't be found, it is omitted in the result.
 	BulkSelectUserNIDs(ctx context.Context, txn *sql.Tx, senderKeys map[types.RoomNID][]ed25519.PublicKey) (map[string]types.UserRoomKeyPair, error)
+	// SelectAllPublicKeysForUser returns all known public keys for a user. Returns a map from room NID -> public key
+	SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error)
 }
 
 // StrippedEvent represents a stripped event for returning extracted content values.
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
index 3c2308954..91a2d63cc 100644
--- a/syncapi/internal/history_visibility.go
+++ b/syncapi/internal/history_visibility.go
@@ -16,6 +16,7 @@ package internal
 
 import (
 	"context"
+	"fmt"
 	"math"
 	"time"
 
@@ -101,13 +102,15 @@ func (ev eventVisibility) allowed() (allowed bool) {
 
 // ApplyHistoryVisibilityFilter applies the room history visibility filter on types.HeaderedEvents.
 // Returns the filtered events and an error, if any.
+//
+// This function assumes that all provided events are from the same room.
 func ApplyHistoryVisibilityFilter(
 	ctx context.Context,
 	syncDB storage.DatabaseTransaction,
 	rsAPI api.SyncRoomserverAPI,
 	events []*types.HeaderedEvent,
 	alwaysIncludeEventIDs map[string]struct{},
-	userID, endpoint string,
+	userID spec.UserID, endpoint string,
 ) ([]*types.HeaderedEvent, error) {
 	if len(events) == 0 {
 		return events, nil
@@ -115,15 +118,29 @@ func ApplyHistoryVisibilityFilter(
 	start := time.Now()
 
 	// try to get the current membership of the user
-	membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID(), userID, math.MaxInt64)
+	membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID(), userID.String(), math.MaxInt64)
 	if err != nil {
 		return nil, err
 	}
 
 	// Get the mapping from eventID -> eventVisibility
 	eventsFiltered := make([]*types.HeaderedEvent, 0, len(events))
-	visibilities := visibilityForEvents(ctx, rsAPI, events, userID, events[0].RoomID())
+	firstEvRoomID, err := spec.NewRoomID(events[0].RoomID())
+	if err != nil {
+		return nil, err
+	}
+	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *firstEvRoomID, userID)
+	if err != nil {
+		return nil, err
+	}
+	visibilities := visibilityForEvents(ctx, rsAPI, events, senderID, *firstEvRoomID)
+
 	for _, ev := range events {
+		// Validate same room assumption
+		if ev.RoomID() != firstEvRoomID.String() {
+			return nil, fmt.Errorf("events from different rooms supplied to ApplyHistoryVisibilityFilter")
+		}
+
 		evVis := visibilities[ev.EventID()]
 		evVis.membershipCurrent = membershipCurrent
 		// Always include specific state events for /sync responses
@@ -133,23 +150,15 @@ func ApplyHistoryVisibilityFilter(
 				continue
 			}
 		}
-		// NOTSPEC: Always allow user to see their own membership events (spec contains more "rules")
 
-		user, err := spec.NewUserID(userID, true)
-		if err != nil {
-			return nil, err
-		}
-		roomID, err := spec.NewRoomID(ev.RoomID())
-		if err != nil {
-			return nil, err
-		}
-		senderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *user)
-		if err == nil && senderID != nil {
+		// NOTSPEC: Always allow user to see their own membership events (spec contains more "rules")
+		if senderID != nil {
 			if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(*senderID)) {
 				eventsFiltered = append(eventsFiltered, ev)
 				continue
 			}
 		}
+
 		// Always allow history evVis events on boundaries. This is done
 		// by setting the effective evVis to the least restrictive
 		// of the old vs new.
@@ -178,13 +187,13 @@ func ApplyHistoryVisibilityFilter(
 }
 
 // visibilityForEvents returns a map from eventID to eventVisibility containing the visibility and the membership
-// of `userID` at the given event.
+// of `senderID` at the given event. If provided sender ID is nil, assume that membership is Leave
 // Returns an error if the roomserver can't calculate the memberships.
 func visibilityForEvents(
 	ctx context.Context,
 	rsAPI api.SyncRoomserverAPI,
 	events []*types.HeaderedEvent,
-	userID, roomID string,
+	senderID *spec.SenderID, roomID spec.RoomID,
 ) map[string]eventVisibility {
 	eventIDs := make([]string, len(events))
 	for i := range events {
@@ -194,15 +203,13 @@ func visibilityForEvents(
 	result := make(map[string]eventVisibility, len(eventIDs))
 
 	// get the membership events for all eventIDs
-	membershipResp := &api.QueryMembershipAtEventResponse{}
-
-	err := rsAPI.QueryMembershipAtEvent(ctx, &api.QueryMembershipAtEventRequest{
-		RoomID:   roomID,
-		EventIDs: eventIDs,
-		UserID:   userID,
-	}, membershipResp)
-	if err != nil {
-		logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'")
+	var err error
+	membershipEvents := make(map[string]*types.HeaderedEvent)
+	if senderID != nil {
+		membershipEvents, err = rsAPI.QueryMembershipAtEvent(ctx, roomID, eventIDs, *senderID)
+		if err != nil {
+			logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'")
+		}
 	}
 
 	// Create a map from eventID -> eventVisibility
@@ -212,7 +219,7 @@ func visibilityForEvents(
 			membershipAtEvent: spec.Leave, // default to leave, to not expose events by accident
 			visibility:        event.Visibility,
 		}
-		ev, ok := membershipResp.Membership[eventID]
+		ev, ok := membershipEvents[eventID]
 		if !ok || ev == nil {
 			result[eventID] = vis
 			continue
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 3f5e990c4..81b82bf6e 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -69,8 +69,8 @@ func (s *mockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spe
 }
 
 // QueryRoomsForUser retrieves a list of room IDs matching the given query.
-func (s *mockRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error {
-	return nil
+func (s *mockRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
+	return nil, nil
 }
 
 // QueryBulkStateContent does a bulk query for state event content in the given rooms.
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 649d77b41..b0c91c40b 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -138,7 +138,7 @@ func Context(
 
 	// verify the user is allowed to see the context for this room/event
 	startTime := time.Now()
-	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*rstypes.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context")
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*rstypes.HeaderedEvent{&requestedEvent}, nil, *userID, "context")
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 		return util.JSONResponse{
@@ -176,7 +176,7 @@ func Context(
 	}
 
 	startTime = time.Now()
-	eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, device.UserID)
+	eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, *userID)
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 		return util.JSONResponse{
@@ -257,7 +257,7 @@ func Context(
 func applyHistoryVisibilityOnContextEvents(
 	ctx context.Context, snapshot storage.DatabaseTransaction, rsAPI roomserver.SyncRoomserverAPI,
 	eventsBefore, eventsAfter []*rstypes.HeaderedEvent,
-	userID string,
+	userID spec.UserID,
 ) (filteredBefore, filteredAfter []*rstypes.HeaderedEvent, err error) {
 	eventIDsBefore := make(map[string]struct{}, len(eventsBefore))
 	eventIDsAfter := make(map[string]struct{}, len(eventsAfter))
diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go
index 09c2aef02..4fa282f3b 100644
--- a/syncapi/routing/getevent.go
+++ b/syncapi/routing/getevent.go
@@ -37,7 +37,7 @@ import (
 func GetEvent(
 	req *http.Request,
 	device *userapi.Device,
-	roomID string,
+	rawRoomID string,
 	eventID string,
 	cfg *config.SyncAPI,
 	syncDB storage.Database,
@@ -47,7 +47,7 @@ func GetEvent(
 	db, err := syncDB.NewDatabaseTransaction(ctx)
 	logger := util.GetLogger(ctx).WithFields(logrus.Fields{
 		"event_id": eventID,
-		"room_id":  roomID,
+		"room_id":  rawRoomID,
 	})
 	if err != nil {
 		logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed")
@@ -57,6 +57,14 @@ func GetEvent(
 		}
 	}
 
+	roomID, err := spec.NewRoomID(rawRoomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid room ID"),
+		}
+	}
+
 	events, err := db.Events(ctx, []string{eventID})
 	if err != nil {
 		logger.WithError(err).Error("GetEvent: syncDB.Events failed")
@@ -76,13 +84,22 @@ func GetEvent(
 	}
 
 	// If the request is coming from an appservice, get the user from the request
-	userID := device.UserID
+	rawUserID := device.UserID
 	if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" {
-		userID = asUserID
+		rawUserID = asUserID
+	}
+
+	userID, err := spec.NewUserID(rawUserID, true)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("invalid device.UserID")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
 	}
 
 	// Apply history visibility to determine if the user is allowed to view the event
-	events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event")
+	events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, *userID, "event")
 	if err != nil {
 		logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed")
 		return util.JSONResponse{
@@ -101,18 +118,14 @@ func GetEvent(
 		}
 	}
 
-	sender := spec.UserID{}
-	validRoomID, err := spec.NewRoomID(roomID)
-	if err != nil {
+	senderUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *roomID, events[0].SenderID())
+	if err != nil || senderUserID == nil {
+		util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("QueryUserIDForSender errored or returned nil-user ID when user should be part of a room")
 		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("roomID is invalid"),
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
 		}
 	}
-	senderUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, events[0].SenderID())
-	if err == nil && senderUserID != nil {
-		sender = *senderUserID
-	}
 
 	sk := events[0].StateKey()
 	if sk != nil && *sk != "" {
@@ -131,6 +144,6 @@ func GetEvent(
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: synctypes.ToClientEvent(events[0], synctypes.FormatAll, sender, sk),
+		JSON: synctypes.ToClientEvent(events[0], synctypes.FormatAll, *senderUserID, sk),
 	}
 }
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 23a095449..3333cb54d 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -50,6 +50,7 @@ type messagesReq struct {
 	from             *types.TopologyToken
 	to               *types.TopologyToken
 	device           *userapi.Device
+	deviceUserID     spec.UserID
 	wasToProvided    bool
 	backwardOrdering bool
 	filter           *synctypes.RoomEventFilter
@@ -77,6 +78,15 @@ func OnIncomingMessagesRequest(
 ) util.JSONResponse {
 	var err error
 
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("device.UserID invalid")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
 	// NewDatabaseTransaction is used here instead of NewDatabaseSnapshot as we
 	// expect to be able to write to the database in response to a /messages
 	// request that requires backfilling from the roomserver or federation.
@@ -240,6 +250,7 @@ func OnIncomingMessagesRequest(
 		filter:           filter,
 		backwardOrdering: backwardOrdering,
 		device:           device,
+		deviceUserID:     *deviceUserID,
 	}
 
 	clientEvents, start, end, err := mReq.retrieveEvents(req.Context(), rsAPI)
@@ -359,7 +370,7 @@ func (r *messagesReq) retrieveEvents(ctx context.Context, rsAPI api.SyncRoomserv
 
 	// Apply room history visibility filter
 	startTime := time.Now()
-	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.device.UserID, "messages")
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.deviceUserID, "messages")
 	if err != nil {
 		return []synctypes.ClientEvent{}, *r.from, *r.to, nil
 	}
diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go
index 17933b2fb..e3d1069a0 100644
--- a/syncapi/routing/relations.go
+++ b/syncapi/routing/relations.go
@@ -43,9 +43,25 @@ func Relations(
 	req *http.Request, device *userapi.Device,
 	syncDB storage.Database,
 	rsAPI api.SyncRoomserverAPI,
-	roomID, eventID, relType, eventType string,
+	rawRoomID, eventID, relType, eventType string,
 ) util.JSONResponse {
-	var err error
+	roomID, err := spec.NewRoomID(rawRoomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("invalid room ID"),
+		}
+	}
+
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("device.UserID invalid")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("internal server error"),
+		}
+	}
+
 	var from, to types.StreamPosition
 	var limit int
 	dir := req.URL.Query().Get("dir")
@@ -93,7 +109,7 @@ func Relations(
 	}
 	var events []types.StreamEvent
 	events, res.PrevBatch, res.NextBatch, err = snapshot.RelationsFor(
-		req.Context(), roomID, eventID, relType, eventType, from, to, dir == "b", limit,
+		req.Context(), roomID.String(), eventID, relType, eventType, from, to, dir == "b", limit,
 	)
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -105,12 +121,7 @@ func Relations(
 	}
 
 	// Apply history visibility to the result events.
-	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, device.UserID, "relations")
-	if err != nil {
-		return util.ErrorResponse(err)
-	}
-
-	validRoomID, err := spec.NewRoomID(roomID)
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, *userID, "relations")
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
@@ -120,14 +131,14 @@ func Relations(
 	res.Chunk = make([]synctypes.ClientEvent, 0, len(filteredEvents))
 	for _, event := range filteredEvents {
 		sender := spec.UserID{}
-		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, event.SenderID())
+		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *roomID, event.SenderID())
 		if err == nil && userID != nil {
 			sender = *userID
 		}
 
 		sk := event.StateKey()
 		if sk != nil && *sk != "" {
-			skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(*event.StateKey()))
+			skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *roomID, spec.SenderID(*event.StateKey()))
 			if err == nil && skUserID != nil {
 				skString := skUserID.String()
 				sk = &skString
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 48daf857d..4622c21ad 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -562,8 +562,13 @@ func applyHistoryVisibilityFilter(
 		}
 	}
 
+	parsedUserID, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return nil, err
+	}
+
 	startTime := time.Now()
-	events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync")
+	events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, *parsedUserID, "sync")
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 996b21e90..ea1183cd2 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -44,6 +44,11 @@ func (s *syncRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spe
 	return spec.NewUserID(string(senderID), true)
 }
 
+func (s *syncRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	senderID := spec.SenderID(userID.String())
+	return &senderID, nil
+}
+
 func (s *syncRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error {
 	var room *test.Room
 	for _, r := range s.rooms {
@@ -74,8 +79,13 @@ func (s *syncRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *rsa
 	return nil
 }
 
-func (s *syncRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, req *rsapi.QueryMembershipAtEventRequest, res *rsapi.QueryMembershipAtEventResponse) error {
-	return nil
+func (s *syncRoomserverAPI) QueryMembershipAtEvent(
+	ctx context.Context,
+	roomID spec.RoomID,
+	eventIDs []string,
+	senderID spec.SenderID,
+) (map[string]*rstypes.HeaderedEvent, error) {
+	return map[string]*rstypes.HeaderedEvent{}, nil
 }
 
 type syncUserAPI struct {

From 57ddbe015d1d5dec428d8d146bc35d45caa9ba89 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 23 Aug 2023 16:24:16 +0200
Subject: [PATCH 10/57] Version 0.13.2 (#3187)

---
 .github/workflows/dendrite.yml |  6 +++---
 CHANGES.md                     | 27 +++++++++++++++++++++++++++
 helm/dendrite/Chart.yaml       |  2 +-
 helm/dendrite/README.md        |  7 ++++---
 internal/version.go            |  2 +-
 5 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 85fd355f2..83701c50c 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -123,7 +123,7 @@ jobs:
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - run: go test -json -v ./... 2>&1 | gotestfmt
+      - run: go test -json -v ./... 2>&1 | gotestfmt -hide all
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -255,7 +255,7 @@ jobs:
           key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-stable-test-race-
-      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
+      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -436,7 +436,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:
diff --git a/CHANGES.md b/CHANGES.md
index c99ed2255..f4a814566 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,32 @@
 # Changelog
 
+## Dendrite 0.13.2 (2023-08-23)
+
+### Fixes:
+
+- Migrations in SQLite are now prepared on the correct context (transaction or database)
+- The `InputRoomEvent` stream now has a maximum age of 24h, which should help with slow start up times of NATS JetStream (contributed by [neilalexander](https://github.com/neilalexander))
+- Event size checks are more in line with Synapse
+- Requests to `/messages` have been optimized, possibly reducing database round trips
+- Re-add the revision of Dendrite when building from source (Note: This only works if git is installed)
+- Getting local members to notify has been optimized, which should significantly reduce memory allocation and cache usage
+- When getting queried about user profiles, we now return HTTP404 if the user/profiles does not exist
+- Background federated joins should now be fixed and not timeout after a short time
+- Database connections are now correctly re-used
+- Restored the old behavior of the `/purgeRoom` admin endpoint (does not evacuate the room before purging)
+- Don't expose information about the system when trying to download files that don't exist
+
+### Features
+
+- Further improvements and fixes for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+  - Lookup correct prev events in the sync API
+  - Populate `prev_sender` correctly in the sync API
+  - Event federation should work better
+- Added new `dendrite_up` Prometheus metric, containing the version of Dendrite
+- Space summaries ([MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)) have been moved from MSC to being natively supported
+- For easier issue investigation, logs for application services now contain the application service ID (contributed by [maxberger](https://github.com/maxberger))
+- The default room version to use when creating rooms can now be configured using `room_server.default_room_version`
+
 ## Dendrite 0.13.1 (2023-07-06)
 
 This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended.
diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index 8fa06dd97..ef8903d17 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: dendrite
 version: "0.13.1"
-appVersion: "0.13.1"
+appVersion: "0.13.2"
 description: Dendrite Matrix Homeserver
 type: application
 keywords:
diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index 7eabe88e6..40c9d162e 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.1](https://img.shields.io/badge/Version-0.13.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.1](https://img.shields.io/badge/AppVersion-0.13.1-informational?style=flat-square)
+![Version: 0.13.1](https://img.shields.io/badge/Version-0.13.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.2](https://img.shields.io/badge/AppVersion-0.13.2-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -63,6 +63,9 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
 | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process |
 | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods |
+| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
+| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process |
+| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods |
 | dendrite_config.version | int | `2` |  |
 | dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. |
 | dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) |
@@ -189,5 +192,3 @@ grafana:
 ```
 PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`)
 
-----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
\ No newline at end of file
diff --git a/internal/version.go b/internal/version.go
index eedc3327c..81e0fc529 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -18,7 +18,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 13
-	VersionPatch = 1
+	VersionPatch = 2
 	VersionTag   = "" // example: "rc1"
 
 	gitRevLen = 7 // 7 matches the displayed characters on github.com

From 845800abfa4acdc4ba082ace98f49de07c26dd37 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 23 Aug 2023 16:44:52 +0200
Subject: [PATCH 11/57] Bump helm chart version

---
 helm/dendrite/Chart.yaml | 2 +-
 helm/dendrite/README.md  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index ef8903d17..46be9f781 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.1"
+version: "0.13.2"
 appVersion: "0.13.2"
 description: Dendrite Matrix Homeserver
 type: application
diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index 40c9d162e..7f7ea484a 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.1](https://img.shields.io/badge/Version-0.13.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.2](https://img.shields.io/badge/AppVersion-0.13.2-informational?style=flat-square)
+![Version: 0.13.2](https://img.shields.io/badge/Version-0.13.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.2](https://img.shields.io/badge/AppVersion-0.13.2-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**

From a721294e2b339b45fe84995deb756bfe66804c45 Mon Sep 17 00:00:00 2001
From: Devon Hudson <devonhudson@librem.one>
Date: Wed, 23 Aug 2023 08:56:44 -0600
Subject: [PATCH 12/57] Bump pinecone docker go version

---
 build/docker/Dockerfile.demo-pinecone | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone
index 90f515167..ab50cf318 100644
--- a/build/docker/Dockerfile.demo-pinecone
+++ b/build/docker/Dockerfile.demo-pinecone
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.19-alpine AS base
+FROM docker.io/golang:1.21-alpine AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

From 9b5be6b9c552a221e1a6f67d1e632ffc76591d4c Mon Sep 17 00:00:00 2001
From: Sam Wedgwood <28223854+swedgwood@users.noreply.github.com>
Date: Thu, 24 Aug 2023 16:43:51 +0100
Subject: [PATCH 13/57] [pseudoIDs] More pseudo ID fixes - Part 2 (#3181)

Fixes include:
- Translating state keys that contain user IDs to their respective room
keys for both querying and sending state events
- **NOTE**: there may be design discussion needed on what should happen
when sender keys cannot be found for users
- A simple fix for kicking guests from rooms properly
- Logic for boundary history visibilities was slightly off (I'm
surprised this only manifested in pseudo ID room versions)

Signed-off-by: `Sam Wedgwood <sam@wedgwood.dev>`
---
 clientapi/routing/sendevent.go              |  25 ++
 clientapi/routing/sendevent_test.go         | 275 ++++++++++++++++++++
 clientapi/routing/state.go                  |  31 +++
 clientapi/routing/state_test.go             | 253 ++++++++++++++++++
 go.mod                                      |   4 +-
 go.sum                                      |   8 +-
 roomserver/internal/input/input_events.go   |   7 +-
 syncapi/internal/history_visibility.go      |  28 +-
 syncapi/internal/history_visibility_test.go | 214 +++++++++++++++
 syncapi/internal/keychange_test.go          |  26 +-
 syncapi/synctypes/clientevent.go            |  30 +++
 11 files changed, 865 insertions(+), 36 deletions(-)
 create mode 100644 clientapi/routing/sendevent_test.go
 create mode 100644 clientapi/routing/state_test.go
 create mode 100644 syncapi/internal/history_visibility_test.go

diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index a167a5a77..f81e9c1e4 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -29,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -92,6 +93,30 @@ func SendEvent(
 		}
 	}
 
+	// Translate user ID state keys to room keys in pseudo ID rooms
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && stateKey != nil {
+		parsedRoomID, innerErr := spec.NewRoomID(roomID)
+		if innerErr != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("invalid room ID"),
+			}
+		}
+
+		newStateKey, innerErr := synctypes.FromClientStateKey(*parsedRoomID, *stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+			return rsAPI.QuerySenderIDForUser(req.Context(), roomID, userID)
+		})
+		if innerErr != nil {
+			// TODO: work out better logic for failure cases (e.g. sender ID not found)
+			util.GetLogger(req.Context()).WithError(innerErr).Error("synctypes.FromClientStateKey failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+		stateKey = newStateKey
+	}
+
 	// create a mutex for the specific user in the specific room
 	// this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order
 	userID := device.UserID
diff --git a/clientapi/routing/sendevent_test.go b/clientapi/routing/sendevent_test.go
new file mode 100644
index 000000000..9cdd75358
--- /dev/null
+++ b/clientapi/routing/sendevent_test.go
@@ -0,0 +1,275 @@
+package routing
+
+import (
+	"context"
+	"crypto/ed25519"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"gotest.tools/v3/assert"
+)
+
+// Mock roomserver API for testing
+//
+// Currently pretty specialised for the pseudo ID test, so will need
+// editing if future (other) sendevent tests are using this.
+type sendEventTestRoomserverAPI struct {
+	rsapi.ClientRoomserverAPI
+	t           *testing.T
+	roomIDStr   string
+	roomVersion gomatrixserverlib.RoomVersion
+	roomState   []*types.HeaderedEvent
+
+	// userID -> room key
+	senderMapping map[string]ed25519.PrivateKey
+
+	savedInputRoomEvents []rsapi.InputRoomEvent
+}
+
+func (s *sendEventTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) {
+	if roomID == s.roomIDStr {
+		return s.roomVersion, nil
+	} else {
+		s.t.Logf("room version queried for %s", roomID)
+		return "", fmt.Errorf("unknown room")
+	}
+}
+
+func (s *sendEventTestRoomserverAPI) QueryCurrentState(ctx context.Context, req *rsapi.QueryCurrentStateRequest, res *rsapi.QueryCurrentStateResponse) error {
+	res.StateEvents = map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{}
+	for _, stateKeyTuple := range req.StateTuples {
+		for _, stateEv := range s.roomState {
+			if stateEv.Type() == stateKeyTuple.EventType && stateEv.StateKey() != nil && *stateEv.StateKey() == stateKeyTuple.StateKey {
+				res.StateEvents[stateKeyTuple] = stateEv
+			}
+		}
+	}
+	return nil
+}
+
+func (s *sendEventTestRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error {
+	if req.RoomID == s.roomIDStr {
+		res.RoomExists = true
+		res.RoomVersion = s.roomVersion
+
+		res.StateEvents = make([]*types.HeaderedEvent, len(s.roomState))
+		copy(res.StateEvents, s.roomState)
+
+		res.LatestEvents = []string{}
+		res.Depth = 1
+		return nil
+	} else {
+		s.t.Logf("room event/state queried for %s", req.RoomID)
+		return fmt.Errorf("unknown room")
+	}
+
+}
+
+func (s *sendEventTestRoomserverAPI) QuerySenderIDForUser(
+	ctx context.Context,
+	roomID spec.RoomID,
+	userID spec.UserID,
+) (*spec.SenderID, error) {
+	if roomID.String() == s.roomIDStr {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			roomKey, ok := s.senderMapping[userID.String()]
+			if ok {
+				sender := spec.SenderIDFromPseudoIDKey(roomKey)
+				return &sender, nil
+			} else {
+				return nil, nil
+			}
+		} else {
+			senderID := spec.SenderIDFromUserID(userID)
+			return &senderID, nil
+		}
+	}
+
+	return nil, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) QueryUserIDForSender(
+	ctx context.Context,
+	roomID spec.RoomID,
+	senderID spec.SenderID,
+) (*spec.UserID, error) {
+	if roomID.String() == s.roomIDStr {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			for uID, roomKey := range s.senderMapping {
+				if string(spec.SenderIDFromPseudoIDKey(roomKey)) == string(senderID) {
+					parsedUserID, err := spec.NewUserID(uID, true)
+					if err != nil {
+						s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err)
+					}
+					return parsedUserID, nil
+				}
+			}
+		} else {
+			userID := senderID.ToUserID()
+			if userID == nil {
+				return nil, fmt.Errorf("bad sender ID")
+			}
+			return userID, nil
+		}
+	}
+
+	return nil, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, sender spec.UserID) (fclient.SigningIdentity, error) {
+	if s.roomIDStr == roomID.String() {
+		if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs {
+			roomKey, ok := s.senderMapping[sender.String()]
+			if !ok {
+				s.t.Logf("SigningIdentityFor used with unknown user ID: %v", sender.String())
+				return fclient.SigningIdentity{}, fmt.Errorf("could not get signing identity for %v", sender.String())
+			}
+			return fclient.SigningIdentity{PrivateKey: roomKey}, nil
+		} else {
+			return fclient.SigningIdentity{PrivateKey: ed25519.NewKeyFromSeed(make([]byte, 32))}, nil
+		}
+	}
+
+	return fclient.SigningIdentity{}, fmt.Errorf("room not found")
+}
+
+func (s *sendEventTestRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+	s.savedInputRoomEvents = req.InputRoomEvents
+}
+
+// Test that user ID state keys are translated correctly
+func Test_SendEvent_PseudoIDStateKeys(t *testing.T) {
+	nonpseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10
+	pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs
+
+	senderKeySeed := make([]byte, 32)
+	senderUserID := "@testuser:domain"
+	senderPrivKey := ed25519.NewKeyFromSeed(senderKeySeed)
+	senderPseudoID := string(spec.SenderIDFromPseudoIDKey(senderPrivKey))
+
+	eventType := "com.example.test"
+	roomIDStr := "!id:domain"
+
+	device := &uapi.Device{
+		UserID: senderUserID,
+	}
+
+	t.Run("user ID state key are not translated to room key in non-pseudo ID room", func(t *testing.T) {
+		eventsJSON := []string{
+			fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderUserID, senderUserID, nonpseudoIDRoomVersion),
+			fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderUserID, roomIDStr, senderUserID),
+		}
+
+		roomState, err := createEvents(eventsJSON, nonpseudoIDRoomVersion)
+		if err != nil {
+			t.Fatalf("failed to prepare state events: %s", err.Error())
+		}
+
+		rsAPI := &sendEventTestRoomserverAPI{
+			t:           t,
+			roomIDStr:   roomIDStr,
+			roomVersion: nonpseudoIDRoomVersion,
+			roomState:   roomState,
+		}
+
+		req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}")))
+		if err != nil {
+			t.Fatalf("failed to make new request: %s", err.Error())
+		}
+
+		cfg := &config.ClientAPI{}
+
+		resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp)
+		}
+
+		assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1)
+
+		ev := rsAPI.savedInputRoomEvents[0]
+		stateKey := ev.Event.StateKey()
+		if stateKey == nil {
+			t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderUserID)
+		}
+		if *stateKey != senderUserID {
+			t.Fatalf("expected submitted InputRoomEvent to have user ID state key\nfound: %v\nexpected: %v", *stateKey, senderUserID)
+		}
+	})
+
+	t.Run("user ID state key are translated to room key in pseudo ID room", func(t *testing.T) {
+		eventsJSON := []string{
+			fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderPseudoID, senderPseudoID, pseudoIDRoomVersion),
+			fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderPseudoID, roomIDStr, senderPseudoID),
+		}
+
+		roomState, err := createEvents(eventsJSON, pseudoIDRoomVersion)
+		if err != nil {
+			t.Fatalf("failed to prepare state events: %s", err.Error())
+		}
+
+		rsAPI := &sendEventTestRoomserverAPI{
+			t:           t,
+			roomIDStr:   roomIDStr,
+			roomVersion: pseudoIDRoomVersion,
+			senderMapping: map[string]ed25519.PrivateKey{
+				senderUserID: senderPrivKey,
+			},
+			roomState: roomState,
+		}
+
+		req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}")))
+		if err != nil {
+			t.Fatalf("failed to make new request: %s", err.Error())
+		}
+
+		cfg := &config.ClientAPI{}
+
+		resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp)
+		}
+
+		assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1)
+
+		ev := rsAPI.savedInputRoomEvents[0]
+		stateKey := ev.Event.StateKey()
+		if stateKey == nil {
+			t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderPseudoID)
+		}
+		if *stateKey != senderPseudoID {
+			t.Fatalf("expected submitted InputRoomEvent to have pseudo ID state key\nfound: %v\nexpected: %v", *stateKey, senderPseudoID)
+		}
+	})
+}
+
+func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([]*types.HeaderedEvent, error) {
+	events := make([]*types.HeaderedEvent, len(eventsJSON))
+
+	roomVerImpl, err := gomatrixserverlib.GetRoomVersion(roomVer)
+	if err != nil {
+		return nil, fmt.Errorf("no roomver impl: %s", err.Error())
+	}
+
+	for i, eventJSON := range eventsJSON {
+		pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false)
+		if evErr != nil {
+			return nil, fmt.Errorf("failed to make event: %s", err.Error())
+		}
+		ev := types.HeaderedEvent{PDU: pdu}
+		events[i] = &ev
+	}
+
+	return events, nil
+}
diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index f53cb3013..7648dc474 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -217,6 +217,37 @@ func OnIncomingStateTypeRequest(
 	var worldReadable bool
 	var wantLatestState bool
 
+	roomVer, err := rsAPI.QueryRoomVersionForRoom(ctx, roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
+		}
+	}
+
+	// Translate user ID state keys to room keys in pseudo ID rooms
+	if roomVer == gomatrixserverlib.RoomVersionPseudoIDs {
+		parsedRoomID, err := spec.NewRoomID(roomID)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: spec.InvalidParam("invalid room ID"),
+			}
+		}
+		newStateKey, err := synctypes.FromClientStateKey(*parsedRoomID, stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+			return rsAPI.QuerySenderIDForUser(ctx, roomID, userID)
+		})
+		if err != nil {
+			// TODO: work out better logic for failure cases (e.g. sender ID not found)
+			util.GetLogger(ctx).WithError(err).Error("synctypes.FromClientStateKey failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+		stateKey = *newStateKey
+	}
+
 	// Always fetch visibility so that we can work out whether to show
 	// the latest events or the last event from when the user was joined.
 	// Then include the requested event type and state key, assuming it
diff --git a/clientapi/routing/state_test.go b/clientapi/routing/state_test.go
new file mode 100644
index 000000000..93b043723
--- /dev/null
+++ b/clientapi/routing/state_test.go
@@ -0,0 +1,253 @@
+package routing
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	"gotest.tools/v3/assert"
+)
+
+var ()
+
+type stateTestRoomserverAPI struct {
+	rsapi.RoomserverInternalAPI
+	t           *testing.T
+	roomState   map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent
+	roomIDStr   string
+	roomVersion gomatrixserverlib.RoomVersion
+	userIDStr   string
+	// userID -> senderID
+	senderMapping map[string]string
+}
+
+func (s stateTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) {
+	if roomID == s.roomIDStr {
+		return s.roomVersion, nil
+	} else {
+		s.t.Logf("room version queried for %s", roomID)
+		return "", fmt.Errorf("unknown room")
+	}
+}
+
+func (s stateTestRoomserverAPI) QueryLatestEventsAndState(
+	ctx context.Context,
+	req *rsapi.QueryLatestEventsAndStateRequest,
+	res *rsapi.QueryLatestEventsAndStateResponse,
+) error {
+	res.RoomExists = req.RoomID == s.roomIDStr
+	if !res.RoomExists {
+		return nil
+	}
+
+	res.StateEvents = []*types.HeaderedEvent{}
+	for _, stateKeyTuple := range req.StateToFetch {
+		val, ok := s.roomState[stateKeyTuple]
+		if ok && val != nil {
+			res.StateEvents = append(res.StateEvents, val)
+		}
+	}
+
+	return nil
+}
+
+func (s stateTestRoomserverAPI) QueryMembershipForUser(
+	ctx context.Context,
+	req *rsapi.QueryMembershipForUserRequest,
+	res *rsapi.QueryMembershipForUserResponse,
+) error {
+	if req.UserID.String() == s.userIDStr {
+		res.HasBeenInRoom = true
+		res.IsInRoom = true
+		res.RoomExists = true
+		res.Membership = spec.Join
+	}
+
+	return nil
+}
+
+func (s stateTestRoomserverAPI) QuerySenderIDForUser(
+	ctx context.Context,
+	roomID spec.RoomID,
+	userID spec.UserID,
+) (*spec.SenderID, error) {
+	sID, ok := s.senderMapping[userID.String()]
+	if ok {
+		sender := spec.SenderID(sID)
+		return &sender, nil
+	} else {
+		return nil, nil
+	}
+}
+
+func (s stateTestRoomserverAPI) QueryUserIDForSender(
+	ctx context.Context,
+	roomID spec.RoomID,
+	senderID spec.SenderID,
+) (*spec.UserID, error) {
+	for uID, sID := range s.senderMapping {
+		if sID == string(senderID) {
+			parsedUserID, err := spec.NewUserID(uID, true)
+			if err != nil {
+				s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err)
+			}
+			return parsedUserID, nil
+		}
+	}
+	return nil, nil
+}
+
+func (s stateTestRoomserverAPI) QueryStateAfterEvents(
+	ctx context.Context,
+	req *rsapi.QueryStateAfterEventsRequest,
+	res *rsapi.QueryStateAfterEventsResponse,
+) error {
+	return nil
+}
+
+func Test_OnIncomingStateTypeRequest(t *testing.T) {
+	var tempRoomServerCfg config.RoomServer
+	tempRoomServerCfg.Defaults(config.DefaultOpts{})
+	defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion
+	pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs
+	nonPseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10
+
+	userIDStr := "@testuser:domain"
+	eventType := "com.example.test"
+	stateKey := "testStateKey"
+	roomIDStr := "!id:domain"
+
+	device := &uapi.Device{
+		UserID: userIDStr,
+	}
+
+	t.Run("request simple state key", func(t *testing.T) {
+		ctx := context.Background()
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: defaultRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateKey,
+				}: mustCreateStatePDU(t, defaultRoomVersion, roomIDStr, eventType, stateKey, map[string]interface{}{
+					"foo": "bar",
+				}),
+			},
+			userIDStr: userIDStr,
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateKey, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+
+	t.Run("user ID key translated to room key in pseudo ID rooms", func(t *testing.T) {
+		ctx := context.Background()
+
+		stateSenderUserID := "@sender:domain"
+		stateSenderRoomKey := "testsenderkey"
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: pseudoIDRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateSenderRoomKey,
+				}: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{
+					"foo": "bar",
+				}),
+				{
+					EventType: eventType,
+					StateKey:  stateSenderUserID,
+				}: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{
+					"not": "thisone",
+				}),
+			},
+			userIDStr: userIDStr,
+			senderMapping: map[string]string{
+				stateSenderUserID: stateSenderRoomKey,
+			},
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+
+	t.Run("user ID key not translated to room key in non-pseudo ID rooms", func(t *testing.T) {
+		ctx := context.Background()
+
+		stateSenderUserID := "@sender:domain"
+		stateSenderRoomKey := "testsenderkey"
+
+		rsAPI := stateTestRoomserverAPI{
+			roomVersion: nonPseudoIDRoomVersion,
+			roomIDStr:   roomIDStr,
+			roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{
+				{
+					EventType: eventType,
+					StateKey:  stateSenderRoomKey,
+				}: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{
+					"not": "thisone",
+				}),
+				{
+					EventType: eventType,
+					StateKey:  stateSenderUserID,
+				}: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{
+					"foo": "bar",
+				}),
+			},
+			userIDStr: userIDStr,
+			senderMapping: map[string]string{
+				stateSenderUserID: stateSenderUserID,
+			},
+		}
+
+		jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false)
+
+		assert.DeepEqual(t, jsonResp, util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: spec.RawJSON(`{"foo":"bar"}`),
+		})
+	})
+}
+
+func mustCreateStatePDU(t *testing.T, roomVer gomatrixserverlib.RoomVersion, roomID string, stateType string, stateKey string, stateContent map[string]interface{}) *types.HeaderedEvent {
+	t.Helper()
+	roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVer)
+
+	evBytes, err := json.Marshal(map[string]interface{}{
+		"room_id":   roomID,
+		"type":      stateType,
+		"state_key": stateKey,
+		"content":   stateContent,
+	})
+	if err != nil {
+		t.Fatalf("failed to create event: %v", err)
+	}
+
+	ev, err := roomVerImpl.NewEventFromTrustedJSON(evBytes, false)
+	if err != nil {
+		t.Fatalf("failed to create event: %v", err)
+	}
+
+	return &types.HeaderedEvent{PDU: ev}
+}
diff --git a/go.mod b/go.mod
index 915e813a6..4be0ede49 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230823153616-484e7693bb8d
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
@@ -36,7 +36,7 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.2
-	github.com/tidwall/gjson v1.15.0
+	github.com/tidwall/gjson v1.16.0
 	github.com/tidwall/sjson v1.2.5
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
diff --git a/go.sum b/go.sum
index 6c7e48dbf..3fdfe01a4 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac h1:s4EZRNT6/TtGAzcO6yzL+UTv96vEeeaH6y2RrIOfsWw=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230807152937-c48e302e15ac/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230823153616-484e7693bb8d h1:yFoT2nyjD4TFrgYMJGgrotFbTLjaYKfZbRmnsj7lvZE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230823153616-484e7693bb8d/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -318,8 +318,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.15.0 h1:5n/pM+v3r5ujuNl4YLZLsQ+UE5jlkLVm7jMzT5Mpolw=
-github.com/tidwall/gjson v1.15.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
+github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 88049ddf0..bf3216623 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -933,12 +933,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r
 			return err
 		}
 
-		userID, err := spec.NewUserID(stateKey, true)
-		if err != nil {
-			return err
-		}
-
-		signingIdentity, err := r.SigningIdentity(ctx, *validRoomID, *userID)
+		signingIdentity, err := r.SigningIdentity(ctx, *validRoomID, *memberUserID)
 		if err != nil {
 			return err
 		}
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
index 91a2d63cc..7aae9fd38 100644
--- a/syncapi/internal/history_visibility.go
+++ b/syncapi/internal/history_visibility.go
@@ -163,17 +163,23 @@ func ApplyHistoryVisibilityFilter(
 		// by setting the effective evVis to the least restrictive
 		// of the old vs new.
 		// https://spec.matrix.org/v1.3/client-server-api/#server-behaviour-5
-		if hisVis, err := ev.HistoryVisibility(); err == nil {
-			prevHisVis := gjson.GetBytes(ev.Unsigned(), "prev_content.history_visibility").String()
-			oldPrio, ok := historyVisibilityPriority[gomatrixserverlib.HistoryVisibility(prevHisVis)]
-			// if we can't get the previous history visibility, default to shared.
-			if !ok {
-				oldPrio = historyVisibilityPriority[gomatrixserverlib.HistoryVisibilityShared]
-			}
-			// no OK check, since this should have been validated when setting the value
-			newPrio := historyVisibilityPriority[hisVis]
-			if oldPrio < newPrio {
-				evVis.visibility = gomatrixserverlib.HistoryVisibility(prevHisVis)
+		if ev.Type() == spec.MRoomHistoryVisibility {
+			hisVis, err := ev.HistoryVisibility()
+
+			if err == nil && hisVis != "" {
+				prevHisVis := gjson.GetBytes(ev.Unsigned(), "prev_content.history_visibility").String()
+				oldPrio, ok := historyVisibilityPriority[gomatrixserverlib.HistoryVisibility(prevHisVis)]
+				// if we can't get the previous history visibility, default to shared.
+				if !ok {
+					oldPrio = historyVisibilityPriority[gomatrixserverlib.HistoryVisibilityShared]
+				}
+				// no OK check, since this should have been validated when setting the value
+				newPrio := historyVisibilityPriority[hisVis]
+				if oldPrio < newPrio {
+					evVis.visibility = gomatrixserverlib.HistoryVisibility(prevHisVis)
+				} else {
+					evVis.visibility = hisVis
+				}
 			}
 		}
 		// do the actual check
diff --git a/syncapi/internal/history_visibility_test.go b/syncapi/internal/history_visibility_test.go
new file mode 100644
index 000000000..984f90edd
--- /dev/null
+++ b/syncapi/internal/history_visibility_test.go
@@ -0,0 +1,214 @@
+package internal
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"testing"
+
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"gotest.tools/v3/assert"
+)
+
+type mockHisVisRoomserverAPI struct {
+	rsapi.RoomserverInternalAPI
+	events []*types.HeaderedEvent
+	roomID string
+}
+
+func (s *mockHisVisRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, roomID spec.RoomID, eventIDs []string, senderID spec.SenderID) (map[string]*types.HeaderedEvent, error) {
+	if roomID.String() == s.roomID {
+		membershipMap := map[string]*types.HeaderedEvent{}
+
+		for _, queriedEventID := range eventIDs {
+			for _, event := range s.events {
+				if event.EventID() == queriedEventID {
+					membershipMap[queriedEventID] = event
+				}
+			}
+		}
+
+		return membershipMap, nil
+	} else {
+		return nil, fmt.Errorf("room not found: \"%v\"", roomID)
+	}
+}
+
+func (s *mockHisVisRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	senderID := spec.SenderIDFromUserID(userID)
+	return &senderID, nil
+}
+
+func (s *mockHisVisRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	userID := senderID.ToUserID()
+	if userID == nil {
+		return nil, fmt.Errorf("sender ID not user ID")
+	}
+	return userID, nil
+}
+
+type mockDB struct {
+	storage.DatabaseTransaction
+	// user ID -> membership (i.e. 'join', 'leave', etc.)
+	currentMembership map[string]string
+	roomID            string
+}
+
+func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int, error) {
+	if roomID == s.roomID {
+		membership, ok := s.currentMembership[userID]
+		if !ok {
+			return spec.Leave, math.MaxInt64, nil
+		}
+		return membership, math.MaxInt64, nil
+	}
+
+	return "", 0, fmt.Errorf("room not found: \"%v\"", roomID)
+}
+
+// Tests logic around history visibility boundaries
+//
+// Specifically that if a room's history visibility before or after a particular history visibility event
+// allows them to see events (a boundary), then the history visibility event itself should be shown
+// ( spec: https://spec.matrix.org/v1.8/client-server-api/#server-behaviour-5 )
+//
+// This also aims to emulate "Only see history_visibility changes on bounadries" in sytest/tests/30rooms/30history-visibility.pl
+func Test_ApplyHistoryVisbility_Boundaries(t *testing.T) {
+	ctx := context.Background()
+
+	roomID := "!roomid:domain"
+
+	creatorUserID := spec.NewUserIDOrPanic("@creator:domain", false)
+	otherUserID := spec.NewUserIDOrPanic("@other:domain", false)
+	roomVersion := gomatrixserverlib.RoomVersionV10
+	roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVersion)
+
+	eventsJSON := []struct {
+		id   string
+		json string
+	}{
+		{id: "$create-event", json: fmt.Sprintf(`{
+			"type": "m.room.create", "state_key": "",
+			"room_id": "%v", "sender": "%v",
+			"content": {"creator": "%v", "room_version": "%v"}
+		}`, roomID, creatorUserID.String(), creatorUserID.String(), roomVersion)},
+		{id: "$creator-joined", json: fmt.Sprintf(`{
+			"type": "m.room.member", "state_key": "%v",
+			"room_id": "%v", "sender": "%v",
+			"content": {"membership": "join"}
+		}`, creatorUserID.String(), roomID, creatorUserID.String())},
+		{id: "$hisvis-1", json: fmt.Sprintf(`{
+			"type": "m.room.history_visibility", "state_key": "",
+			"room_id": "%v", "sender": "%v",
+			"content": {"history_visibility": "shared"}
+		}`, roomID, creatorUserID.String())},
+		{id: "$msg-1", json: fmt.Sprintf(`{
+			"type": "m.room.message",
+			"room_id": "%v", "sender": "%v",
+			"content": {"body": "1"}
+		}`, roomID, creatorUserID.String())},
+		{id: "$hisvis-2", json: fmt.Sprintf(`{
+			"type": "m.room.history_visibility", "state_key": "",
+			"room_id": "%v", "sender": "%v",
+			"content": {"history_visibility": "joined"},
+			"unsigned": {"prev_content": {"history_visibility": "shared"}}
+		}`, roomID, creatorUserID.String())},
+		{id: "$msg-2", json: fmt.Sprintf(`{
+			"type": "m.room.message",
+			"room_id": "%v", "sender": "%v",
+			"content": {"body": "1"}
+		}`, roomID, creatorUserID.String())},
+		{id: "$hisvis-3", json: fmt.Sprintf(`{
+			"type": "m.room.history_visibility", "state_key": "",
+			"room_id": "%v", "sender": "%v",
+			"content": {"history_visibility": "invited"},
+			"unsigned": {"prev_content": {"history_visibility": "joined"}}
+		}`, roomID, creatorUserID.String())},
+		{id: "$msg-3", json: fmt.Sprintf(`{
+			"type": "m.room.message",
+			"room_id": "%v", "sender": "%v",
+			"content": {"body": "2"}
+		}`, roomID, creatorUserID.String())},
+		{id: "$hisvis-4", json: fmt.Sprintf(`{
+			"type": "m.room.history_visibility", "state_key": "",
+			"room_id": "%v", "sender": "%v",
+			"content": {"history_visibility": "shared"},
+			"unsigned": {"prev_content": {"history_visibility": "invited"}}
+		}`, roomID, creatorUserID.String())},
+		{id: "$msg-4", json: fmt.Sprintf(`{
+			"type": "m.room.message",
+			"room_id": "%v", "sender": "%v",
+			"content": {"body": "3"}
+		}`, roomID, creatorUserID.String())},
+		{id: "$other-joined", json: fmt.Sprintf(`{
+			"type": "m.room.member", "state_key": "%v",
+			"room_id": "%v", "sender": "%v",
+			"content": {"membership": "join"}
+		}`, otherUserID.String(), roomID, otherUserID.String())},
+	}
+
+	events := make([]*types.HeaderedEvent, len(eventsJSON))
+
+	hisVis := gomatrixserverlib.HistoryVisibilityShared
+
+	for i, eventJSON := range eventsJSON {
+		pdu, err := roomVerImpl.NewEventFromTrustedJSONWithEventID(eventJSON.id, []byte(eventJSON.json), false)
+		if err != nil {
+			t.Fatalf("failed to prepare event %s for test: %s", eventJSON.id, err.Error())
+		}
+		events[i] = &types.HeaderedEvent{PDU: pdu}
+
+		// 'Visibility' should be the visibility of the room just before this event was sent
+		// (according to processRoomEvent in roomserver/internal/input/input_events.go)
+		events[i].Visibility = hisVis
+		if pdu.Type() == spec.MRoomHistoryVisibility {
+			newHisVis, err := pdu.HistoryVisibility()
+			if err != nil {
+				t.Fatalf("failed to prepare history visibility event: %s", err.Error())
+			}
+			hisVis = newHisVis
+		}
+	}
+
+	rsAPI := &mockHisVisRoomserverAPI{
+		events: events,
+		roomID: roomID,
+	}
+	syncDB := &mockDB{
+		roomID: roomID,
+		currentMembership: map[string]string{
+			creatorUserID.String(): spec.Join,
+			otherUserID.String():   spec.Join,
+		},
+	}
+
+	filteredEvents, err := ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, events, nil, otherUserID, "hisVisTest")
+	if err != nil {
+		t.Fatalf("ApplyHistoryVisibility returned non-nil error: %s", err.Error())
+	}
+
+	filteredEventIDs := make([]string, len(filteredEvents))
+	for i, event := range filteredEvents {
+		filteredEventIDs[i] = event.EventID()
+	}
+
+	assert.DeepEqual(t,
+		[]string{
+			"$create-event",   // Always see m.room.create
+			"$creator-joined", // Always see membership
+			"$hisvis-1",       // Sets room to shared (technically the room is already shared since shared is default)
+			"$msg-1",          // Room currently 'shared'
+			"$hisvis-2",       // Room changed from 'shared' to 'joined', so boundary event and should be shared
+			// Other events hidden, as other is not joined yet
+			// hisvis-3 is also hidden, as it changes from joined to invited, neither of which is visible to other
+			"$hisvis-4",     // Changes from 'invited' to 'shared', so is a boundary event and visible
+			"$msg-4",        // Room is 'shared', so visible
+			"$other-joined", // other's membership
+		},
+		filteredEventIDs,
+	)
+}
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 81b82bf6e..56954cfa0 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -59,22 +59,22 @@ func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *userapi.Query
 func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *userapi.QuerySignaturesRequest, res *userapi.QuerySignaturesResponse) {
 }
 
-type mockRoomserverAPI struct {
+type keyChangeMockRoomserverAPI struct {
 	api.RoomserverInternalAPI
 	roomIDToJoinedMembers map[string][]string
 }
 
-func (s *mockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+func (s *keyChangeMockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 	return spec.NewUserID(string(senderID), true)
 }
 
 // QueryRoomsForUser retrieves a list of room IDs matching the given query.
-func (s *mockRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
+func (s *keyChangeMockRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) {
 	return nil, nil
 }
 
 // QueryBulkStateContent does a bulk query for state event content in the given rooms.
-func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error {
+func (s *keyChangeMockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error {
 	res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string)
 	if req.AllowWildcards && len(req.StateTuples) == 1 && req.StateTuples[0].EventType == spec.MRoomMember && req.StateTuples[0].StateKey == "*" {
 		for _, roomID := range req.RoomIDs {
@@ -91,7 +91,7 @@ func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.
 }
 
 // QuerySharedUsers returns a list of users who share at least 1 room in common with the given user.
-func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error {
+func (s *keyChangeMockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error {
 	roomsToQuery := req.IncludeRoomIDs
 	for roomID, members := range s.roomIDToJoinedMembers {
 		exclude := false
@@ -123,7 +123,7 @@ func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.Query
 
 // This is actually a database function, but seeing as we track the state inside the
 // *mockRoomserverAPI, we'll just comply with the interface here instead.
-func (s *mockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) {
+func (s *keyChangeMockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) {
 	commonUsers := []string{}
 	for _, members := range s.roomIDToJoinedMembers {
 		for _, member := range members {
@@ -211,7 +211,7 @@ func TestKeyChangeCatchupOnJoinShareNewUser(t *testing.T) {
 	syncResponse := types.NewResponse()
 	syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom})
 
-	rsAPI := &mockRoomserverAPI{
+	rsAPI := &keyChangeMockRoomserverAPI{
 		roomIDToJoinedMembers: map[string][]string{
 			newlyJoinedRoom: {syncingUser, newShareUser},
 			"!another:room": {syncingUser},
@@ -234,7 +234,7 @@ func TestKeyChangeCatchupOnLeaveShareLeftUser(t *testing.T) {
 	syncResponse := types.NewResponse()
 	syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom})
 
-	rsAPI := &mockRoomserverAPI{
+	rsAPI := &keyChangeMockRoomserverAPI{
 		roomIDToJoinedMembers: map[string][]string{
 			newlyLeftRoom:   {removeUser},
 			"!another:room": {syncingUser},
@@ -257,7 +257,7 @@ func TestKeyChangeCatchupOnJoinShareNoNewUsers(t *testing.T) {
 	syncResponse := types.NewResponse()
 	syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom})
 
-	rsAPI := &mockRoomserverAPI{
+	rsAPI := &keyChangeMockRoomserverAPI{
 		roomIDToJoinedMembers: map[string][]string{
 			newlyJoinedRoom: {syncingUser, existingUser},
 			"!another:room": {syncingUser, existingUser},
@@ -279,7 +279,7 @@ func TestKeyChangeCatchupOnLeaveShareNoUsers(t *testing.T) {
 	syncResponse := types.NewResponse()
 	syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom})
 
-	rsAPI := &mockRoomserverAPI{
+	rsAPI := &keyChangeMockRoomserverAPI{
 		roomIDToJoinedMembers: map[string][]string{
 			newlyLeftRoom:   {existingUser},
 			"!another:room": {syncingUser, existingUser},
@@ -343,7 +343,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) {
 	jr.Timeline = &types.Timeline{Events: roomTimelineEvents}
 	syncResponse.Rooms.Join[roomID] = jr
 
-	rsAPI := &mockRoomserverAPI{
+	rsAPI := &keyChangeMockRoomserverAPI{
 		roomIDToJoinedMembers: map[string][]string{
 			roomID: {syncingUser, existingUser},
 		},
@@ -369,7 +369,7 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) {
 	syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom})
 	syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom})
 
-	rsAPI := &mockRoomserverAPI{
+	rsAPI := &keyChangeMockRoomserverAPI{
 		roomIDToJoinedMembers: map[string][]string{
 			newlyJoinedRoom: {syncingUser, newShareUser, newShareUser2},
 			newlyLeftRoom:   {newlyLeftUser, newlyLeftUser2},
@@ -459,7 +459,7 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) {
 	lr.Timeline = &types.Timeline{Events: roomEvents}
 	syncResponse.Rooms.Leave[roomID] = lr
 
-	rsAPI := &mockRoomserverAPI{
+	rsAPI := &keyChangeMockRoomserverAPI{
 		roomIDToJoinedMembers: map[string][]string{
 			roomID:          {newShareUser, newShareUser2},
 			"!another:room": {syncingUser},
diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go
index 6f03d9ff0..a78aea1c6 100644
--- a/syncapi/synctypes/clientevent.go
+++ b/syncapi/synctypes/clientevent.go
@@ -16,6 +16,8 @@
 package synctypes
 
 import (
+	"fmt"
+
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 )
@@ -118,3 +120,31 @@ func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserver
 	}
 	return ToClientEvent(event, FormatAll, sender, sk)
 }
+
+// If provided state key is a user ID (state keys beginning with @ are reserved for this purpose)
+// fetch it's associated sender ID and use that instead. Otherwise returns the same state key back.
+//
+// # This function either returns the state key that should be used, or an error
+//
+// TODO: handle failure cases better (e.g. no sender ID)
+func FromClientStateKey(roomID spec.RoomID, stateKey string, senderIDQuery spec.SenderIDForUser) (*string, error) {
+	if len(stateKey) >= 1 && stateKey[0] == '@' {
+		parsedStateKey, err := spec.NewUserID(stateKey, true)
+		if err != nil {
+			// If invalid user ID, then there is no associated state event.
+			return nil, fmt.Errorf("Provided state key begins with @ but is not a valid user ID: %s", err.Error())
+		}
+		senderID, err := senderIDQuery(roomID, *parsedStateKey)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to query sender ID: %s", err.Error())
+		}
+		if senderID == nil {
+			// If no sender ID, then there is no associated state event.
+			return nil, fmt.Errorf("No associated sender ID found.")
+		}
+		newStateKey := string(*senderID)
+		return &newStateKey, nil
+	} else {
+		return &stateKey, nil
+	}
+}

From 1c4ec67bb6897859617fea263c658142a694e26d Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Thu, 24 Aug 2023 21:08:40 +0000
Subject: [PATCH 14/57] Add configuration option for sliding sync when hosting
 /.well-known/matrix/client (#3189)

Adds the `org.matrix.msc3575.proxy` field (used for configuring sliding
sync) to /.well-known/matrix/client when Dendrite is serving that
endpoint and `well_known_sliding_sync_proxy` has been configured.

ie. Config values of:
``` yaml
global:
    well_known_client_name: https://example.com
    well_known_sliding_sync_proxy: https://syncv3.example.com
```
results in a /.well-known/matrix/client of:
``` json
{
    "m.homeserver": {
        "base_url": "https://example.com"
    },
    "org.matrix.msc3575.proxy": {
        "url": "https://syncv3.example.com"
    }
}
```

If `well_known_sliding_sync_proxy` is not provided, the json provided by
/.well-known/matrix/client does not include the proxy field.
ie.
``` json
{
    "m.homeserver": {
        "base_url": "https://example.com"
    }
}
```
---
 clientapi/routing/routing.go  | 37 ++++++++++++++++++++++++-----------
 setup/config/config_global.go |  4 ++++
 2 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 8b3ae5e1e..d4aa1d08d 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -44,6 +44,19 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 )
 
+type WellKnownClientHomeserver struct {
+	BaseUrl string `json:"base_url"`
+}
+
+type WellKnownSlidingSyncProxy struct {
+	Url string `json:"url"`
+}
+
+type WellKnownClientResponse struct {
+	Homeserver       WellKnownClientHomeserver  `json:"m.homeserver"`
+	SlidingSyncProxy *WellKnownSlidingSyncProxy `json:"org.matrix.msc3575.proxy,omitempty"`
+}
+
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
 //
@@ -96,20 +109,22 @@ func Setup(
 
 	if cfg.Matrix.WellKnownClientName != "" {
 		logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName)
+		if cfg.Matrix.WellKnownSlidingSyncProxy != "" {
+			logrus.Infof("Setting org.matrix.msc3575.proxy url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownSlidingSyncProxy)
+		}
 		wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse {
+			response := WellKnownClientResponse{
+				Homeserver: WellKnownClientHomeserver{cfg.Matrix.WellKnownClientName},
+			}
+			if cfg.Matrix.WellKnownSlidingSyncProxy != "" {
+				response.SlidingSyncProxy = &WellKnownSlidingSyncProxy{
+					Url: cfg.Matrix.WellKnownSlidingSyncProxy,
+				}
+			}
+
 			return util.JSONResponse{
 				Code: http.StatusOK,
-				JSON: struct {
-					HomeserverName struct {
-						BaseUrl string `json:"base_url"`
-					} `json:"m.homeserver"`
-				}{
-					HomeserverName: struct {
-						BaseUrl string `json:"base_url"`
-					}{
-						BaseUrl: cfg.Matrix.WellKnownClientName,
-					},
-				},
+				JSON: response,
 			}
 		})).Methods(http.MethodGet, http.MethodOptions)
 	}
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index 1622bf357..5b4ccf400 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -48,6 +48,10 @@ type Global struct {
 	// The server name to delegate client-server communications to, with optional port
 	WellKnownClientName string `yaml:"well_known_client_name"`
 
+	// The server name to delegate sliding sync communications to, with optional port.
+	// Requires `well_known_client_name` to also be configured.
+	WellKnownSlidingSyncProxy string `yaml:"well_known_sliding_sync_proxy"`
+
 	// Disables federation. Dendrite will not be able to make any outbound HTTP requests
 	// to other servers and the federation API will not be exposed.
 	DisableFederation bool `yaml:"disable_federation"`

From 43b1ddb89bc08849c77bac0a5f1b030722732780 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Aug 2023 12:51:47 +0200
Subject: [PATCH 15/57] Bump commonmarker from 0.23.9 to 0.23.10 in /docs
 (#3172)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [commonmarker](https://github.com/gjtorikian/commonmarker) from
0.23.9 to 0.23.10.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/gjtorikian/commonmarker/releases">commonmarker's
releases</a>.</em></p>
<blockquote>
<h2>v0.23.10</h2>
<h2>What's Changed</h2>
<ul>
<li>Update to 0.29.0.gfm.13 by <a
href="https://github.com/anticomputer"><code>@​anticomputer</code></a>
in <a
href="https://redirect.github.com/gjtorikian/commonmarker/pull/247">gjtorikian/commonmarker#247</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/gjtorikian/commonmarker/compare/v0.23.9...v0.23.10">https://github.com/gjtorikian/commonmarker/compare/v0.23.9...v0.23.10</a></p>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/gjtorikian/commonmarker/blob/v0.23.10/CHANGELOG.md">commonmarker's
changelog</a>.</em></p>
<blockquote>
<h2>[v0.23.10] (2023-07-31)</h2>
<ul>
<li>Update GFM release to <a
href="https://github.com/github/cmark-gfm/releases/tag/0.29.0.gfm.12"><code>0.29.0.gfm.12</code></a>
and <a
href="https://github.com/github/cmark-gfm/releases/tag/0.29.0.gfm.13"><code>0.29.0.gfm.13</code></a>,
thereby <a
href="https://github.com/github/cmark-gfm/security/advisories/GHSA-w4qg-3vf7-m9x5">fixing
a polynomial time complexity security vulnerability</a>.</li>
<li>Of note to users of this library, GFM releases
<code>0.29.0.gfm.12</code> and <code>0.29.0.gfm.13</code> also:
<ul>
<li>Normalized marker row vs. delimiter row nomenclature (<a
href="https://redirect.github.com/github/cmark-gfm/pull/273">#273</a>)</li>
<li>Exposed CMARK_NODE_FOOTNOTE_DEFINITION literal value (<a
href="https://redirect.github.com/github/cmark-gfm/pull/336">#336</a>)</li>
</ul>
</li>
</ul>
<h2><a
href="https://github.com/gjtorikian/commonmarker/tree/v0.23.4">v0.23.4</a>
(2022-03-03)</h2>
<p><a
href="https://github.com/gjtorikian/commonmarker/compare/v0.23.2...v0.23.4">Full
Changelog</a></p>
<p><strong>Fixed bugs:</strong></p>
<ul>
<li><code>#render_html</code> way slower than
<code>#render_doc.to_html</code> <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/141">#141</a></li>
</ul>
<p><strong>Closed issues:</strong></p>
<ul>
<li>allow keeping text content of unknown tags <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/169">#169</a></li>
<li>STRIKETHROUGH_DOUBLE_TILDE not working <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/168">#168</a></li>
<li>Allow disabling 4-space code blocks <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/167">#167</a></li>
<li>tables with escaped pipes are not recognized <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/166">#166</a></li>
</ul>
<p><strong>Merged pull requests:</strong></p>
<ul>
<li>CI: Drop a duplicate 'bundle install' <a
href="https://redirect.github.com/gjtorikian/commonmarker/pull/173">#173</a>
(<a href="https://github.com/olleolleolle">olleolleolle</a>)</li>
<li>CI: Drop duplicate bundle install <a
href="https://redirect.github.com/gjtorikian/commonmarker/pull/172">#172</a>
(<a href="https://github.com/olleolleolle">olleolleolle</a>)</li>
<li>Fixup benchmark and speedup a little, fixes <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/141">#141</a>
<a
href="https://redirect.github.com/gjtorikian/commonmarker/pull/171">#171</a>
(<a href="https://github.com/ojab">ojab</a>)</li>
</ul>
<h2><a
href="https://github.com/gjtorikian/commonmarker/tree/v0.23.2">v0.23.2</a>
(2021-09-17)</h2>
<p><a
href="https://github.com/gjtorikian/commonmarker/compare/v0.23.1...v0.23.2">Full
Changelog</a></p>
<p><strong>Merged pull requests:</strong></p>
<ul>
<li>Update GFM release to <code>0.29.0.gfm.2</code> <a
href="https://redirect.github.com/gjtorikian/commonmarker/pull/148">#148</a>
(<a href="https://github.com/phillmv">phillmv</a>)</li>
</ul>
<h2><a
href="https://github.com/gjtorikian/commonmarker/tree/v0.23.1">v0.23.1</a>
(2021-09-03)</h2>
<p><a
href="https://github.com/gjtorikian/commonmarker/compare/v0.23.0...v0.23.1">Full
Changelog</a></p>
<p><strong>Closed issues:</strong></p>
<ul>
<li>Incorrect processing of list and next block of code <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/146">#146</a></li>
</ul>
<p><strong>Merged pull requests:</strong></p>
<ul>
<li>Normalize parse and render options <a
href="https://redirect.github.com/gjtorikian/commonmarker/pull/145">#145</a>
(<a href="https://github.com/phillmv">phillmv</a>)</li>
</ul>
<h2><a
href="https://github.com/gjtorikian/commonmarker/tree/v0.23.0">v0.23.0</a>
(2021-08-30)</h2>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/db8cd377b54541f7fd484d168b7682a282a680f7"><code>db8cd37</code></a>
Merge pull request <a
href="https://redirect.github.com/gjtorikian/commonmarker/issues/247">#247</a>
from anticomputer/update-to-0.29.0.gfm.13</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/e1e450c381e1fac5021a08bdc5f72bbac9cf6038"><code>e1e450c</code></a>
:gem: release 0.23.10</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/08b7c4b96c2835edcc2f14e978f758f6ac58b158"><code>08b7c4b</code></a>
Update cmark-upstream to <a
href="https://github.com/github/cmark-gfm/commit/587a12bb5">https://github.com/github/cmark-gfm/commit/587a12bb5</a>...</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/d0e81e2392861ad3f6f27e895080cdfc6a8e72e7"><code>d0e81e2</code></a>
I've used this version of the update_submodules script for several
releases, ...</li>
<li>See full diff in <a
href="https://github.com/gjtorikian/commonmarker/compare/v0.23.9...v0.23.10">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commonmarker&package-manager=bundler&previous-version=0.23.9&new-version=0.23.10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

[skip ci]
---
 docs/Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
index 0901965c1..195f60c6f 100644
--- a/docs/Gemfile.lock
+++ b/docs/Gemfile.lock
@@ -14,7 +14,7 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.9)
+    commonmarker (0.23.10)
     concurrent-ruby (1.2.0)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)

From e3a7039c81ae7a123bb705585cfea8c93910d381 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 28 Aug 2023 13:28:22 +0200
Subject: [PATCH 16/57] Fix CI, upgrade image used for upgrade tests (#3151)

---
 .github/workflows/dendrite.yml     |  4 ++++
 cmd/dendrite-upgrade-tests/main.go | 20 ++++++++++----------
 go.mod                             |  2 +-
 go.sum                             |  4 ++--
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 83701c50c..772b45cb2 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -280,6 +280,8 @@ jobs:
         with:
           go-version: "stable"
           cache: true
+      - name: Docker version
+        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
@@ -300,6 +302,8 @@ jobs:
         with:
           go-version: "stable"
           cache: true
+      - name: Docker version
+        run: docker version
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
       - name: Test upgrade (PostgreSQL)
diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index dcc45bdcc..68919e525 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -55,7 +55,7 @@ var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD"
 // due to the error:
 // When using COPY with more than one source file, the destination must be a directory and end with a /
 // We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
-const DockerfilePostgreSQL = `FROM golang:1.18-buster as build
+const DockerfilePostgreSQL = `FROM golang:1.20-bookworm as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 ARG BINARY
@@ -74,16 +74,16 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key
 # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password
 RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml
 # No password when connecting over localhost
-RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/11/main/pg_hba.conf
+RUN sed -i "s%127.0.0.1/32            scram-sha-256%127.0.0.1/32            trust%g" /etc/postgresql/15/main/pg_hba.conf
 # Bump up max conns for moar concurrency
-RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/11/main/postgresql.conf
+RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/15/main/postgresql.conf
 RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
 #!/bin/bash -eu \n\
 pg_lsclusters \n\
-pg_ctlcluster 11 main start \n\
+pg_ctlcluster 15 main start \n\
  \n\
 until pg_isready \n\
 do \n\
@@ -101,7 +101,7 @@ ENV BINARY=dendrite
 EXPOSE 8008 8448
 CMD /build/run_dendrite.sh`
 
-const DockerfileSQLite = `FROM golang:1.18-buster as build
+const DockerfileSQLite = `FROM golang:1.20-bookworm as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 ARG BINARY
@@ -119,7 +119,7 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key
 
 # Make sure the SQLite databases are in a persistent location, we're already mapping
 # the postgresql folder so let's just use that for simplicity
-RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/11\/main\/%g" dendrite.yaml
+RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/15\/main\/%g" dendrite.yaml
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
@@ -402,7 +402,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI
 			{
 				Type:   mount.TypeVolume,
 				Source: volumeName,
-				Target: "/var/lib/postgresql/11/main",
+				Target: "/var/lib/postgresql/15/main",
 			},
 		},
 	}, nil, nil, "dendrite_upgrade_test_"+branchName)
@@ -557,8 +557,8 @@ func cleanup(dockerClient *client.Client) {
 	})
 	for _, c := range containers {
 		log.Printf("Removing container: %v %v\n", c.ID, c.Names)
-		s := time.Second
-		_ = dockerClient.ContainerStop(context.Background(), c.ID, &s)
+		timeout := 1
+		_ = dockerClient.ContainerStop(context.Background(), c.ID, container.StopOptions{Timeout: &timeout})
 		_ = dockerClient.ContainerRemove(context.Background(), c.ID, types.ContainerRemoveOptions{
 			Force: true,
 		})
@@ -592,7 +592,7 @@ func main() {
 	branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions)
 
 	// make a shared postgres volume
-	volume, err := dockerClient.VolumeCreate(context.Background(), volume.VolumeCreateBody{
+	volume, err := dockerClient.VolumeCreate(context.Background(), volume.CreateOptions{
 		Name: "dendrite_upgrade_test",
 		Labels: map[string]string{
 			dendriteUpgradeTestLabel: "yes",
diff --git a/go.mod b/go.mod
index 4be0ede49..710b50376 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/blevesearch/bleve/v2 v2.3.8
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1
-	github.com/docker/docker v20.10.24+incompatible
+	github.com/docker/docker v24.0.5+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
 	github.com/gologme/log v1.3.0
diff --git a/go.sum b/go.sum
index 3fdfe01a4..863caee72 100644
--- a/go.sum
+++ b/go.sum
@@ -89,8 +89,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE=
-github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY=
+github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=

From b538f237df0b78634d3b2f092309faeca102ed36 Mon Sep 17 00:00:00 2001
From: Omar Pakker <Omar007@users.noreply.github.com>
Date: Tue, 29 Aug 2023 08:20:37 +0200
Subject: [PATCH 17/57] [helm] Update Ingress hosts to account for IPv6
 (server+client) and scheme (client) (#3182)

This updates the matchers for deriving the host values from the dendrite
config. The original version turned out to have 2 complications:
- It did not support IPv6 addresses as host value
- It failed for `well_known_client_host` which is a (base) URL instead
of a hostname+port.

I've verified `well_known_server_name` with
```
dendrite.example.net:443
dendrite.example.net
192.168.1.1
192.168.1.1:1324
[dead::beef]:1234
[dead::beef]
[ffff:dead::beef]
```
and `well_known_client_name` with:
```
https://dendrite.example.net:443
https://dendrite.example.net
https://dendrite.example.net/
http://dendrite.example.net:8080/
http://192.168.1.1
http://192.168.1.1:8080/
http://[dead::beef]:1234
http://[dead::beef]/
http://[ffff:dead::beef]
```

Fixes #3175

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `Omar Pakker <Omar007@users.noreply.github.com>`

---------

Signed-off-by: Omar Pakker <Omar007@users.noreply.github.com>

[skip CI]
---
 helm/dendrite/Chart.yaml                |  2 +-
 helm/dendrite/templates/deployment.yaml |  9 +--------
 helm/dendrite/templates/ingress.yaml    |  4 ++--
 helm/dendrite/values.yaml               | 10 ----------
 4 files changed, 4 insertions(+), 21 deletions(-)

diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index 46be9f781..5590a39b1 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.2"
+version: "0.13.3"
 appVersion: "0.13.2"
 description: Dendrite Matrix Homeserver
 type: application
diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml
index 3a0bd68d8..e3f84cdae 100644
--- a/helm/dendrite/templates/deployment.yaml
+++ b/helm/dendrite/templates/deployment.yaml
@@ -26,13 +26,6 @@ spec:
       annotations:
         confighash: secret-{{ .Values.dendrite_config | toYaml | sha256sum | trunc 32 }}
     spec:
-      strategy:
-        type: {{ $.Values.strategy.type }}
-        {{- if eq $.Values.strategy.type "RollingUpdate" }}
-        rollingUpdate:
-          maxSurge: {{ $.Values.strategy.rollingUpdate.maxSurge }}
-          maxUnavailable: {{ $.Values.strategy.rollingUpdate.maxUnavailable }}
-        {{- end }}
       volumes:
       - name: {{ include "dendrite.fullname" . }}-conf-vol
         secret:
@@ -116,4 +109,4 @@ spec:
           failureThreshold: 10
           httpGet:
             path: /_dendrite/monitor/up
-            port: http
\ No newline at end of file
+            port: http
diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml
index 9ef413dc9..4bcaee12d 100644
--- a/helm/dendrite/templates/ingress.yaml
+++ b/helm/dendrite/templates/ingress.yaml
@@ -1,8 +1,8 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "dendrite.fullname" . -}}
 {{- $serverNameHost := .Values.dendrite_config.global.server_name -}}
-{{- $wellKnownServerHost := default $serverNameHost (regexFind "^[^:]+" .Values.dendrite_config.global.well_known_server_name) -}}
-{{- $wellKnownClientHost := default $serverNameHost (regexFind "^[^:]+" .Values.dendrite_config.global.well_known_client_name) -}}
+{{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}}
+{{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}}
 {{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml
index 396e70319..8a72f6693 100644
--- a/helm/dendrite/values.yaml
+++ b/helm/dendrite/values.yaml
@@ -65,16 +65,6 @@ extraVolumeMounts: []
 # - mountPath: /etc/dendrite/extra-config
 #   name: extra-config
 
-strategy:
-  # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
-  # If you are using ReadWriteOnce volumes, you should probably use Recreate
-  type: RollingUpdate
-  rollingUpdate:
-    # -- Maximum number of pods that can be unavailable during the update process
-    maxUnavailable: 25%
-    # -- Maximum number of pods that can be scheduled above the desired number of pods
-    maxSurge: 25%
-
 strategy:
   # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
   # If you are using ReadWriteOnce volumes, you should probably use Recreate

From 11fd2f019bb6325155c2fa825b82c1fbef07b300 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 30 Aug 2023 07:37:14 +0200
Subject: [PATCH 18/57] Fix Complement scheduled CI

[skip CI]
---
 .github/workflows/schedules.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml
index e76cc82f3..509861860 100644
--- a/.github/workflows/schedules.yaml
+++ b/.github/workflows/schedules.yaml
@@ -128,7 +128,7 @@ jobs:
         # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
       - name: Run actions/checkout@v3 for dendrite
         uses: actions/checkout@v3
         with:

From bb2ab62cbf02abb6f600e6eb39fde67aa2ff3215 Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Thu, 31 Aug 2023 15:33:38 +0000
Subject: [PATCH 19/57] Handle event_format federation in /sync responses
 (#3192)

---
 clientapi/routing/state.go            |   2 +-
 internal/eventutil/events.go          |   2 +-
 syncapi/consumers/roomserver.go       |  11 +-
 syncapi/routing/getevent.go           |   2 +-
 syncapi/routing/relations.go          |   2 +-
 syncapi/routing/search.go             |   2 +-
 syncapi/streams/stream_invite.go      |   7 +-
 syncapi/streams/stream_pdu.go         |  67 ++--
 syncapi/syncapi_test.go               | 150 ++++++++
 syncapi/synctypes/clientevent.go      |  79 ++++-
 syncapi/synctypes/clientevent_test.go | 473 ++++++++++++++++++++++++--
 syncapi/synctypes/filter.go           |   7 +-
 syncapi/types/types.go                |  11 +-
 syncapi/types/types_test.go           |   2 +-
 userapi/consumers/roomserver.go       |   4 +-
 userapi/util/notify_test.go           |   2 +-
 16 files changed, 739 insertions(+), 84 deletions(-)

diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index 7648dc474..d7f0b40f8 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -193,7 +193,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 			}
 			stateEvents = append(
 				stateEvents,
-				synctypes.ToClientEvent(ev, synctypes.FormatAll, sender, sk),
+				synctypes.ToClientEvent(ev, synctypes.FormatAll, sender.String(), sk, ev.Unsigned()),
 			)
 		}
 	}
diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go
index 56ee576a0..aa99e5860 100644
--- a/internal/eventutil/events.go
+++ b/internal/eventutil/events.go
@@ -184,7 +184,7 @@ func RedactEvent(ctx context.Context, redactionEvent, redactedEvent gomatrixserv
 	if err != nil {
 		return err
 	}
-	redactedBecause := synctypes.ToClientEvent(redactionEvent, synctypes.FormatSync, *senderID, redactionEvent.StateKey())
+	redactedBecause := synctypes.ToClientEvent(redactionEvent, synctypes.FormatSync, senderID.String(), redactionEvent.StateKey(), redactionEvent.Unsigned())
 	if err := redactedEvent.SetUnsignedField("redacted_because", redactedBecause); err != nil {
 		return err
 	}
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 1e87aee99..9df5e0f9c 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -33,6 +33,7 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/streams"
+	"github.com/matrix-org/dendrite/syncapi/synctypes"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
@@ -592,16 +593,10 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent)
 		return event, nil
 	}
 
-	prevEventSender := string(prevEvent.SenderID())
-	prevUser, err := s.rsAPI.QueryUserIDForSender(s.ctx, *validRoomID, prevEvent.SenderID())
-	if err == nil && prevUser != nil {
-		prevEventSender = prevUser.String()
-	}
-
-	prev := types.PrevEventRef{
+	prev := synctypes.PrevEventRef{
 		PrevContent:   prevEvent.Content(),
 		ReplacesState: prevEvent.EventID(),
-		PrevSenderID:  prevEventSender,
+		PrevSenderID:  string(prevEvent.SenderID()),
 	}
 
 	event.PDU, err = event.SetUnsigned(prev)
diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go
index 4fa282f3b..bf0f9bf8c 100644
--- a/syncapi/routing/getevent.go
+++ b/syncapi/routing/getevent.go
@@ -144,6 +144,6 @@ func GetEvent(
 	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: synctypes.ToClientEvent(events[0], synctypes.FormatAll, *senderUserID, sk),
+		JSON: synctypes.ToClientEvent(events[0], synctypes.FormatAll, senderUserID.String(), sk, events[0].Unsigned()),
 	}
 }
diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go
index e3d1069a0..b451a7e2e 100644
--- a/syncapi/routing/relations.go
+++ b/syncapi/routing/relations.go
@@ -146,7 +146,7 @@ func Relations(
 		}
 		res.Chunk = append(
 			res.Chunk,
-			synctypes.ToClientEvent(event.PDU, synctypes.FormatAll, sender, sk),
+			synctypes.ToClientEvent(event.PDU, synctypes.FormatAll, sender.String(), sk, event.Unsigned()),
 		)
 	}
 
diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go
index d892b604a..7d5c061b7 100644
--- a/syncapi/routing/search.go
+++ b/syncapi/routing/search.go
@@ -267,7 +267,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 				ProfileInfo: profileInfos,
 			},
 			Rank:   eventScore[event.EventID()].Score,
-			Result: synctypes.ToClientEvent(event, synctypes.FormatAll, sender, sk),
+			Result: synctypes.ToClientEvent(event, synctypes.FormatAll, sender.String(), sk, event.Unsigned()),
 		})
 		roomGroup := groups[event.RoomID()]
 		roomGroup.Results = append(roomGroup.Results, event.EventID())
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 7c29d84ae..1ce3346f4 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -63,6 +63,11 @@ func (p *InviteStreamProvider) IncrementalSync(
 		return from
 	}
 
+	eventFormat := synctypes.FormatSync
+	if req.Filter.EventFormat == synctypes.EventFormatFederation {
+		eventFormat = synctypes.FormatSyncFederation
+	}
+
 	for roomID, inviteEvent := range invites {
 		user := spec.UserID{}
 		validRoomID, err := spec.NewRoomID(inviteEvent.RoomID())
@@ -87,7 +92,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 		if _, ok := req.IgnoredUsers.List[user.String()]; ok {
 			continue
 		}
-		ir := types.NewInviteResponse(inviteEvent, user, sk)
+		ir := types.NewInviteResponse(inviteEvent, user, sk, eventFormat)
 		req.Response.Rooms.Invite[roomID] = ir
 	}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 4622c21ad..ee524f726 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -88,6 +88,11 @@ func (p *PDUStreamProvider) CompleteSync(
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
 	}
 
+	eventFormat := synctypes.FormatSync
+	if req.Filter.EventFormat == synctypes.EventFormatFederation {
+		eventFormat = synctypes.FormatSyncFederation
+	}
+
 	recentEvents, err := snapshot.RecentEvents(ctx, joinedRoomIDs, r, &eventFilter, true, true)
 	if err != nil {
 		return from
@@ -105,7 +110,7 @@ func (p *PDUStreamProvider) CompleteSync(
 		// get the join response for each room
 		jr, jerr := p.getJoinResponseForCompleteSync(
 			ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, false,
-			events.Events, events.Limited,
+			events.Events, events.Limited, eventFormat,
 		)
 		if jerr != nil {
 			req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
@@ -142,7 +147,7 @@ func (p *PDUStreamProvider) CompleteSync(
 			events := recentEvents[roomID]
 			jr, err = p.getJoinResponseForCompleteSync(
 				ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, true,
-				events.Events, events.Limited,
+				events.Events, events.Limited, eventFormat,
 			)
 			if err != nil {
 				req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed")
@@ -346,6 +351,11 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err)
 	}
 
+	eventFormat := synctypes.FormatSync
+	if req.Filter.EventFormat == synctypes.EventFormatFederation {
+		eventFormat = synctypes.FormatSyncFederation
+	}
+
 	// Now that we've filtered the timeline, work out which state events are still
 	// left. Anything that appears in the filtered timeline will be removed from the
 	// "state" section and kept in "timeline".
@@ -359,7 +369,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 			continue
 		}
 		var newEvent gomatrixserverlib.PDU
-		newEvent, err = p.updatePowerLevelEvent(ctx, ev)
+		newEvent, err = p.updatePowerLevelEvent(ctx, ev, eventFormat)
 		if err != nil {
 			return r.From, err
 		}
@@ -383,7 +393,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		// update the powerlevel event for state events
 		if ev.Version() == gomatrixserverlib.RoomVersionPseudoIDs && ev.Type() == spec.MRoomPowerLevels && ev.StateKeyEquals("") {
 			var newEvent gomatrixserverlib.PDU
-			newEvent, err = p.updatePowerLevelEvent(ctx, he)
+			newEvent, err = p.updatePowerLevelEvent(ctx, he, eventFormat)
 			if err != nil {
 				return r.From, err
 			}
@@ -413,13 +423,13 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 			}
 		}
 		jr.Timeline.PrevBatch = &prevBatch
-		jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		})
 		// If we are limited by the filter AND the history visibility filter
 		// didn't "remove" events, return that the response is limited.
 		jr.Timeline.Limited = (limited && len(events) == len(recentEvents)) || delta.NewlyJoined
-		jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		})
 		req.Response.Rooms.Join[delta.RoomID] = jr
@@ -428,11 +438,11 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		jr := types.NewJoinResponse()
 		jr.Timeline.PrevBatch = &prevBatch
 		// TODO: Apply history visibility on peeked rooms
-		jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(recentEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(recentEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		})
 		jr.Timeline.Limited = limited
-		jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		})
 		req.Response.Rooms.Peek[delta.RoomID] = jr
@@ -443,13 +453,13 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	case spec.Ban:
 		lr := types.NewLeaveResponse()
 		lr.Timeline.PrevBatch = &prevBatch
-		lr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		lr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		})
 		// If we are limited by the filter AND the history visibility filter
 		// didn't "remove" events, return that the response is limited.
 		lr.Timeline.Limited = limited && len(events) == len(recentEvents)
-		lr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		lr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		})
 		req.Response.Rooms.Leave[delta.RoomID] = lr
@@ -458,7 +468,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	return latestPosition, nil
 }
 
-func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstypes.HeaderedEvent) (gomatrixserverlib.PDU, error) {
+func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstypes.HeaderedEvent, eventFormat synctypes.ClientEventFormat) (gomatrixserverlib.PDU, error) {
 	pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(ev)
 	if err != nil {
 		return nil, err
@@ -467,11 +477,14 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 	var userID *spec.UserID
 	for user, level := range pls.Users {
 		validRoomID, _ := spec.NewRoomID(ev.RoomID())
-		userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user))
-		if err != nil {
-			return nil, err
+		if eventFormat != synctypes.FormatSyncFederation {
+			userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user))
+			if err != nil {
+				return nil, err
+			}
+			user = userID.String()
 		}
-		newPls[userID.String()] = level
+		newPls[user] = level
 	}
 	var newPlBytes, newEv []byte
 	newPlBytes, err = json.Marshal(newPls)
@@ -487,7 +500,7 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 	prevContent := gjson.GetBytes(ev.JSON(), "unsigned.prev_content")
 	if !prevContent.Exists() {
 		var evNew gomatrixserverlib.PDU
-		evNew, err = gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON(newEv, false)
+		evNew, err = gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSON(newEv, false)
 		if err != nil {
 			return nil, err
 		}
@@ -503,11 +516,14 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 	newPls = make(map[string]int64)
 	for user, level := range pls.Users {
 		validRoomID, _ := spec.NewRoomID(ev.RoomID())
-		userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user))
-		if err != nil {
-			return nil, err
+		if eventFormat != synctypes.FormatSyncFederation {
+			userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user))
+			if err != nil {
+				return nil, err
+			}
+			user = userID.String()
 		}
-		newPls[userID.String()] = level
+		newPls[user] = level
 	}
 	newPlBytes, err = json.Marshal(newPls)
 	if err != nil {
@@ -519,7 +535,7 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 	}
 
 	var evNew gomatrixserverlib.PDU
-	evNew, err = gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSONWithEventID(ev.EventID(), newEv, false)
+	evNew, err = gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSONWithEventID(ev.EventID(), newEv, false)
 	if err != nil {
 		return nil, err
 	}
@@ -592,6 +608,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	isPeek bool,
 	recentStreamEvents []types.StreamEvent,
 	limited bool,
+	eventFormat synctypes.ClientEventFormat,
 ) (jr *types.JoinResponse, err error) {
 	jr = types.NewJoinResponse()
 	// TODO: When filters are added, we may need to call this multiple times to get enough events.
@@ -683,7 +700,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") {
 			continue
 		}
-		newEvent, err := p.updatePowerLevelEvent(ctx, ev)
+		newEvent, err := p.updatePowerLevelEvent(ctx, ev, eventFormat)
 		if err != nil {
 			return nil, err
 		}
@@ -697,7 +714,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") {
 			continue
 		}
-		newEvent, err := p.updatePowerLevelEvent(ctx, ev)
+		newEvent, err := p.updatePowerLevelEvent(ctx, ev, eventFormat)
 		if err != nil {
 			return nil, err
 		}
@@ -705,13 +722,13 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	}
 
 	jr.Timeline.PrevBatch = prevBatch
-	jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 		return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 	})
 	// If we are limited by the filter AND the history visibility filter
 	// didn't "remove" events, return that the response is limited.
 	jr.Timeline.Limited = limited && len(events) == len(recentEvents)
-	jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(stateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(stateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 		return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 	})
 	return jr, nil
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index ea1183cd2..f29719953 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -209,6 +209,156 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
 	}
 }
 
+func TestSyncAPIEventFormatPowerLevels(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testSyncEventFormatPowerLevels(t, dbType)
+	})
+}
+
+func testSyncEventFormatPowerLevels(t *testing.T, dbType test.DBType) {
+	user := test.NewUser(t)
+	setRoomVersion := func(t *testing.T, r *test.Room) { r.Version = gomatrixserverlib.RoomVersionPseudoIDs }
+	room := test.NewRoom(t, user, setRoomVersion)
+	alice := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      user.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "Alice",
+		AccountType: userapi.AccountTypeUser,
+	}
+
+	room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{
+		Users: map[string]int64{
+			user.ID: 100,
+		},
+	}, test.WithStateKey(""))
+
+	cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+	routers := httputil.NewRouters()
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	natsInstance := jetstream.NATSInstance{}
+	defer close()
+
+	jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+	msgs := toNATSMsgs(t, cfg, room.Events()...)
+	AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics)
+	testrig.MustPublishMsgs(t, jsctx, msgs...)
+
+	testCases := []struct {
+		name            string
+		wantCode        int
+		wantJoinedRooms []string
+		eventFormat     synctypes.ClientEventFormat
+	}{
+		{
+			name:            "Client format",
+			wantCode:        200,
+			wantJoinedRooms: []string{room.ID},
+			eventFormat:     synctypes.FormatSync,
+		},
+		{
+			name:            "Federation format",
+			wantCode:        200,
+			wantJoinedRooms: []string{room.ID},
+			eventFormat:     synctypes.FormatSyncFederation,
+		},
+	}
+
+	syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool {
+		// wait for the last sent eventID to come down sync
+		path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID())
+		return gjson.Get(syncBody, path).Exists()
+	})
+
+	for _, tc := range testCases {
+		format := ""
+		if tc.eventFormat == synctypes.FormatSyncFederation {
+			format = "federation"
+		}
+
+		w := httptest.NewRecorder()
+		routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+			"access_token": alice.AccessToken,
+			"timeout":      "0",
+			"filter":       fmt.Sprintf(`{"event_format":"%s"}`, format),
+		})))
+		if w.Code != tc.wantCode {
+			t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode)
+		}
+		if tc.wantJoinedRooms != nil {
+			var res types.Response
+			if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+				t.Fatalf("%s: failed to decode response body: %s", tc.name, err)
+			}
+			if len(res.Rooms.Join) != len(tc.wantJoinedRooms) {
+				t.Errorf("%s: got %v joined rooms, want %v.\nResponse: %+v", tc.name, len(res.Rooms.Join), len(tc.wantJoinedRooms), res)
+			}
+			t.Logf("res: %+v", res.Rooms.Join[room.ID])
+
+			gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events))
+			for i, ev := range res.Rooms.Join[room.ID].Timeline.Events {
+				gotEventIDs[i] = ev.EventID
+			}
+			test.AssertEventIDsEqual(t, gotEventIDs, room.Events())
+
+			event := room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{
+				Users: map[string]int64{
+					user.ID:                100,
+					"@otheruser:localhost": 50,
+				},
+			}, test.WithStateKey(""))
+
+			msgs := toNATSMsgs(t, cfg, event)
+			testrig.MustPublishMsgs(t, jsctx, msgs...)
+
+			syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool {
+				// wait for the last sent eventID to come down sync
+				path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID())
+				return gjson.Get(syncBody, path).Exists()
+			})
+
+			since := res.NextBatch.String()
+			w := httptest.NewRecorder()
+			routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+				"access_token": alice.AccessToken,
+				"timeout":      "0",
+				"filter":       fmt.Sprintf(`{"event_format":"%s"}`, format),
+				"since":        since,
+			})))
+			if w.Code != 200 {
+				t.Errorf("since=%s got HTTP %d want 200", since, w.Code)
+			}
+
+			res = *types.NewResponse()
+			if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+				t.Errorf("failed to decode response body: %s", err)
+			}
+			if len(res.Rooms.Join) != 1 {
+				t.Fatalf("since=%s got %d joined rooms, want 1", since, len(res.Rooms.Join))
+			}
+			gotEventIDs = make([]string, len(res.Rooms.Join[room.ID].Timeline.Events))
+			for j, ev := range res.Rooms.Join[room.ID].Timeline.Events {
+				gotEventIDs[j] = ev.EventID
+				if ev.Type == spec.MRoomPowerLevels {
+					content := gomatrixserverlib.PowerLevelContent{}
+					err := json.Unmarshal(ev.Content, &content)
+					if err != nil {
+						t.Errorf("failed to unmarshal power level content: %s", err)
+					}
+					otherUserLevel := content.UserLevel("@otheruser:localhost")
+					if otherUserLevel != 50 {
+						t.Errorf("Expected user PL of %d but got %d", 50, otherUserLevel)
+					}
+				}
+			}
+			events := []*rstypes.HeaderedEvent{room.Events()[len(room.Events())-1]}
+			test.AssertEventIDsEqual(t, gotEventIDs, events)
+		}
+	}
+}
+
 // Tests what happens when we create a room and then /sync before all events from /createRoom have
 // been sent to the syncapi
 func TestSyncAPICreateRoomSyncEarly(t *testing.T) {
diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go
index a78aea1c6..7e5b1c1bc 100644
--- a/syncapi/synctypes/clientevent.go
+++ b/syncapi/synctypes/clientevent.go
@@ -16,12 +16,21 @@
 package synctypes
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/sirupsen/logrus"
 )
 
+// PrevEventRef represents a reference to a previous event in a state event upgrade
+type PrevEventRef struct {
+	PrevContent   json.RawMessage `json:"prev_content"`
+	ReplacesState string          `json:"replaces_state"`
+	PrevSenderID  string          `json:"prev_sender"`
+}
+
 type ClientEventFormat int
 
 const (
@@ -30,8 +39,21 @@ const (
 	// FormatSync will include only the event keys required by the /sync API. Notably, this
 	// means the 'room_id' will be missing from the events.
 	FormatSync
+	// FormatSyncFederation will include all event keys normally included in federated events.
+	// This allows clients to request federated formatted events via the /sync API.
+	FormatSyncFederation
 )
 
+// ClientFederationFields extends a ClientEvent to contain the additional fields present in a
+// federation event. Used when the client requests `event_format` of type `federation`.
+type ClientFederationFields struct {
+	Depth      int64        `json:"depth,omitempty"`
+	PrevEvents []string     `json:"prev_events,omitempty"`
+	AuthEvents []string     `json:"auth_events,omitempty"`
+	Signatures spec.RawJSON `json:"signatures,omitempty"`
+	Hashes     spec.RawJSON `json:"hashes,omitempty"`
+}
+
 // ClientEvent is an event which is fit for consumption by clients, in accordance with the specification.
 type ClientEvent struct {
 	Content        spec.RawJSON   `json:"content"`
@@ -44,6 +66,9 @@ type ClientEvent struct {
 	Type           string         `json:"type"`
 	Unsigned       spec.RawJSON   `json:"unsigned,omitempty"`
 	Redacts        string         `json:"redacts,omitempty"`
+
+	// Only sent to clients when `event_format` == `federation`.
+	ClientFederationFields
 }
 
 // ToClientEvents converts server events to client events.
@@ -53,6 +78,11 @@ func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat,
 		if se == nil {
 			continue // TODO: shouldn't happen?
 		}
+		if format == FormatSyncFederation {
+			evs = append(evs, ToClientEvent(se, format, string(se.SenderID()), se.StateKey(), spec.RawJSON(se.Unsigned())))
+			continue
+		}
+
 		sender := spec.UserID{}
 		validRoomID, err := spec.NewRoomID(se.RoomID())
 		if err != nil {
@@ -71,28 +101,61 @@ func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat,
 				sk = &skString
 			}
 		}
-		evs = append(evs, ToClientEvent(se, format, sender, sk))
+
+		unsigned := se.Unsigned()
+		var prev PrevEventRef
+		if err := json.Unmarshal(se.Unsigned(), &prev); err == nil && prev.PrevSenderID != "" {
+			prevUserID, err := userIDForSender(*validRoomID, spec.SenderID(prev.PrevSenderID))
+			if err == nil && userID != nil {
+				prev.PrevSenderID = prevUserID.String()
+			} else {
+				errString := "userID unknown"
+				if err != nil {
+					errString = err.Error()
+				}
+				logrus.Warnf("Failed to find userID for prev_sender in ClientEvent: %s", errString)
+				// NOTE: Not much can be done here, so leave the previous value in place.
+			}
+			unsigned, err = json.Marshal(prev)
+			if err != nil {
+				logrus.Errorf("Failed to marshal unsigned content for ClientEvent: %s", err.Error())
+				continue
+			}
+		}
+		evs = append(evs, ToClientEvent(se, format, sender.String(), sk, spec.RawJSON(unsigned)))
 	}
 	return evs
 }
 
 // ToClientEvent converts a single server event to a client event.
-func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, sender spec.UserID, stateKey *string) ClientEvent {
+func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, sender string, stateKey *string, unsigned spec.RawJSON) ClientEvent {
 	ce := ClientEvent{
 		Content:        spec.RawJSON(se.Content()),
-		Sender:         sender.String(),
+		Sender:         sender,
 		Type:           se.Type(),
 		StateKey:       stateKey,
-		Unsigned:       spec.RawJSON(se.Unsigned()),
+		Unsigned:       unsigned,
 		OriginServerTS: se.OriginServerTS(),
 		EventID:        se.EventID(),
 		Redacts:        se.Redacts(),
 	}
-	if format == FormatAll {
+
+	switch format {
+	case FormatAll:
 		ce.RoomID = se.RoomID()
+	case FormatSync:
+	case FormatSyncFederation:
+		ce.RoomID = se.RoomID()
+		ce.AuthEvents = se.AuthEventIDs()
+		ce.PrevEvents = se.PrevEventIDs()
+		ce.Depth = se.Depth()
+		// TODO: Set Signatures & Hashes fields
 	}
-	if se.Version() == gomatrixserverlib.RoomVersionPseudoIDs {
-		ce.SenderKey = se.SenderID()
+
+	if format != FormatSyncFederation {
+		if se.Version() == gomatrixserverlib.RoomVersionPseudoIDs {
+			ce.SenderKey = se.SenderID()
+		}
 	}
 	return ce
 }
@@ -118,7 +181,7 @@ func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserver
 			sk = &skString
 		}
 	}
-	return ToClientEvent(event, FormatAll, sender, sk)
+	return ToClientEvent(event, FormatAll, sender.String(), sk, event.Unsigned())
 }
 
 // If provided state key is a user ID (state keys beginning with @ are reserved for this purpose)
diff --git a/syncapi/synctypes/clientevent_test.go b/syncapi/synctypes/clientevent_test.go
index 63c65b2af..202c185f1 100644
--- a/syncapi/synctypes/clientevent_test.go
+++ b/syncapi/synctypes/clientevent_test.go
@@ -18,12 +18,69 @@ package synctypes
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
+const testSenderID = "testSenderID"
+const testUserID = "@test:localhost"
+
+type EventFieldsToVerify struct {
+	EventID        string
+	Type           string
+	OriginServerTS spec.Timestamp
+	StateKey       *string
+	Content        spec.RawJSON
+	Unsigned       spec.RawJSON
+	Sender         string
+	Depth          int64
+	PrevEvents     []string
+	AuthEvents     []string
+}
+
+func verifyEventFields(t *testing.T, got EventFieldsToVerify, want EventFieldsToVerify) {
+	if got.EventID != want.EventID {
+		t.Errorf("ClientEvent.EventID: wanted %s, got %s", want.EventID, got.EventID)
+	}
+	if got.OriginServerTS != want.OriginServerTS {
+		t.Errorf("ClientEvent.OriginServerTS: wanted %d, got %d", want.OriginServerTS, got.OriginServerTS)
+	}
+	if got.StateKey == nil && want.StateKey != nil {
+		t.Errorf("ClientEvent.StateKey: no state key present when one was wanted: %s", *want.StateKey)
+	}
+	if got.StateKey != nil && want.StateKey == nil {
+		t.Errorf("ClientEvent.StateKey: state key present when one was not wanted: %s", *got.StateKey)
+	}
+	if got.StateKey != nil && want.StateKey != nil && *got.StateKey != *want.StateKey {
+		t.Errorf("ClientEvent.StateKey: wanted %s, got %s", *want.StateKey, *got.StateKey)
+	}
+	if got.Type != want.Type {
+		t.Errorf("ClientEvent.Type: wanted %s, got %s", want.Type, got.Type)
+	}
+	if !bytes.Equal(got.Content, want.Content) {
+		t.Errorf("ClientEvent.Content: wanted %s, got %s", string(want.Content), string(got.Content))
+	}
+	if !bytes.Equal(got.Unsigned, want.Unsigned) {
+		t.Errorf("ClientEvent.Unsigned: wanted %s, got %s", string(want.Unsigned), string(got.Unsigned))
+	}
+	if got.Sender != want.Sender {
+		t.Errorf("ClientEvent.Sender: wanted %s, got %s", want.Sender, got.Sender)
+	}
+	if got.Depth != want.Depth {
+		t.Errorf("ClientEvent.Depth: wanted %d, got %d", want.Depth, got.Depth)
+	}
+	if !reflect.DeepEqual(got.PrevEvents, want.PrevEvents) {
+		t.Errorf("ClientEvent.PrevEvents: wanted %v, got %v", want.PrevEvents, got.PrevEvents)
+	}
+	if !reflect.DeepEqual(got.AuthEvents, want.AuthEvents) {
+		t.Errorf("ClientEvent.AuthEvents: wanted %v, got %v", want.AuthEvents, got.AuthEvents)
+	}
+}
+
 func TestToClientEvent(t *testing.T) { // nolint: gocyclo
 	ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV1).NewEventFromTrustedJSON([]byte(`{
 		"type": "m.room.name",
@@ -49,28 +106,28 @@ func TestToClientEvent(t *testing.T) { // nolint: gocyclo
 		t.Fatalf("failed to create userID: %s", err)
 	}
 	sk := ""
-	ce := ToClientEvent(ev, FormatAll, *userID, &sk)
-	if ce.EventID != ev.EventID() {
-		t.Errorf("ClientEvent.EventID: wanted %s, got %s", ev.EventID(), ce.EventID)
-	}
-	if ce.OriginServerTS != ev.OriginServerTS() {
-		t.Errorf("ClientEvent.OriginServerTS: wanted %d, got %d", ev.OriginServerTS(), ce.OriginServerTS)
-	}
-	if ce.StateKey == nil || *ce.StateKey != "" {
-		t.Errorf("ClientEvent.StateKey: wanted '', got %v", ce.StateKey)
-	}
-	if ce.Type != ev.Type() {
-		t.Errorf("ClientEvent.Type: wanted %s, got %s", ev.Type(), ce.Type)
-	}
-	if !bytes.Equal(ce.Content, ev.Content()) {
-		t.Errorf("ClientEvent.Content: wanted %s, got %s", string(ev.Content()), string(ce.Content))
-	}
-	if !bytes.Equal(ce.Unsigned, ev.Unsigned()) {
-		t.Errorf("ClientEvent.Unsigned: wanted %s, got %s", string(ev.Unsigned()), string(ce.Unsigned))
-	}
-	if ce.Sender != userID.String() {
-		t.Errorf("ClientEvent.Sender: wanted %s, got %s", userID.String(), ce.Sender)
-	}
+	ce := ToClientEvent(ev, FormatAll, userID.String(), &sk, ev.Unsigned())
+
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce.EventID,
+			Type:           ce.Type,
+			OriginServerTS: ce.OriginServerTS,
+			StateKey:       ce.StateKey,
+			Content:        ce.Content,
+			Unsigned:       ce.Unsigned,
+			Sender:         ce.Sender,
+		},
+		EventFieldsToVerify{
+			EventID:        ev.EventID(),
+			Type:           ev.Type(),
+			OriginServerTS: ev.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev.Content(),
+			Unsigned:       ev.Unsigned(),
+			Sender:         userID.String(),
+		})
+
 	j, err := json.Marshal(ce)
 	if err != nil {
 		t.Fatalf("failed to Marshal ClientEvent: %s", err)
@@ -109,8 +166,378 @@ func TestToClientFormatSync(t *testing.T) {
 		t.Fatalf("failed to create userID: %s", err)
 	}
 	sk := ""
-	ce := ToClientEvent(ev, FormatSync, *userID, &sk)
+	ce := ToClientEvent(ev, FormatSync, userID.String(), &sk, ev.Unsigned())
 	if ce.RoomID != "" {
 		t.Errorf("ClientEvent.RoomID: wanted '', got %s", ce.RoomID)
 	}
 }
+
+func TestToClientEventFormatSyncFederation(t *testing.T) { // nolint: gocyclo
+	ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(`{
+		"type": "m.room.name",
+		"state_key": "",
+		"event_id": "$test:localhost",
+		"room_id": "!test:localhost",
+		"sender": "@test:localhost",
+		"content": {
+			"name": "Hello World"
+		},
+		"origin_server_ts": 123456,
+		"unsigned": {
+			"prev_content": {
+				"name": "Goodbye World"
+			}
+		},
+        "depth": 8,
+        "prev_events": [
+          "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo"
+        ],
+        "auth_events": [
+          "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ",
+          "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ",
+          "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q"
+        ]
+	}`), false)
+	if err != nil {
+		t.Fatalf("failed to create Event: %s", err)
+	}
+	userID, err := spec.NewUserID("@test:localhost", true)
+	if err != nil {
+		t.Fatalf("failed to create userID: %s", err)
+	}
+	sk := ""
+	ce := ToClientEvent(ev, FormatSyncFederation, userID.String(), &sk, ev.Unsigned())
+
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce.EventID,
+			Type:           ce.Type,
+			OriginServerTS: ce.OriginServerTS,
+			StateKey:       ce.StateKey,
+			Content:        ce.Content,
+			Unsigned:       ce.Unsigned,
+			Sender:         ce.Sender,
+			Depth:          ce.Depth,
+			PrevEvents:     ce.PrevEvents,
+			AuthEvents:     ce.AuthEvents,
+		},
+		EventFieldsToVerify{
+			EventID:        ev.EventID(),
+			Type:           ev.Type(),
+			OriginServerTS: ev.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev.Content(),
+			Unsigned:       ev.Unsigned(),
+			Sender:         userID.String(),
+			Depth:          ev.Depth(),
+			PrevEvents:     ev.PrevEventIDs(),
+			AuthEvents:     ev.AuthEventIDs(),
+		})
+}
+
+func userIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	if senderID == "unknownSenderID" {
+		return nil, fmt.Errorf("Cannot find userID")
+	}
+	return spec.NewUserID(testUserID, true)
+}
+
+func TestToClientEventsFormatSyncFederation(t *testing.T) { // nolint: gocyclo
+	ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{
+		"type": "m.room.name",
+        "state_key": "testSenderID",
+		"event_id": "$test:localhost",
+		"room_id": "!test:localhost",
+		"sender": "testSenderID",
+		"content": {
+			"name": "Hello World"
+		},
+		"origin_server_ts": 123456,
+		"unsigned": {
+			"prev_content": {
+				"name": "Goodbye World"
+			}
+		},
+        "depth": 8,
+        "prev_events": [
+          "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo"
+        ],
+        "auth_events": [
+          "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ",
+          "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ",
+          "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q"
+        ]
+	}`), false)
+	if err != nil {
+		t.Fatalf("failed to create Event: %s", err)
+	}
+	ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{
+		"type": "m.room.name",
+        "state_key": "testSenderID",
+		"event_id": "$test2:localhost",
+		"room_id": "!test:localhost",
+		"sender": "testSenderID",
+		"content": {
+			"name": "Hello World 2"
+		},
+		"origin_server_ts": 1234567,
+		"unsigned": {
+			"prev_content": {
+				"name": "Goodbye World 2"
+			},
+            "prev_sender": "testSenderID"
+		},
+        "depth": 9,
+        "prev_events": [
+          "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo"
+        ],
+        "auth_events": [
+          "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ",
+          "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ",
+          "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q"
+        ]
+    }`), false)
+	if err != nil {
+		t.Fatalf("failed to create Event: %s", err)
+	}
+
+	clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSyncFederation, userIDForSender)
+	ce := clientEvents[0]
+	sk := testSenderID
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce.EventID,
+			Type:           ce.Type,
+			OriginServerTS: ce.OriginServerTS,
+			StateKey:       ce.StateKey,
+			Content:        ce.Content,
+			Unsigned:       ce.Unsigned,
+			Sender:         ce.Sender,
+			Depth:          ce.Depth,
+			PrevEvents:     ce.PrevEvents,
+			AuthEvents:     ce.AuthEvents,
+		},
+		EventFieldsToVerify{
+			EventID:        ev.EventID(),
+			Type:           ev.Type(),
+			OriginServerTS: ev.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev.Content(),
+			Unsigned:       ev.Unsigned(),
+			Sender:         testSenderID,
+			Depth:          ev.Depth(),
+			PrevEvents:     ev.PrevEventIDs(),
+			AuthEvents:     ev.AuthEventIDs(),
+		})
+
+	ce2 := clientEvents[1]
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce2.EventID,
+			Type:           ce2.Type,
+			OriginServerTS: ce2.OriginServerTS,
+			StateKey:       ce2.StateKey,
+			Content:        ce2.Content,
+			Unsigned:       ce2.Unsigned,
+			Sender:         ce2.Sender,
+			Depth:          ce2.Depth,
+			PrevEvents:     ce2.PrevEvents,
+			AuthEvents:     ce2.AuthEvents,
+		},
+		EventFieldsToVerify{
+			EventID:        ev2.EventID(),
+			Type:           ev2.Type(),
+			OriginServerTS: ev2.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev2.Content(),
+			Unsigned:       ev2.Unsigned(),
+			Sender:         testSenderID,
+			Depth:          ev2.Depth(),
+			PrevEvents:     ev2.PrevEventIDs(),
+			AuthEvents:     ev2.AuthEventIDs(),
+		})
+}
+
+func TestToClientEventsFormatSync(t *testing.T) { // nolint: gocyclo
+	ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{
+		"type": "m.room.name",
+        "state_key": "testSenderID",
+		"event_id": "$test:localhost",
+		"room_id": "!test:localhost",
+		"sender": "testSenderID",
+		"content": {
+			"name": "Hello World"
+		},
+		"origin_server_ts": 123456,
+		"unsigned": {
+			"prev_content": {
+				"name": "Goodbye World"
+			}
+		}
+    }`), false)
+	if err != nil {
+		t.Fatalf("failed to create Event: %s", err)
+	}
+	ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{
+		"type": "m.room.name",
+        "state_key": "testSenderID",
+		"event_id": "$test2:localhost",
+		"room_id": "!test:localhost",
+		"sender": "testSenderID",
+		"content": {
+			"name": "Hello World 2"
+		},
+		"origin_server_ts": 1234567,
+		"unsigned": {
+			"prev_content": {
+				"name": "Goodbye World 2"
+			},
+            "prev_sender": "testSenderID"
+		},
+        "depth": 9	
+    }`), false)
+	if err != nil {
+		t.Fatalf("failed to create Event: %s", err)
+	}
+
+	clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSync, userIDForSender)
+	ce := clientEvents[0]
+	sk := testUserID
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce.EventID,
+			Type:           ce.Type,
+			OriginServerTS: ce.OriginServerTS,
+			StateKey:       ce.StateKey,
+			Content:        ce.Content,
+			Unsigned:       ce.Unsigned,
+			Sender:         ce.Sender,
+		},
+		EventFieldsToVerify{
+			EventID:        ev.EventID(),
+			Type:           ev.Type(),
+			OriginServerTS: ev.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev.Content(),
+			Unsigned:       ev.Unsigned(),
+			Sender:         testUserID,
+		})
+
+	var prev PrevEventRef
+	prev.PrevContent = []byte(`{"name": "Goodbye World 2"}`)
+	prev.PrevSenderID = testUserID
+	expectedUnsigned, _ := json.Marshal(prev)
+
+	ce2 := clientEvents[1]
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce2.EventID,
+			Type:           ce2.Type,
+			OriginServerTS: ce2.OriginServerTS,
+			StateKey:       ce2.StateKey,
+			Content:        ce2.Content,
+			Unsigned:       ce2.Unsigned,
+			Sender:         ce2.Sender,
+		},
+		EventFieldsToVerify{
+			EventID:        ev2.EventID(),
+			Type:           ev2.Type(),
+			OriginServerTS: ev2.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev2.Content(),
+			Unsigned:       expectedUnsigned,
+			Sender:         testUserID,
+		})
+}
+
+func TestToClientEventsFormatSyncUnknownPrevSender(t *testing.T) { // nolint: gocyclo
+	ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{
+		"type": "m.room.name",
+        "state_key": "testSenderID",
+		"event_id": "$test:localhost",
+		"room_id": "!test:localhost",
+		"sender": "testSenderID",
+		"content": {
+			"name": "Hello World"
+		},
+		"origin_server_ts": 123456,
+		"unsigned": {
+			"prev_content": {
+				"name": "Goodbye World"
+			}
+		}
+    }`), false)
+	if err != nil {
+		t.Fatalf("failed to create Event: %s", err)
+	}
+	ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{
+		"type": "m.room.name",
+        "state_key": "testSenderID",
+		"event_id": "$test2:localhost",
+		"room_id": "!test:localhost",
+		"sender": "testSenderID",
+		"content": {
+			"name": "Hello World 2"
+		},
+		"origin_server_ts": 1234567,
+		"unsigned": {
+			"prev_content": {
+				"name": "Goodbye World 2"
+			},
+            "prev_sender": "unknownSenderID"
+		},
+        "depth": 9	
+    }`), false)
+	if err != nil {
+		t.Fatalf("failed to create Event: %s", err)
+	}
+
+	clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSync, userIDForSender)
+	ce := clientEvents[0]
+	sk := testUserID
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce.EventID,
+			Type:           ce.Type,
+			OriginServerTS: ce.OriginServerTS,
+			StateKey:       ce.StateKey,
+			Content:        ce.Content,
+			Unsigned:       ce.Unsigned,
+			Sender:         ce.Sender,
+		},
+		EventFieldsToVerify{
+			EventID:        ev.EventID(),
+			Type:           ev.Type(),
+			OriginServerTS: ev.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev.Content(),
+			Unsigned:       ev.Unsigned(),
+			Sender:         testUserID,
+		})
+
+	var prev PrevEventRef
+	prev.PrevContent = []byte(`{"name": "Goodbye World 2"}`)
+	prev.PrevSenderID = "unknownSenderID"
+	expectedUnsigned, _ := json.Marshal(prev)
+
+	ce2 := clientEvents[1]
+	verifyEventFields(t,
+		EventFieldsToVerify{
+			EventID:        ce2.EventID,
+			Type:           ce2.Type,
+			OriginServerTS: ce2.OriginServerTS,
+			StateKey:       ce2.StateKey,
+			Content:        ce2.Content,
+			Unsigned:       ce2.Unsigned,
+			Sender:         ce2.Sender,
+		},
+		EventFieldsToVerify{
+			EventID:        ev2.EventID(),
+			Type:           ev2.Type(),
+			OriginServerTS: ev2.OriginServerTS(),
+			StateKey:       &sk,
+			Content:        ev2.Content(),
+			Unsigned:       expectedUnsigned,
+			Sender:         testUserID,
+		})
+}
diff --git a/syncapi/synctypes/filter.go b/syncapi/synctypes/filter.go
index c994ddb96..8998d4433 100644
--- a/syncapi/synctypes/filter.go
+++ b/syncapi/synctypes/filter.go
@@ -78,9 +78,14 @@ type RoomEventFilter struct {
 	ContainsURL               *bool     `json:"contains_url,omitempty"`
 }
 
+const (
+	EventFormatClient     = "client"
+	EventFormatFederation = "federation"
+)
+
 // Validate checks if the filter contains valid property values
 func (filter *Filter) Validate() error {
-	if filter.EventFormat != "" && filter.EventFormat != "client" && filter.EventFormat != "federation" {
+	if filter.EventFormat != "" && filter.EventFormat != EventFormatClient && filter.EventFormat != EventFormatFederation {
 		return errors.New("Bad event_format value. Must be one of [\"client\", \"federation\"]")
 	}
 	return nil
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index cb3c362d5..b90c128c3 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -339,13 +339,6 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 	return token, nil
 }
 
-// PrevEventRef represents a reference to a previous event in a state event upgrade
-type PrevEventRef struct {
-	PrevContent   json.RawMessage `json:"prev_content"`
-	ReplacesState string          `json:"replaces_state"`
-	PrevSenderID  string          `json:"prev_sender"`
-}
-
 type DeviceLists struct {
 	Changed []string `json:"changed,omitempty"`
 	Left    []string `json:"left,omitempty"`
@@ -539,7 +532,7 @@ type InviteResponse struct {
 }
 
 // NewInviteResponse creates an empty response with initialised arrays.
-func NewInviteResponse(event *types.HeaderedEvent, userID spec.UserID, stateKey *string) *InviteResponse {
+func NewInviteResponse(event *types.HeaderedEvent, userID spec.UserID, stateKey *string, eventFormat synctypes.ClientEventFormat) *InviteResponse {
 	res := InviteResponse{}
 	res.InviteState.Events = []json.RawMessage{}
 
@@ -552,7 +545,7 @@ func NewInviteResponse(event *types.HeaderedEvent, userID spec.UserID, stateKey
 
 	// Then we'll see if we can create a partial of the invite event itself.
 	// This is needed for clients to work out *who* sent the invite.
-	inviteEvent := synctypes.ToClientEvent(event.PDU, synctypes.FormatSync, userID, stateKey)
+	inviteEvent := synctypes.ToClientEvent(event.PDU, eventFormat, userID.String(), stateKey, event.Unsigned())
 	inviteEvent.Unsigned = nil
 	if ev, err := json.Marshal(inviteEvent); err == nil {
 		res.InviteState.Events = append(res.InviteState.Events, ev)
diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go
index c1b7f70bd..a79b9fc5d 100644
--- a/syncapi/types/types_test.go
+++ b/syncapi/types/types_test.go
@@ -72,7 +72,7 @@ func TestNewInviteResponse(t *testing.T) {
 	skString := skUserID.String()
 	sk := &skString
 
-	res := NewInviteResponse(&types.HeaderedEvent{PDU: ev}, *sender, sk)
+	res := NewInviteResponse(&types.HeaderedEvent{PDU: ev}, *sender, sk, synctypes.FormatSync)
 	j, err := json.Marshal(res)
 	if err != nil {
 		t.Fatal(err)
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index a88b2129d..8863d258a 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -321,7 +321,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst
 				return fmt.Errorf("queryUserIDForSender: userID unknown for %s", *sk)
 			}
 		}
-		cevent := synctypes.ToClientEvent(event, synctypes.FormatAll, sender, sk)
+		cevent := synctypes.ToClientEvent(event, synctypes.FormatAll, sender.String(), sk, event.Unsigned())
 		var member *localMembership
 		member, err = newLocalMembership(&cevent)
 		if err != nil {
@@ -566,7 +566,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 		// UNSPEC: the spec doesn't say this is a ClientEvent, but the
 		// fields seem to match. room_id should be missing, which
 		// matches the behaviour of FormatSync.
-		Event: synctypes.ToClientEvent(event, synctypes.FormatSync, sender, sk),
+		Event: synctypes.ToClientEvent(event, synctypes.FormatSync, sender.String(), sk, event.Unsigned()),
 		// TODO: this is per-device, but it's not part of the primary
 		// key. So inserting one notification per profile tag doesn't
 		// make sense. What is this supposed to be? Sytests require it
diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go
index 3017069bc..27e77cf7a 100644
--- a/userapi/util/notify_test.go
+++ b/userapi/util/notify_test.go
@@ -106,7 +106,7 @@ func TestNotifyUserCountsAsync(t *testing.T) {
 		}
 		sk := ""
 		if err := db.InsertNotification(ctx, aliceLocalpart, serverName, dummyEvent.EventID(), 0, nil, &api.Notification{
-			Event: synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll, *sender, &sk),
+			Event: synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll, sender.String(), &sk, dummyEvent.Unsigned()),
 		}); err != nil {
 			t.Error(err)
 		}

From 478827459c5d09062bc965d25007c0b81bcf2ba8 Mon Sep 17 00:00:00 2001
From: Sam Wedgwood <28223854+swedgwood@users.noreply.github.com>
Date: Fri, 8 Sep 2023 16:30:21 +0100
Subject: [PATCH 20/57] bump GMSL back to main (#3197)

In a [previous PR](https://github.com/matrix-org/dendrite/pull/3181) I
accidentally left GMSL on a dev branch, this PR fixes it by bringing it
back to the main branch of GMSL

Signed-off-by: `Sam Wedgwood <sam@wedgwood.dev>`
---
 go.mod | 10 +++++-----
 go.sum | 20 ++++++++++----------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/go.mod b/go.mod
index 710b50376..661a64933 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230823153616-484e7693bb8d
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230908150629-47bceffecd9e
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
@@ -42,12 +42,12 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
 	go.uber.org/atomic v1.10.0
-	golang.org/x/crypto v0.12.0
+	golang.org/x/crypto v0.13.0
 	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
 	golang.org/x/image v0.5.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
 	golang.org/x/sync v0.3.0
-	golang.org/x/term v0.11.0
+	golang.org/x/term v0.12.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
@@ -124,8 +124,8 @@ require (
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.12.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
diff --git a/go.sum b/go.sum
index 863caee72..3fe25e9fa 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230823153616-484e7693bb8d h1:yFoT2nyjD4TFrgYMJGgrotFbTLjaYKfZbRmnsj7lvZE=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230823153616-484e7693bb8d/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230908150629-47bceffecd9e h1:WSqq/Pk+4Tna2F7zxEXMPrlZUAfBep3Y2gFbPhKgJHs=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230908150629-47bceffecd9e/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -354,8 +354,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -418,19 +418,19 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

From bea73c765a5c3bfda50df67461771c1f3793030a Mon Sep 17 00:00:00 2001
From: Tulir Asokan <tulir@maunium.net>
Date: Tue, 12 Sep 2023 10:44:51 -0400
Subject: [PATCH 21/57] Fix `user_id` query param breaking auth for
 non-appservices (#3196)

The `user_id` query param only has defined behavior when authenticating
with an `as_token`. For any other tokens, the presence of the parameter
should simply be ignored.

Fixes #1738

Signed-off-by: Tulir Asokan <tulir@maunium.net>
Co-authored-by: devonh <devon.dmytro@gmail.com>
---
 userapi/internal/user_api.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go
index 4305c13a9..4e3c2671a 100644
--- a/userapi/internal/user_api.go
+++ b/userapi/internal/user_api.go
@@ -563,12 +563,15 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc
 func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error {
 	if req.AppServiceUserID != "" {
 		appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID)
-		if err != nil {
-			res.Err = err.Error()
-		}
-		res.Device = appServiceDevice
+		if err != nil || appServiceDevice != nil {
+			if err != nil {
+				res.Err = err.Error()
+			}
+			res.Device = appServiceDevice
 
-		return nil
+			return nil
+		}
+		// If the provided token wasn't an as_token (both err and appServiceDevice are nil), continue with normal auth.
 	}
 	device, err := a.DB.GetDeviceByAccessToken(ctx, req.AccessToken)
 	if err != nil {

From 058081e68e4e23400645c6206cedddba8a31507e Mon Sep 17 00:00:00 2001
From: Sam Wedgwood <28223854+swedgwood@users.noreply.github.com>
Date: Tue, 12 Sep 2023 16:32:24 +0100
Subject: [PATCH 22/57] [pseudoIDs] changing event ID fix (#3195)

power levels events in pseudo IDs sometimes changed event IDs (this was
already fixed earlier, but one of the edgecases was not covered, and is
now covered)

Signed-off-by: `Sam Wedgwood <sam@wedgwood.dev>`
---
 syncapi/streams/stream_pdu.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index ee524f726..3b27acd8a 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -500,7 +500,7 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 	prevContent := gjson.GetBytes(ev.JSON(), "unsigned.prev_content")
 	if !prevContent.Exists() {
 		var evNew gomatrixserverlib.PDU
-		evNew, err = gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSON(newEv, false)
+		evNew, err = gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSONWithEventID(ev.EventID(), newEv, false)
 		if err != nil {
 			return nil, err
 		}

From 8245b24100b0afaa046bb3fe52f0994f906c8ab1 Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Fri, 15 Sep 2023 14:39:06 +0000
Subject: [PATCH 23/57] Update gmsl to use new validated RoomID on PDUs (#3200)

GMSL returns a `spec.RoomID` when calling `PDU.RoomID()`
---
 appservice/consumers/roomserver.go            | 18 ++--
 clientapi/routing/redaction.go                |  2 +-
 clientapi/routing/sendevent.go                |  2 +-
 clientapi/routing/state.go                    |  9 +-
 federationapi/consumers/roomserver.go         | 16 ++--
 federationapi/federationapi_test.go           |  2 +-
 federationapi/internal/perform.go             |  8 +-
 federationapi/queue/queue.go                  |  2 +-
 federationapi/queue/queue_test.go             |  2 +-
 federationapi/routing/backfill.go             |  2 +-
 federationapi/routing/eventauth.go            |  4 +-
 federationapi/routing/events.go               |  2 +-
 federationapi/routing/leave.go                | 11 +--
 federationapi/routing/missingevents.go        |  2 +-
 federationapi/routing/state.go                |  4 +-
 go.mod                                        |  2 +-
 go.sum                                        |  4 +-
 internal/eventutil/events.go                  |  6 +-
 internal/pushrules/evaluate.go                |  8 +-
 internal/pushrules/evaluate_test.go           | 85 +++++++++----------
 internal/transactionrequest.go                |  2 +-
 roomserver/acls/acls.go                       |  2 +-
 roomserver/api/wrapper.go                     |  2 +-
 roomserver/auth/auth.go                       |  6 +-
 roomserver/internal/alias.go                  |  2 +-
 roomserver/internal/api.go                    |  2 +-
 roomserver/internal/helpers/auth.go           |  4 +-
 roomserver/internal/helpers/helpers.go        |  6 +-
 roomserver/internal/input/input.go            |  2 +-
 roomserver/internal/input/input_events.go     | 54 +++++-------
 .../internal/input/input_latest_events.go     |  4 +-
 roomserver/internal/input/input_membership.go | 10 +--
 roomserver/internal/input/input_missing.go    | 14 +--
 .../internal/perform/perform_backfill.go      |  8 +-
 roomserver/internal/perform/perform_invite.go |  8 +-
 .../internal/query/query_room_hierarchy.go    |  4 +-
 roomserver/internal/query/query_test.go       |  1 +
 roomserver/storage/shared/storage.go          | 21 ++---
 setup/mscs/msc2836/msc2836.go                 | 10 +--
 setup/mscs/msc2836/storage.go                 |  2 +-
 syncapi/consumers/clientapi.go                |  2 +-
 syncapi/consumers/roomserver.go               | 55 ++++--------
 syncapi/internal/history_visibility.go        | 13 ++-
 syncapi/notifier/notifier.go                  | 17 ++--
 syncapi/routing/getevent.go                   |  9 +-
 syncapi/routing/memberships.go                | 10 +--
 syncapi/routing/search.go                     | 34 +++-----
 syncapi/routing/search_test.go                |  2 +-
 .../postgres/current_room_state_table.go      |  2 +-
 syncapi/storage/postgres/invites_table.go     |  2 +-
 syncapi/storage/postgres/memberships_table.go |  2 +-
 .../postgres/output_room_events_table.go      |  6 +-
 .../output_room_events_topology_table.go      |  2 +-
 syncapi/storage/shared/storage_consumer.go    | 23 ++---
 .../sqlite3/current_room_state_table.go       |  2 +-
 syncapi/storage/sqlite3/invites_table.go      |  2 +-
 syncapi/storage/sqlite3/memberships_table.go  |  2 +-
 .../sqlite3/output_room_events_table.go       |  6 +-
 .../output_room_events_topology_table.go      |  2 +-
 syncapi/streams/stream_invite.go              |  8 +-
 syncapi/streams/stream_pdu.go                 |  6 +-
 syncapi/syncapi_test.go                       |  2 +-
 syncapi/synctypes/clientevent.go              | 22 ++---
 userapi/consumers/roomserver.go               | 68 +++++----------
 64 files changed, 241 insertions(+), 413 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index 1877de37a..e8b9211c4 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -128,7 +128,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 			if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
 				newEventID := output.NewRoomEvent.Event.EventID()
 				eventsReq := &api.QueryEventsByIDRequest{
-					RoomID:   output.NewRoomEvent.Event.RoomID(),
+					RoomID:   output.NewRoomEvent.Event.RoomID().String(),
 					EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
 				}
 				eventsRes := &api.QueryEventsByIDResponse{}
@@ -236,11 +236,7 @@ func (s *appserviceState) backoffAndPause(err error) error {
 // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
 func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool {
 	user := ""
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return false
-	}
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 	if err == nil {
 		user = userID.String()
 	}
@@ -250,7 +246,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 		return false
 	case appservice.IsInterestedInUserID(user):
 		return true
-	case appservice.IsInterestedInRoomID(event.RoomID()):
+	case appservice.IsInterestedInRoomID(event.RoomID().String()):
 		return true
 	}
 
@@ -261,7 +257,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	}
 
 	// Check all known room aliases of the room the event came from
-	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
+	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID().String()}
 	var queryRes api.GetAliasesForRoomIDResponse
 	if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
@@ -272,7 +268,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get aliases for room")
 	}
 
@@ -288,7 +284,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	// until we have a lighter way of checking the state before the event that
 	// doesn't involve state res, then this is probably OK.
 	membershipReq := &api.QueryMembershipsForRoomRequest{
-		RoomID:     event.RoomID(),
+		RoomID:     event.RoomID().String(),
 		JoinedOnly: true,
 	}
 	membershipRes := &api.QueryMembershipsForRoomResponse{}
@@ -317,7 +313,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 	} else {
 		log.WithFields(log.Fields{
 			"appservice": appservice.ID,
-			"room_id":    event.RoomID(),
+			"room_id":    event.RoomID().String(),
 		}).WithError(err).Errorf("Unable to get membership for room")
 	}
 	return false
diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go
index 230c96d28..aa579db64 100644
--- a/clientapi/routing/redaction.go
+++ b/clientapi/routing/redaction.go
@@ -98,7 +98,7 @@ func SendRedaction(
 			JSON: spec.NotFound("unknown event ID"), // TODO: is it ok to leak existence?
 		}
 	}
-	if ev.RoomID() != roomID {
+	if ev.RoomID().String() != roomID {
 		return util.JSONResponse{
 			Code: 400,
 			JSON: spec.NotFound("cannot redact event in another room"),
diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index f81e9c1e4..fc9c05ba9 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -437,7 +437,7 @@ func generateSendEvent(
 				JSON: spec.BadJSON("Cannot unmarshal the event content."),
 			}
 		}
-		if content["replacement_room"] == e.RoomID() {
+		if content["replacement_room"] == e.RoomID().String() {
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.InvalidParam("Cannot send tombstone event that points to the same room."),
diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index d7f0b40f8..6f363349b 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -173,19 +173,14 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		}
 		for _, ev := range stateAfterRes.StateEvents {
 			sender := spec.UserID{}
-			evRoomID, err := spec.NewRoomID(ev.RoomID())
-			if err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Event roomID is invalid")
-				continue
-			}
-			userID, err := rsAPI.QueryUserIDForSender(ctx, *evRoomID, ev.SenderID())
+			userID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID())
 			if err == nil && userID != nil {
 				sender = *userID
 			}
 
 			sk := ev.StateKey()
 			if sk != nil && *sk != "" {
-				skUserID, err := rsAPI.QueryUserIDForSender(ctx, *evRoomID, spec.SenderID(*ev.StateKey()))
+				skUserID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey()))
 				if err == nil && skUserID != nil {
 					skString := skUserID.String()
 					sk = &skString
diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index 6c0580322..f1dcb1175 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -176,7 +176,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	// Finally, work out if there are any more events missing.
 	if len(missingEventIDs) > 0 {
 		eventsReq := &api.QueryEventsByIDRequest{
-			RoomID:   ore.Event.RoomID(),
+			RoomID:   ore.Event.RoomID().String(),
 			EventIDs: missingEventIDs,
 		}
 		eventsRes := &api.QueryEventsByIDResponse{}
@@ -205,7 +205,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	// talking to the roomserver
 	oldJoinedHosts, err := s.db.UpdateRoom(
 		s.ctx,
-		ore.Event.RoomID(),
+		ore.Event.RoomID().String(),
 		addsJoinedHosts,
 		ore.RemovesStateEventIDs,
 		rewritesState, // if we're re-writing state, nuke all joined hosts before adding
@@ -218,7 +218,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == spec.MRoomMember && ore.Event.StateKey() != nil {
 		membership, _ := ore.Event.Membership()
 		if membership == spec.Join {
-			s.sendPresence(ore.Event.RoomID(), addsJoinedHosts)
+			s.sendPresence(ore.Event.RoomID().String(), addsJoinedHosts)
 		}
 	}
 
@@ -376,7 +376,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 	}
 
 	// handle peeking hosts
-	inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID())
+	inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID().String())
 	if err != nil {
 		return nil, err
 	}
@@ -409,12 +409,8 @@ func JoinedHostsFromEvents(ctx context.Context, evs []gomatrixserverlib.PDU, rsA
 		if membership != spec.Join {
 			continue
 		}
-		validRoomID, err := spec.NewRoomID(ev.RoomID())
-		if err != nil {
-			return nil, err
-		}
 		var domain spec.ServerName
-		userID, err := rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*ev.StateKey()))
+		userID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey()))
 		if err != nil {
 			if errors.As(err, new(base64.CorruptInputError)) {
 				// Fallback to using the "old" way of getting the user domain, avoids
@@ -510,7 +506,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 	// At this point the missing events are neither the event itself nor are
 	// they present in our local database. Our only option is to fetch them
 	// from the roomserver using the query API.
-	eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID()}
+	eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID().String()}
 	var eventResp api.QueryEventsByIDResponse
 	if err := s.rsAPI.QueryEventsByID(s.ctx, &eventReq, &eventResp); err != nil {
 		return nil, err
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index 4c2a99bbc..1ea8c40ea 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -146,7 +146,7 @@ func (f *fedClient) SendJoin(ctx context.Context, origin, s spec.ServerName, eve
 	f.fedClientMutex.Lock()
 	defer f.fedClientMutex.Unlock()
 	for _, r := range f.allowJoins {
-		if r.ID == event.RoomID() {
+		if r.ID == event.RoomID().String() {
 			r.InsertEvent(f.t, &types.HeaderedEvent{PDU: event})
 			f.t.Logf("Join event: %v", event.EventID())
 			res.StateEvents = types.NewEventJSONsFromHeaderedEvents(r.CurrentState())
diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 3bba3ea0d..0200cf69b 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -548,11 +548,7 @@ func (r *FederationInternalAPI) SendInvite(
 	event gomatrixserverlib.PDU,
 	strippedState []gomatrixserverlib.InviteStrippedState,
 ) (gomatrixserverlib.PDU, error) {
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return nil, err
-	}
-	inviter, err := r.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+	inviter, err := r.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 	if err != nil {
 		return nil, err
 	}
@@ -575,7 +571,7 @@ func (r *FederationInternalAPI) SendInvite(
 	logrus.WithFields(logrus.Fields{
 		"event_id":     event.EventID(),
 		"user_id":      *event.StateKey(),
-		"room_id":      event.RoomID(),
+		"room_id":      event.RoomID().String(),
 		"room_version": event.Version(),
 		"destination":  destination,
 	}).Info("Sending invite")
diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go
index 26305ed7a..24b3efd2d 100644
--- a/federationapi/queue/queue.go
+++ b/federationapi/queue/queue.go
@@ -218,7 +218,7 @@ func (oqs *OutgoingQueues) SendEvent(
 		if api.IsServerBannedFromRoom(
 			oqs.process.Context(),
 			oqs.rsAPI,
-			ev.RoomID(),
+			ev.RoomID().String(),
 			destination,
 		) {
 			delete(destmap, destination)
diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go
index cc38e136f..e75615e05 100644
--- a/federationapi/queue/queue_test.go
+++ b/federationapi/queue/queue_test.go
@@ -104,7 +104,7 @@ func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u
 
 func mustCreatePDU(t *testing.T) *types.HeaderedEvent {
 	t.Helper()
-	content := `{"type":"m.room.message"}`
+	content := `{"type":"m.room.message", "room_id":"!room:a"}`
 	ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(content), false)
 	if err != nil {
 		t.Fatalf("failed to create event: %v", err)
diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go
index 552c4eac2..75a007265 100644
--- a/federationapi/routing/backfill.go
+++ b/federationapi/routing/backfill.go
@@ -109,7 +109,7 @@ func Backfill(
 
 	var ev *types.HeaderedEvent
 	for _, ev = range res.Events {
-		if ev.RoomID() == roomID {
+		if ev.RoomID().String() == roomID {
 			evs = append(evs, ev.PDU)
 		}
 	}
diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go
index c26aa2f15..2be3ecdb1 100644
--- a/federationapi/routing/eventauth.go
+++ b/federationapi/routing/eventauth.go
@@ -42,10 +42,10 @@ func GetEventAuth(
 		return *resErr
 	}
 
-	if event.RoomID() != roomID {
+	if event.RoomID().String() != roomID {
 		return util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")}
 	}
-	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID())
+	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String())
 	if resErr != nil {
 		return *resErr
 	}
diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go
index d3f0e81c3..f4659f528 100644
--- a/federationapi/routing/events.go
+++ b/federationapi/routing/events.go
@@ -42,7 +42,7 @@ func GetEvent(
 		return *err
 	}
 
-	err = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID())
+	err = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String())
 	if err != nil {
 		return *err
 	}
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index f28c82115..7c86ba69b 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -211,7 +211,7 @@ func SendLeave(
 	}
 
 	// Check that the room ID is correct.
-	if event.RoomID() != roomID {
+	if event.RoomID().String() != roomID {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: spec.BadJSON("The room ID in the request path must match the room ID in the leave event JSON"),
@@ -242,14 +242,7 @@ func SendLeave(
 	// Check that the sender belongs to the server that is sending us
 	// the request. By this point we've already asserted that the sender
 	// and the state key are equal so we don't need to check both.
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Room ID is invalid."),
-		}
-	}
-	sender, err := rsAPI.QueryUserIDForSender(httpReq.Context(), *validRoomID, event.SenderID())
+	sender, err := rsAPI.QueryUserIDForSender(httpReq.Context(), event.RoomID(), event.SenderID())
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go
index f57d30204..b1cefe7b4 100644
--- a/federationapi/routing/missingevents.go
+++ b/federationapi/routing/missingevents.go
@@ -87,7 +87,7 @@ func filterEvents(
 ) []*types.HeaderedEvent {
 	ref := events[:0]
 	for _, ev := range events {
-		if ev.RoomID() == roomID {
+		if ev.RoomID().String() == roomID {
 			ref = append(ref, ev)
 		}
 	}
diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go
index 11ad1ebfc..d10910573 100644
--- a/federationapi/routing/state.go
+++ b/federationapi/routing/state.go
@@ -113,10 +113,10 @@ func getState(
 		return nil, nil, resErr
 	}
 
-	if event.RoomID() != roomID {
+	if event.RoomID().String() != roomID {
 		return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")}
 	}
-	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID())
+	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String())
 	if resErr != nil {
 		return nil, nil, resErr
 	}
diff --git a/go.mod b/go.mod
index 661a64933..564b4250a 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230908150629-47bceffecd9e
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
diff --git a/go.sum b/go.sum
index 3fe25e9fa..f7d21d96b 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230908150629-47bceffecd9e h1:WSqq/Pk+4Tna2F7zxEXMPrlZUAfBep3Y2gFbPhKgJHs=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230908150629-47bceffecd9e/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a h1:+RC9Ddmt5v4y58qmdz5WuEEWCJ9gBWuYLyndnWkGfXU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go
index aa99e5860..b3523e129 100644
--- a/internal/eventutil/events.go
+++ b/internal/eventutil/events.go
@@ -176,11 +176,7 @@ func RedactEvent(ctx context.Context, redactionEvent, redactedEvent gomatrixserv
 		return fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type())
 	}
 	redactedEvent.Redact()
-	validRoomID, err := spec.NewRoomID(redactionEvent.RoomID())
-	if err != nil {
-		return err
-	}
-	senderID, err := querier.QueryUserIDForSender(ctx, *validRoomID, redactionEvent.SenderID())
+	senderID, err := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactionEvent.SenderID())
 	if err != nil {
 		return err
 	}
diff --git a/internal/pushrules/evaluate.go b/internal/pushrules/evaluate.go
index 28dea97c4..6baef4347 100644
--- a/internal/pushrules/evaluate.go
+++ b/internal/pushrules/evaluate.go
@@ -111,15 +111,11 @@ func ruleMatches(rule *Rule, kind Kind, event gomatrixserverlib.PDU, ec Evaluati
 		return patternMatches("content.body", *rule.Pattern, event)
 
 	case RoomKind:
-		return rule.RuleID == event.RoomID(), nil
+		return rule.RuleID == event.RoomID().String(), nil
 
 	case SenderKind:
 		userID := ""
-		validRoomID, err := spec.NewRoomID(event.RoomID())
-		if err != nil {
-			return false, err
-		}
-		sender, err := userIDForSender(*validRoomID, event.SenderID())
+		sender, err := userIDForSender(event.RoomID(), event.SenderID())
 		if err == nil {
 			userID = sender.String()
 		}
diff --git a/internal/pushrules/evaluate_test.go b/internal/pushrules/evaluate_test.go
index a4ccc3d0f..fbc88b2e7 100644
--- a/internal/pushrules/evaluate_test.go
+++ b/internal/pushrules/evaluate_test.go
@@ -13,7 +13,7 @@ func UserIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID,
 }
 
 func TestRuleSetEvaluatorMatchEvent(t *testing.T) {
-	ev := mustEventFromJSON(t, `{}`)
+	ev := mustEventFromJSON(t, `{"room_id":"!room:a"}`)
 	defaultEnabled := &Rule{
 		RuleID:  ".default.enabled",
 		Default: true,
@@ -44,8 +44,8 @@ func TestRuleSetEvaluatorMatchEvent(t *testing.T) {
 		{"overrideRoom", RuleSet{Override: []*Rule{userEnabled}, Room: []*Rule{userEnabled2}}, userEnabled, ev},
 		{"overrideSender", RuleSet{Override: []*Rule{userEnabled}, Sender: []*Rule{userEnabled2}}, userEnabled, ev},
 		{"overrideUnderride", RuleSet{Override: []*Rule{userEnabled}, Underride: []*Rule{userEnabled2}}, userEnabled, ev},
-		{"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"type":"m.reaction"}`)},
-		{"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"type":"m.receipt"}`)},
+		{"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.reaction"}`)},
+		{"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.receipt"}`)},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -70,28 +70,27 @@ func TestRuleMatches(t *testing.T) {
 		EventJSON string
 		Want      bool
 	}{
-		{"emptyOverride", OverrideKind, emptyRule, `{}`, true},
-		{"emptyContent", ContentKind, emptyRule, `{}`, false},
-		{"emptyRoom", RoomKind, emptyRule, `{}`, true},
+		{"emptyOverride", OverrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true},
+		{"emptyContent", ContentKind, emptyRule, `{"room_id":"!room:example.com"}`, false},
 		{"emptySender", SenderKind, emptyRule, `{"room_id":"!room:example.com"}`, true},
-		{"emptyUnderride", UnderrideKind, emptyRule, `{}`, true},
+		{"emptyUnderride", UnderrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true},
 
-		{"disabled", OverrideKind, Rule{}, `{}`, false},
+		{"disabled", OverrideKind, Rule{}, `{"room_id":"!room:example.com"}`, false},
 
-		{"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{}`, true},
-		{"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false},
+		{"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true},
+		{"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false},
 
-		{"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{}`, true},
-		{"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false},
+		{"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true},
+		{"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false},
 
-		{"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"content":{"body":"abc"}}`, true},
-		{"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"content":{"body":"abc"}}`, false},
+		{"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, true},
+		{"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, false},
 
 		{"roomMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!room:example.com"}`, true},
 		{"roomNoMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!otherroom:example.com"}`, false},
 
-		{"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"sender":"@user:example.com","room_id":"!room:example.com"}`, true},
-		{"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"sender":"@otheruser:example.com","room_id":"!room:example.com"}`, false},
+		{"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@user:example.com","room_id":"!room:example.com"}`, true},
+		{"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@otheruser:example.com","room_id":"!room:example.com"}`, false},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -114,32 +113,32 @@ func TestConditionMatches(t *testing.T) {
 		WantMatch bool
 		WantErr   bool
 	}{
-		{Name: "empty", Cond: Condition{}, EventJSON: `{}`, WantMatch: false, WantErr: false},
-		{Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{}`, WantMatch: false, WantErr: false},
+		{Name: "empty", Cond: Condition{}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false},
+		{Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false},
 
 		// Neither of these should match because `content` is not a full string match,
 		// and `content.body` is not a string value.
-		{Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"content":{}}`, WantMatch: false, WantErr: false},
-		{Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"content":{"body": "3"}}`, WantMatch: false, WantErr: false},
-		{Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false},
-		{Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true},
+		{Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{}}`, WantMatch: false, WantErr: false},
+		{Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "3"}}`, WantMatch: false, WantErr: false},
+		{Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false},
+		{Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true},
 
-		{Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false},
-		{Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false},
+		{Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false},
+		{Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false},
 
-		{Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{}`, WantMatch: false, WantErr: false},
-		{Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{}`, WantMatch: true, WantErr: false},
-		{Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{}`, WantMatch: false, WantErr: false},
-		{Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false},
-		{Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{}`, WantMatch: false, WantErr: false},
-		{Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{}`, WantMatch: true, WantErr: false},
-		{Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{}`, WantMatch: false, WantErr: false},
-		{Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false},
-		{Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{}`, WantMatch: false, WantErr: false},
-		{Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{}`, WantMatch: true, WantErr: false},
+		{Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false},
+		{Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false},
+		{Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false},
+		{Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false},
+		{Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false},
+		{Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false},
+		{Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false},
+		{Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false},
+		{Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false},
+		{Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false},
 
-		{Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false},
-		{Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false},
+		{Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false},
+		{Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -170,15 +169,15 @@ func TestPatternMatches(t *testing.T) {
 		EventJSON string
 		Want      bool
 	}{
-		{"empty", "", "", `{}`, false},
+		{"empty", "", "", `{"room_id":"!room:a"}`, false},
 
-		{"patternEmpty", "content", "", `{"content":{}}`, false},
+		{"patternEmpty", "content", "", `{"room_id":"!room:a","content":{}}`, false},
 
-		{"literal", "content.creator", "acreator", `{"content":{"creator":"acreator"}}`, true},
-		{"substring", "content.creator", "reat", `{"content":{"creator":"acreator"}}`, true},
-		{"singlePattern", "content.creator", "acr?ator", `{"content":{"creator":"acreator"}}`, true},
-		{"multiPattern", "content.creator", "a*ea*r", `{"content":{"creator":"acreator"}}`, true},
-		{"patternNoSubstring", "content.creator", "r*t", `{"content":{"creator":"acreator"}}`, false},
+		{"literal", "content.creator", "acreator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true},
+		{"substring", "content.creator", "reat", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true},
+		{"singlePattern", "content.creator", "acr?ator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true},
+		{"multiPattern", "content.creator", "a*ea*r", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true},
+		{"patternNoSubstring", "content.creator", "r*t", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, false},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
diff --git a/internal/transactionrequest.go b/internal/transactionrequest.go
index 5bf7d819c..0663c2dcb 100644
--- a/internal/transactionrequest.go
+++ b/internal/transactionrequest.go
@@ -161,7 +161,7 @@ func (t *TxnReq) ProcessTransaction(ctx context.Context) (*fclient.RespSend, *ut
 		if event.Type() == spec.MRoomCreate && event.StateKeyEquals("") {
 			continue
 		}
-		if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID(), t.Origin) {
+		if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID().String(), t.Origin) {
 			results[event.EventID()] = fclient.PDUResult{
 				Error: "Forbidden by server ACLs",
 			}
diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go
index b04828b69..601ce9063 100644
--- a/roomserver/acls/acls.go
+++ b/roomserver/acls/acls.go
@@ -119,7 +119,7 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
 	}).Debugf("Updating server ACLs for %q", state.RoomID())
 	s.aclsMutex.Lock()
 	defer s.aclsMutex.Unlock()
-	s.acls[state.RoomID()] = acls
+	s.acls[state.RoomID().String()] = acls
 }
 
 func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID string) bool {
diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go
index 2505a993b..0ad5d2013 100644
--- a/roomserver/api/wrapper.go
+++ b/roomserver/api/wrapper.go
@@ -75,7 +75,7 @@ func SendEventWithState(
 	}
 
 	logrus.WithContext(ctx).WithFields(logrus.Fields{
-		"room_id":   event.RoomID(),
+		"room_id":   event.RoomID().String(),
 		"event_id":  event.EventID(),
 		"outliers":  len(ires),
 		"state_ids": len(stateEventIDs),
diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go
index df95851e3..d5172dab9 100644
--- a/roomserver/auth/auth.go
+++ b/roomserver/auth/auth.go
@@ -85,11 +85,7 @@ func IsAnyUserOnServerWithMembership(ctx context.Context, querier api.QuerySende
 			continue
 		}
 
-		validRoomID, err := spec.NewRoomID(ev.RoomID())
-		if err != nil {
-			continue
-		}
-		userID, err := querier.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*stateKey))
+		userID, err := querier.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*stateKey))
 		if err != nil {
 			continue
 		}
diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go
index a7f0aab9c..5ceda7e01 100644
--- a/roomserver/internal/alias.go
+++ b/roomserver/internal/alias.go
@@ -189,7 +189,7 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias(ctx context.Context, senderID sp
 
 			proto := &gomatrixserverlib.ProtoEvent{
 				SenderID: string(canonicalSenderID),
-				RoomID:   ev.RoomID(),
+				RoomID:   ev.RoomID().String(),
 				Type:     ev.Type(),
 				StateKey: ev.StateKey(),
 				Content:  res,
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index 530147daa..1e08f6a3a 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -239,7 +239,7 @@ func (r *RoomserverInternalAPI) HandleInvite(
 	if err != nil {
 		return err
 	}
-	return r.OutputProducer.ProduceRoomEvents(inviteEvent.RoomID(), outputEvents)
+	return r.OutputProducer.ProduceRoomEvents(inviteEvent.RoomID().String(), outputEvents)
 }
 
 func (r *RoomserverInternalAPI) PerformCreateRoom(
diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go
index 89fae244f..9da751b1a 100644
--- a/roomserver/internal/helpers/auth.go
+++ b/roomserver/internal/helpers/auth.go
@@ -218,9 +218,9 @@ func loadAuthEvents(
 	roomID := ""
 	for _, ev := range result.events {
 		if roomID == "" {
-			roomID = ev.RoomID()
+			roomID = ev.RoomID().String()
 		}
-		if ev.RoomID() != roomID {
+		if ev.RoomID().String() != roomID {
 			result.valid = false
 			break
 		}
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index febabf411..b2e21bf54 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -54,7 +54,7 @@ func UpdateToInviteMembership(
 			Type: api.OutputTypeRetireInviteEvent,
 			RetireInviteEvent: &api.OutputRetireInviteEvent{
 				EventID:          eventID,
-				RoomID:           add.RoomID(),
+				RoomID:           add.RoomID().String(),
 				Membership:       spec.Join,
 				RetiredByEventID: add.EventID(),
 				TargetSenderID:   spec.SenderID(*add.StateKey()),
@@ -396,7 +396,7 @@ BFSLoop:
 			// It's nasty that we have to extract the room ID from an event, but many federation requests
 			// only talk in event IDs, no room IDs at all (!!!)
 			ev := events[0]
-			isServerInRoom, err = IsServerCurrentlyInRoom(ctx, db, querier, serverName, ev.RoomID())
+			isServerInRoom, err = IsServerCurrentlyInRoom(ctx, db, querier, serverName, ev.RoomID().String())
 			if err != nil {
 				util.GetLogger(ctx).WithError(err).Error("Failed to check if server is currently in room, assuming not.")
 			}
@@ -419,7 +419,7 @@ BFSLoop:
 				// hasn't been seen before.
 				if !visited[pre] {
 					visited[pre] = true
-					allowed, err = CheckServerAllowedToSeeEvent(ctx, db, info, ev.RoomID(), pre, serverName, isServerInRoom, querier)
+					allowed, err = CheckServerAllowedToSeeEvent(ctx, db, info, ev.RoomID().String(), pre, serverName, isServerInRoom, querier)
 					if err != nil {
 						util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).WithError(err).Error(
 							"Error checking if allowed to see event",
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 990563599..404751532 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -358,7 +358,7 @@ func (r *Inputer) queueInputRoomEvents(
 	// For each event, marshal the input room event and then
 	// send it into the input queue.
 	for _, e := range request.InputRoomEvents {
-		roomID := e.Event.RoomID()
+		roomID := e.Event.RoomID().String()
 		subj := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(roomID))
 		msg := &nats.Msg{
 			Subject: subj,
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index bf3216623..77b50d0e2 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -87,7 +87,7 @@ func (r *Inputer) processRoomEvent(
 	}
 
 	trace, ctx := internal.StartRegion(ctx, "processRoomEvent")
-	trace.SetTag("room_id", input.Event.RoomID())
+	trace.SetTag("room_id", input.Event.RoomID().String())
 	trace.SetTag("event_id", input.Event.EventID())
 	defer trace.EndRegion()
 
@@ -96,7 +96,7 @@ func (r *Inputer) processRoomEvent(
 	defer func() {
 		timetaken := time.Since(started)
 		processRoomEventDuration.With(prometheus.Labels{
-			"room_id": input.Event.RoomID(),
+			"room_id": input.Event.RoomID().String(),
 		}).Observe(float64(timetaken.Milliseconds()))
 	}()
 
@@ -105,7 +105,7 @@ func (r *Inputer) processRoomEvent(
 	event := headered.PDU
 	logger := util.GetLogger(ctx).WithFields(logrus.Fields{
 		"event_id": event.EventID(),
-		"room_id":  event.RoomID(),
+		"room_id":  event.RoomID().String(),
 		"kind":     input.Kind,
 		"origin":   input.Origin,
 		"type":     event.Type(),
@@ -120,19 +120,15 @@ func (r *Inputer) processRoomEvent(
 	// Don't waste time processing the event if the room doesn't exist.
 	// A room entry locally will only be created in response to a create
 	// event.
-	roomInfo, rerr := r.DB.RoomInfo(ctx, event.RoomID())
+	roomInfo, rerr := r.DB.RoomInfo(ctx, event.RoomID().String())
 	if rerr != nil {
 		return fmt.Errorf("r.DB.RoomInfo: %w", rerr)
 	}
 	isCreateEvent := event.Type() == spec.MRoomCreate && event.StateKeyEquals("")
 	if roomInfo == nil && !isCreateEvent {
-		return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID())
+		return fmt.Errorf("room %s does not exist for event %s", event.RoomID().String(), event.EventID())
 	}
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return err
-	}
-	sender, err := r.Queryer.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+	sender, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 	if err != nil {
 		return fmt.Errorf("failed getting userID for sender %q. %w", event.SenderID(), err)
 	}
@@ -179,7 +175,7 @@ func (r *Inputer) processRoomEvent(
 	// If we have missing events (auth or prev), we build a list of servers to ask
 	if missingAuth || missingPrev {
 		serverReq := &fedapi.QueryJoinedHostServerNamesInRoomRequest{
-			RoomID:             event.RoomID(),
+			RoomID:             event.RoomID().String(),
 			ExcludeSelf:        true,
 			ExcludeBlacklisted: true,
 		}
@@ -395,12 +391,12 @@ func (r *Inputer) processRoomEvent(
 
 	// Request the room info again — it's possible that the room has been
 	// created by now if it didn't exist already.
-	roomInfo, err = r.DB.RoomInfo(ctx, event.RoomID())
+	roomInfo, err = r.DB.RoomInfo(ctx, event.RoomID().String())
 	if err != nil {
 		return fmt.Errorf("updater.RoomInfo: %w", err)
 	}
 	if roomInfo == nil {
-		return fmt.Errorf("updater.RoomInfo missing for room %s", event.RoomID())
+		return fmt.Errorf("updater.RoomInfo missing for room %s", event.RoomID().String())
 	}
 
 	if input.HasState || (!missingPrev && stateAtEvent.BeforeStateSnapshotNID == 0) {
@@ -459,7 +455,7 @@ func (r *Inputer) processRoomEvent(
 			if userErr != nil {
 				return userErr
 			}
-			err = r.RSAPI.StoreUserRoomPublicKey(ctx, mapping.MXIDMapping.UserRoomKey, *storeUserID, *validRoomID)
+			err = r.RSAPI.StoreUserRoomPublicKey(ctx, mapping.MXIDMapping.UserRoomKey, *storeUserID, event.RoomID())
 			if err != nil {
 				return fmt.Errorf("failed storing user room public key: %w", err)
 			}
@@ -481,7 +477,7 @@ func (r *Inputer) processRoomEvent(
 			return fmt.Errorf("r.updateLatestEvents: %w", err)
 		}
 	case api.KindOld:
-		err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{
+		err = r.OutputProducer.ProduceRoomEvents(event.RoomID().String(), []api.OutputEvent{
 			{
 				Type: api.OutputTypeOldRoomEvent,
 				OldRoomEvent: &api.OutputOldRoomEvent{
@@ -507,7 +503,7 @@ func (r *Inputer) processRoomEvent(
 	// so notify downstream components to redact this event - they should have it if they've
 	// been tracking our output log.
 	if redactedEventID != "" {
-		err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{
+		err = r.OutputProducer.ProduceRoomEvents(event.RoomID().String(), []api.OutputEvent{
 			{
 				Type: api.OutputTypeRedactedEvent,
 				RedactedEvent: &api.OutputRedactedEvent{
@@ -536,7 +532,7 @@ func (r *Inputer) processRoomEvent(
 
 // handleRemoteRoomUpgrade updates published rooms and room aliases
 func (r *Inputer) handleRemoteRoomUpgrade(ctx context.Context, event gomatrixserverlib.PDU) error {
-	oldRoomID := event.RoomID()
+	oldRoomID := event.RoomID().String()
 	newRoomID := gjson.GetBytes(event.Content(), "replacement_room").Str
 	return r.DB.UpgradeRoom(ctx, oldRoomID, newRoomID, string(event.SenderID()))
 }
@@ -596,7 +592,7 @@ func (r *Inputer) processStateBefore(
 			StateKey:  "",
 		})
 		stateBeforeReq := &api.QueryStateAfterEventsRequest{
-			RoomID:       event.RoomID(),
+			RoomID:       event.RoomID().String(),
 			PrevEventIDs: event.PrevEventIDs(),
 			StateToFetch: tuplesNeeded,
 		}
@@ -606,7 +602,7 @@ func (r *Inputer) processStateBefore(
 		}
 		switch {
 		case !stateBeforeRes.RoomExists:
-			rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID())
+			rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID().String())
 			return
 		case !stateBeforeRes.PrevEventsExist:
 			rejectionErr = fmt.Errorf("prev events of %q are not known", event.EventID())
@@ -707,7 +703,7 @@ func (r *Inputer) fetchAuthEvents(
 		// Request the entire auth chain for the event in question. This should
 		// contain all of the auth events — including ones that we already know —
 		// so we'll need to filter through those in the next section.
-		res, err = r.FSAPI.GetEventAuth(ctx, virtualHost, serverName, event.Version(), event.RoomID(), event.EventID())
+		res, err = r.FSAPI.GetEventAuth(ctx, virtualHost, serverName, event.Version(), event.RoomID().String(), event.EventID())
 		if err != nil {
 			logger.WithError(err).Warnf("Failed to get event auth from federation for %q: %s", event.EventID(), err)
 			continue
@@ -866,25 +862,20 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r
 
 	inputEvents := make([]api.InputRoomEvent, 0, len(memberEvents))
 	latestReq := &api.QueryLatestEventsAndStateRequest{
-		RoomID: event.RoomID(),
+		RoomID: event.RoomID().String(),
 	}
 	latestRes := &api.QueryLatestEventsAndStateResponse{}
 	if err = r.Queryer.QueryLatestEventsAndState(ctx, latestReq, latestRes); err != nil {
 		return err
 	}
 
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return err
-	}
-
 	prevEvents := latestRes.LatestEvents
 	for _, memberEvent := range memberEvents {
 		if memberEvent.StateKey() == nil {
 			continue
 		}
 
-		memberUserID, err := r.Queryer.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*memberEvent.StateKey()))
+		memberUserID, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*memberEvent.StateKey()))
 		if err != nil {
 			continue
 		}
@@ -912,7 +903,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r
 
 		stateKey := *memberEvent.StateKey()
 		fledglingEvent := &gomatrixserverlib.ProtoEvent{
-			RoomID:     event.RoomID(),
+			RoomID:     event.RoomID().String(),
 			Type:       spec.MRoomMember,
 			StateKey:   &stateKey,
 			SenderID:   stateKey,
@@ -928,12 +919,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r
 			return err
 		}
 
-		validRoomID, err := spec.NewRoomID(event.RoomID())
-		if err != nil {
-			return err
-		}
-
-		signingIdentity, err := r.SigningIdentity(ctx, *validRoomID, *memberUserID)
+		signingIdentity, err := r.SigningIdentity(ctx, event.RoomID(), *memberUserID)
 		if err != nil {
 			return err
 		}
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 940783e03..ec03d6f13 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -197,7 +197,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// send the event asynchronously but we would need to ensure that 1) the events are written to the log in
 	// the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the
 	// necessary bookkeeping we'll keep the event sending synchronous for now.
-	if err = u.api.OutputProducer.ProduceRoomEvents(u.event.RoomID(), updates); err != nil {
+	if err = u.api.OutputProducer.ProduceRoomEvents(u.event.RoomID().String(), updates); err != nil {
 		return fmt.Errorf("u.api.WriteOutputEvents: %w", err)
 	}
 
@@ -290,7 +290,7 @@ func (u *latestEventsUpdater) latestState() error {
 	if removed := len(u.removed) - len(u.added); !u.rewritesState && removed > 0 {
 		logrus.WithFields(logrus.Fields{
 			"event_id":      u.event.EventID(),
-			"room_id":       u.event.RoomID(),
+			"room_id":       u.event.RoomID().String(),
 			"old_state_nid": u.oldStateNID,
 			"new_state_nid": u.newStateNID,
 			"old_latest":    u.oldLatest.EventIDs(),
diff --git a/roomserver/internal/input/input_membership.go b/roomserver/internal/input/input_membership.go
index c46f8dba1..4cfc2cda9 100644
--- a/roomserver/internal/input/input_membership.go
+++ b/roomserver/internal/input/input_membership.go
@@ -139,11 +139,7 @@ func (r *Inputer) updateMembership(
 func (r *Inputer) isLocalTarget(ctx context.Context, event *types.Event) bool {
 	isTargetLocalUser := false
 	if statekey := event.StateKey(); statekey != nil {
-		validRoomID, err := spec.NewRoomID(event.RoomID())
-		if err != nil {
-			return isTargetLocalUser
-		}
-		userID, err := r.Queryer.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*statekey))
+		userID, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*statekey))
 		if err != nil || userID == nil {
 			return isTargetLocalUser
 		}
@@ -168,7 +164,7 @@ func updateToJoinMembership(
 			Type: api.OutputTypeRetireInviteEvent,
 			RetireInviteEvent: &api.OutputRetireInviteEvent{
 				EventID:          eventID,
-				RoomID:           add.RoomID(),
+				RoomID:           add.RoomID().String(),
 				Membership:       spec.Join,
 				RetiredByEventID: add.EventID(),
 				TargetSenderID:   spec.SenderID(*add.StateKey()),
@@ -195,7 +191,7 @@ func updateToLeaveMembership(
 			Type: api.OutputTypeRetireInviteEvent,
 			RetireInviteEvent: &api.OutputRetireInviteEvent{
 				EventID:          eventID,
-				RoomID:           add.RoomID(),
+				RoomID:           add.RoomID().String(),
 				Membership:       newMembership,
 				RetiredByEventID: add.EventID(),
 				TargetSenderID:   spec.SenderID(*add.StateKey()),
diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index 5b4c0727b..d9ab291e9 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -84,7 +84,7 @@ func (t *missingStateReq) processEventWithMissingState(
 	// need to fallback to /state.
 	t.log = util.GetLogger(ctx).WithFields(map[string]interface{}{
 		"txn_event":       e.EventID(),
-		"room_id":         e.RoomID(),
+		"room_id":         e.RoomID().String(),
 		"txn_prev_events": e.PrevEventIDs(),
 	})
 
@@ -264,7 +264,7 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e
 		// Look up what the state is after the backward extremity. This will either
 		// come from the roomserver, if we know all the required events, or it will
 		// come from a remote server via /state_ids if not.
-		prevState, trustworthy, err := t.lookupStateAfterEvent(ctx, roomVersion, e.RoomID(), prevEventID)
+		prevState, trustworthy, err := t.lookupStateAfterEvent(ctx, roomVersion, e.RoomID().String(), prevEventID)
 		switch err2 := err.(type) {
 		case gomatrixserverlib.EventValidationError:
 			if !err2.Persistable {
@@ -316,9 +316,9 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e
 		}
 		// There's more than one previous state - run them all through state res
 		var err error
-		t.roomsMu.Lock(e.RoomID())
+		t.roomsMu.Lock(e.RoomID().String())
 		resolvedState, err = t.resolveStatesAndCheck(ctx, roomVersion, respStates, e)
-		t.roomsMu.Unlock(e.RoomID())
+		t.roomsMu.Unlock(e.RoomID().String())
 		switch err2 := err.(type) {
 		case gomatrixserverlib.EventValidationError:
 			if !err2.Persistable {
@@ -510,7 +510,7 @@ retryAllowedState:
 	}); err != nil {
 		switch missing := err.(type) {
 		case gomatrixserverlib.MissingAuthEventError:
-			h, err2 := t.lookupEvent(ctx, roomVersion, backwardsExtremity.RoomID(), missing.AuthEventID, true)
+			h, err2 := t.lookupEvent(ctx, roomVersion, backwardsExtremity.RoomID().String(), missing.AuthEventID, true)
 			switch e := err2.(type) {
 			case gomatrixserverlib.EventValidationError:
 				if !e.Persistable {
@@ -546,7 +546,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver
 	trace, ctx := internal.StartRegion(ctx, "getMissingEvents")
 	defer trace.EndRegion()
 
-	logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
+	logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID().String())
 	latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID)
 	if err != nil {
 		return nil, false, false, fmt.Errorf("t.DB.LatestEventIDs: %w", err)
@@ -560,7 +560,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver
 	var missingResp *fclient.RespMissingEvents
 	for _, server := range t.servers {
 		var m fclient.RespMissingEvents
-		if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID(), fclient.MissingEvents{
+		if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID().String(), fclient.MissingEvents{
 			Limit: 20,
 			// The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events.
 			EarliestEvents: latestEvents,
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 33200e819..dafa58736 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -301,7 +301,7 @@ func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent
 		return ids, nil
 	}
 	if len(targetEvent.PrevEventIDs()) == 0 && targetEvent.Type() == "m.room.create" && targetEvent.StateKeyEquals("") {
-		util.GetLogger(ctx).WithField("room_id", targetEvent.RoomID()).Info("Backfilled to the beginning of the room")
+		util.GetLogger(ctx).WithField("room_id", targetEvent.RoomID().String()).Info("Backfilled to the beginning of the room")
 		b.eventIDToBeforeStateIDs[targetEvent.EventID()] = []string{}
 		return nil, nil
 	}
@@ -494,11 +494,7 @@ FindSuccessor:
 	// Store the server names in a temporary map to avoid duplicates.
 	serverSet := make(map[spec.ServerName]bool)
 	for _, event := range memberEvents {
-		validRoomID, err := spec.NewRoomID(event.RoomID())
-		if err != nil {
-			continue
-		}
-		if sender, err := b.querier.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()); err == nil {
+		if sender, err := b.querier.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()); err == nil {
 			serverSet[sender.Domain()] = true
 		}
 	}
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index e07780d68..3abb69cb9 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -100,16 +100,12 @@ func (r *Inviter) ProcessInviteMembership(
 	var outputUpdates []api.OutputEvent
 	var updater *shared.MembershipUpdater
 
-	validRoomID, err := spec.NewRoomID(inviteEvent.RoomID())
-	if err != nil {
-		return nil, err
-	}
-	userID, err := r.RSAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*inviteEvent.StateKey()))
+	userID, err := r.RSAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), spec.SenderID(*inviteEvent.StateKey()))
 	if err != nil {
 		return nil, api.ErrInvalidID{Err: fmt.Errorf("the user ID %s is invalid", *inviteEvent.StateKey())}
 	}
 	isTargetLocal := r.Cfg.Matrix.IsLocalServerName(userID.Domain())
-	if updater, err = r.DB.MembershipUpdater(ctx, inviteEvent.RoomID(), *inviteEvent.StateKey(), isTargetLocal, inviteEvent.Version()); err != nil {
+	if updater, err = r.DB.MembershipUpdater(ctx, inviteEvent.RoomID().String(), *inviteEvent.StateKey(), isTargetLocal, inviteEvent.Version()); err != nil {
 		return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err)
 	}
 	outputUpdates, err = helpers.UpdateToInviteMembership(updater, &types.Event{
diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go
index 7274be520..76eba12be 100644
--- a/roomserver/internal/query/query_room_hierarchy.go
+++ b/roomserver/internal/query/query_room_hierarchy.go
@@ -513,14 +513,14 @@ func restrictedJoinRuleAllowedRooms(ctx context.Context, joinRuleEv *types.Heade
 	}
 	var jrContent gomatrixserverlib.JoinRuleContent
 	if err := json.Unmarshal(joinRuleEv.Content(), &jrContent); err != nil {
-		util.GetLogger(ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID(), err)
+		util.GetLogger(ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID().String(), err)
 		return nil
 	}
 	for _, allow := range jrContent.Allow {
 		if allow.Type == spec.MRoomMembership {
 			allowedRoomID, err := spec.NewRoomID(allow.RoomID)
 			if err != nil {
-				util.GetLogger(ctx).Warnf("invalid room ID '%s' found in join_rule on room %s: %s", allow.RoomID, joinRuleEv.RoomID(), err)
+				util.GetLogger(ctx).Warnf("invalid room ID '%s' found in join_rule on room %s: %s", allow.RoomID, joinRuleEv.RoomID().String(), err)
 			} else {
 				allows = append(allows, *allowedRoomID)
 			}
diff --git a/roomserver/internal/query/query_test.go b/roomserver/internal/query/query_test.go
index 619d93030..296960b2f 100644
--- a/roomserver/internal/query/query_test.go
+++ b/roomserver/internal/query/query_test.go
@@ -49,6 +49,7 @@ func (db *getEventDB) addFakeEvent(eventID string, authIDs []string) error {
 	}
 	builder := map[string]interface{}{
 		"event_id":    eventID,
+		"room_id":     "!room:a",
 		"auth_events": authEvents,
 	}
 
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index b09c5afbd..3331c6029 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -696,8 +696,8 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserver
 		return nil, fmt.Errorf("extractRoomVersionFromCreateEvent: %w", err)
 	}
 
-	roomNID, nidOK := d.Cache.GetRoomServerRoomNID(event.RoomID())
-	cachedRoomVersion, versionOK := d.Cache.GetRoomVersion(event.RoomID())
+	roomNID, nidOK := d.Cache.GetRoomServerRoomNID(event.RoomID().String())
+	cachedRoomVersion, versionOK := d.Cache.GetRoomVersion(event.RoomID().String())
 	// if we found both, the roomNID and version in our cache, no need to query the database
 	if nidOK && versionOK {
 		return &types.RoomInfo{
@@ -707,14 +707,14 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserver
 	}
 
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion)
+		roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID().String(), roomVersion)
 		if err != nil {
 			return err
 		}
 		return nil
 	})
 	if roomVersion != "" {
-		d.Cache.StoreRoomVersion(event.RoomID(), roomVersion)
+		d.Cache.StoreRoomVersion(event.RoomID().String(), roomVersion)
 	}
 	return &types.RoomInfo{
 		RoomVersion: roomVersion,
@@ -1026,24 +1026,19 @@ func (d *EventDatabase) MaybeRedactEvent(
 		case validated || redactedEvent == nil || redactionEvent == nil:
 			// we've seen this redaction before or there is nothing to redact
 			return nil
-		case redactedEvent.RoomID() != redactionEvent.RoomID():
+		case redactedEvent.RoomID().String() != redactionEvent.RoomID().String():
 			// redactions across rooms aren't allowed
 			ignoreRedaction = true
 			return nil
 		}
 
-		var validRoomID *spec.RoomID
-		validRoomID, err = spec.NewRoomID(redactedEvent.RoomID())
-		if err != nil {
-			return err
-		}
 		sender1Domain := ""
-		sender1, err1 := querier.QueryUserIDForSender(ctx, *validRoomID, redactedEvent.SenderID())
+		sender1, err1 := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactedEvent.SenderID())
 		if err1 == nil {
 			sender1Domain = string(sender1.Domain())
 		}
 		sender2Domain := ""
-		sender2, err2 := querier.QueryUserIDForSender(ctx, *validRoomID, redactionEvent.SenderID())
+		sender2, err2 := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactionEvent.SenderID())
 		if err2 == nil {
 			sender2Domain = string(sender2.Domain())
 		}
@@ -1522,7 +1517,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu
 		}
 		result[i] = tables.StrippedEvent{
 			EventType:    ev.Type(),
-			RoomID:       ev.RoomID(),
+			RoomID:       ev.RoomID().String(),
 			StateKey:     *ev.StateKey(),
 			ContentValue: tables.ExtractContentValue(&types.HeaderedEvent{PDU: ev}),
 		}
diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go
index 7f8e2de03..15811710d 100644
--- a/setup/mscs/msc2836/msc2836.go
+++ b/setup/mscs/msc2836/msc2836.go
@@ -271,7 +271,7 @@ func (rc *reqCtx) process() (*MSC2836EventRelationshipsResponse, *util.JSONRespo
 		event = rc.fetchUnknownEvent(rc.req.EventID, rc.req.RoomID)
 	}
 	if rc.req.RoomID == "" && event != nil {
-		rc.req.RoomID = event.RoomID()
+		rc.req.RoomID = event.RoomID().String()
 	}
 	if event == nil || !rc.authorisedToSeeEvent(event) {
 		return nil, &util.JSONResponse{
@@ -526,7 +526,7 @@ func (rc *reqCtx) authorisedToSeeEvent(event *types.HeaderedEvent) bool {
 		// make sure the server is in this room
 		var res fs.QueryJoinedHostServerNamesInRoomResponse
 		err := rc.fsAPI.QueryJoinedHostServerNamesInRoom(rc.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{
-			RoomID: event.RoomID(),
+			RoomID: event.RoomID().String(),
 		}, &res)
 		if err != nil {
 			util.GetLogger(rc.ctx).WithError(err).Error("authorisedToSeeEvent: failed to QueryJoinedHostServerNamesInRoom")
@@ -545,7 +545,7 @@ func (rc *reqCtx) authorisedToSeeEvent(event *types.HeaderedEvent) bool {
 	// TODO: This does not honour m.room.create content
 	var queryMembershipRes roomserver.QueryMembershipForUserResponse
 	err := rc.rsAPI.QueryMembershipForUser(rc.ctx, &roomserver.QueryMembershipForUserRequest{
-		RoomID: event.RoomID(),
+		RoomID: event.RoomID().String(),
 		UserID: rc.userID,
 	}, &queryMembershipRes)
 	if err != nil {
@@ -612,7 +612,7 @@ func (rc *reqCtx) lookForEvent(eventID string) *types.HeaderedEvent {
 			// inject all the events into the roomserver then return the event in question
 			rc.injectResponseToRoomserver(queryRes)
 			for _, ev := range queryRes.ParsedEvents {
-				if ev.EventID() == eventID && rc.req.RoomID == ev.RoomID() {
+				if ev.EventID() == eventID && rc.req.RoomID == ev.RoomID().String() {
 					return &types.HeaderedEvent{PDU: ev}
 				}
 			}
@@ -629,7 +629,7 @@ func (rc *reqCtx) lookForEvent(eventID string) *types.HeaderedEvent {
 			}
 		}
 	}
-	if rc.req.RoomID == event.RoomID() {
+	if rc.req.RoomID == event.RoomID().String() {
 		return event
 	}
 	return nil
diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go
index 73bd6ed4f..ade2a1616 100644
--- a/setup/mscs/msc2836/storage.go
+++ b/setup/mscs/msc2836/storage.go
@@ -239,7 +239,7 @@ func (p *DB) StoreRelation(ctx context.Context, ev *types.HeaderedEvent) error {
 			return err
 		}
 		util.GetLogger(ctx).Infof("StoreRelation child=%s parent=%s rel_type=%s", child, parent, relType)
-		_, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID(), count, base64.RawStdEncoding.EncodeToString(hash), 0)
+		_, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID().String(), count, base64.RawStdEncoding.EncodeToString(hash), 0)
 		return err
 	})
 }
diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go
index 3ed455e9f..76b447133 100644
--- a/syncapi/consumers/clientapi.go
+++ b/syncapi/consumers/clientapi.go
@@ -113,7 +113,7 @@ func (s *OutputClientDataConsumer) Start() error {
 				id = streamPos
 				e := fulltext.IndexElement{
 					EventID:        ev.EventID(),
-					RoomID:         ev.RoomID(),
+					RoomID:         ev.RoomID().String(),
 					StreamPosition: streamPos,
 				}
 				e.SetContentType(ev.Type())
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 9df5e0f9c..666f900d7 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -166,9 +166,9 @@ func (s *OutputRoomEventConsumer) onRedactEvent(
 		return err
 	}
 
-	if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID(), msg.RedactedEventID); err != nil {
+	if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID().String(), msg.RedactedEventID); err != nil {
 		log.WithFields(log.Fields{
-			"room_id":           msg.RedactedBecause.RoomID(),
+			"room_id":           msg.RedactedBecause.RoomID().String(),
 			"event_id":          msg.RedactedBecause.EventID(),
 			"redacted_event_id": msg.RedactedEventID,
 		}).WithError(err).Warn("Failed to redact relations")
@@ -222,7 +222,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 	// Finally, work out if there are any more events missing.
 	if len(missingEventIDs) > 0 {
 		eventsReq := &api.QueryEventsByIDRequest{
-			RoomID:   ev.RoomID(),
+			RoomID:   ev.RoomID().String(),
 			EventIDs: missingEventIDs,
 		}
 		eventsRes := &api.QueryEventsByIDResponse{}
@@ -257,17 +257,12 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 	}
 
 	if msg.RewritesState {
-		if err = s.db.PurgeRoomState(ctx, ev.RoomID()); err != nil {
+		if err = s.db.PurgeRoomState(ctx, ev.RoomID().String()); err != nil {
 			return fmt.Errorf("s.db.PurgeRoom: %w", err)
 		}
 	}
 
-	validRoomID, err := spec.NewRoomID(ev.RoomID())
-	if err != nil {
-		return err
-	}
-
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, ev.SenderID())
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID())
 	if err != nil {
 		return err
 	}
@@ -307,7 +302,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 	}
 
 	s.pduStream.Advance(pduPos)
-	s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos})
+	s.notifier.OnNewEvent(ev, ev.RoomID().String(), nil, types.StreamingToken{PDUPosition: pduPos})
 
 	return nil
 }
@@ -324,12 +319,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 	// old events in the sync API, this should at least prevent us
 	// from confusing clients into thinking they've joined/left rooms.
 
-	validRoomID, err := spec.NewRoomID(ev.RoomID())
-	if err != nil {
-		return err
-	}
-
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, ev.SenderID())
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID())
 	if err != nil {
 		return err
 	}
@@ -355,7 +345,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 
 	if err = s.db.UpdateRelations(ctx, ev); err != nil {
 		log.WithFields(log.Fields{
-			"room_id":  ev.RoomID(),
+			"room_id":  ev.RoomID().String(),
 			"event_id": ev.EventID(),
 			"type":     ev.Type(),
 		}).WithError(err).Warn("Failed to update relations")
@@ -368,7 +358,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 	}
 
 	s.pduStream.Advance(pduPos)
-	s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos})
+	s.notifier.OnNewEvent(ev, ev.RoomID().String(), nil, types.StreamingToken{PDUPosition: pduPos})
 
 	return nil
 }
@@ -388,11 +378,7 @@ func (s *OutputRoomEventConsumer) notifyJoinedPeeks(ctx context.Context, ev *rst
 			return sp, fmt.Errorf("unexpected nil state_key")
 		}
 
-		validRoomID, err := spec.NewRoomID(ev.RoomID())
-		if err != nil {
-			return sp, err
-		}
-		userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*ev.StateKey()))
+		userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey()))
 		if err != nil || userID == nil {
 			return sp, fmt.Errorf("failed getting userID for sender: %w", err)
 		}
@@ -401,7 +387,7 @@ func (s *OutputRoomEventConsumer) notifyJoinedPeeks(ctx context.Context, ev *rst
 		}
 
 		// cancel any peeks for it
-		peekSP, peekErr := s.db.DeletePeeks(ctx, ev.RoomID(), *ev.StateKey())
+		peekSP, peekErr := s.db.DeletePeeks(ctx, ev.RoomID().String(), *ev.StateKey())
 		if peekErr != nil {
 			return sp, fmt.Errorf("s.db.DeletePeeks: %w", peekErr)
 		}
@@ -419,11 +405,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent(
 		return
 	}
 
-	validRoomID, err := spec.NewRoomID(msg.Event.RoomID())
-	if err != nil {
-		return
-	}
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*msg.Event.StateKey()))
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, msg.Event.RoomID(), spec.SenderID(*msg.Event.StateKey()))
 	if err != nil || userID == nil {
 		return
 	}
@@ -560,15 +542,10 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent)
 	var succeeded bool
 	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return event, err
-	}
-
 	sKeyUser := ""
 	if stateKey != "" {
 		var sku *spec.UserID
-		sku, err = s.rsAPI.QueryUserIDForSender(s.ctx, *validRoomID, spec.SenderID(stateKey))
+		sku, err = s.rsAPI.QueryUserIDForSender(s.ctx, event.RoomID(), spec.SenderID(stateKey))
 		if err == nil && sku != nil {
 			sKeyUser = sku.String()
 			event.StateKeyResolved = &sKeyUser
@@ -576,13 +553,13 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent)
 	}
 
 	prevEvent, err := snapshot.GetStateEvent(
-		s.ctx, event.RoomID(), event.Type(), sKeyUser,
+		s.ctx, event.RoomID().String(), event.Type(), sKeyUser,
 	)
 	if err != nil {
 		return event, err
 	}
 
-	userID, err := s.rsAPI.QueryUserIDForSender(s.ctx, *validRoomID, event.SenderID())
+	userID, err := s.rsAPI.QueryUserIDForSender(s.ctx, event.RoomID(), event.SenderID())
 	if err != nil {
 		return event, err
 	}
@@ -610,7 +587,7 @@ func (s *OutputRoomEventConsumer) writeFTS(ev *rstypes.HeaderedEvent, pduPositio
 	}
 	e := fulltext.IndexElement{
 		EventID:        ev.EventID(),
-		RoomID:         ev.RoomID(),
+		RoomID:         ev.RoomID().String(),
 		StreamPosition: int64(pduPosition),
 	}
 	e.SetContentType(ev.Type())
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
index 7aae9fd38..48475327d 100644
--- a/syncapi/internal/history_visibility.go
+++ b/syncapi/internal/history_visibility.go
@@ -118,26 +118,23 @@ func ApplyHistoryVisibilityFilter(
 	start := time.Now()
 
 	// try to get the current membership of the user
-	membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID(), userID.String(), math.MaxInt64)
+	membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID().String(), userID.String(), math.MaxInt64)
 	if err != nil {
 		return nil, err
 	}
 
 	// Get the mapping from eventID -> eventVisibility
 	eventsFiltered := make([]*types.HeaderedEvent, 0, len(events))
-	firstEvRoomID, err := spec.NewRoomID(events[0].RoomID())
+	firstEvRoomID := events[0].RoomID()
+	senderID, err := rsAPI.QuerySenderIDForUser(ctx, firstEvRoomID, userID)
 	if err != nil {
 		return nil, err
 	}
-	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *firstEvRoomID, userID)
-	if err != nil {
-		return nil, err
-	}
-	visibilities := visibilityForEvents(ctx, rsAPI, events, senderID, *firstEvRoomID)
+	visibilities := visibilityForEvents(ctx, rsAPI, events, senderID, firstEvRoomID)
 
 	for _, ev := range events {
 		// Validate same room assumption
-		if ev.RoomID() != firstEvRoomID.String() {
+		if ev.RoomID().String() != firstEvRoomID.String() {
 			return nil, fmt.Errorf("events from different rooms supplied to ApplyHistoryVisibilityFilter")
 		}
 
diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index a8733f6fe..07b80b165 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -101,20 +101,13 @@ func (n *Notifier) OnNewEvent(
 	n._removeEmptyUserStreams()
 
 	if ev != nil {
-		validRoomID, err := spec.NewRoomID(ev.RoomID())
-		if err != nil {
-			log.WithError(err).WithField("event_id", ev.EventID()).Errorf(
-				"Notifier.OnNewEvent: RoomID is invalid",
-			)
-			return
-		}
 		// Map this event's room_id to a list of joined users, and wake them up.
-		usersToNotify := n._joinedUsers(ev.RoomID())
+		usersToNotify := n._joinedUsers(ev.RoomID().String())
 		// Map this event's room_id to a list of peeking devices, and wake them up.
-		peekingDevicesToNotify := n._peekingDevices(ev.RoomID())
+		peekingDevicesToNotify := n._peekingDevices(ev.RoomID().String())
 		// If this is an invite, also add in the invitee to this list.
 		if ev.Type() == "m.room.member" && ev.StateKey() != nil {
-			targetUserID, err := n.rsAPI.QueryUserIDForSender(context.Background(), *validRoomID, spec.SenderID(*ev.StateKey()))
+			targetUserID, err := n.rsAPI.QueryUserIDForSender(context.Background(), ev.RoomID(), spec.SenderID(*ev.StateKey()))
 			if err != nil || targetUserID == nil {
 				log.WithError(err).WithField("event_id", ev.EventID()).Errorf(
 					"Notifier.OnNewEvent: Failed to find the userID for this event",
@@ -134,11 +127,11 @@ func (n *Notifier) OnNewEvent(
 						// Manually append the new user's ID so they get notified
 						// along all members in the room
 						usersToNotify = append(usersToNotify, targetUserID.String())
-						n._addJoinedUser(ev.RoomID(), targetUserID.String())
+						n._addJoinedUser(ev.RoomID().String(), targetUserID.String())
 					case spec.Leave:
 						fallthrough
 					case spec.Ban:
-						n._removeJoinedUser(ev.RoomID(), targetUserID.String())
+						n._removeJoinedUser(ev.RoomID().String(), targetUserID.String())
 					}
 				}
 			}
diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go
index bf0f9bf8c..886b11675 100644
--- a/syncapi/routing/getevent.go
+++ b/syncapi/routing/getevent.go
@@ -129,14 +129,7 @@ func GetEvent(
 
 	sk := events[0].StateKey()
 	if sk != nil && *sk != "" {
-		evRoomID, err := spec.NewRoomID(events[0].RoomID())
-		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.BadJSON("roomID is invalid"),
-			}
-		}
-		skUserID, err := rsAPI.QueryUserIDForSender(ctx, *evRoomID, spec.SenderID(*events[0].StateKey()))
+		skUserID, err := rsAPI.QueryUserIDForSender(ctx, events[0].RoomID(), spec.SenderID(*events[0].StateKey()))
 		if err == nil && skUserID != nil {
 			skString := skUserID.String()
 			sk = &skString
diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go
index 5e5d0125f..e849adf6d 100644
--- a/syncapi/routing/memberships.go
+++ b/syncapi/routing/memberships.go
@@ -152,15 +152,7 @@ func GetMemberships(
 				}
 			}
 
-			validRoomID, err := spec.NewRoomID(ev.RoomID())
-			if err != nil {
-				util.GetLogger(req.Context()).WithError(err).Error("roomID is invalid")
-				return util.JSONResponse{
-					Code: http.StatusInternalServerError,
-					JSON: spec.InternalServerError{},
-				}
-			}
-			userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, ev.SenderID())
+			userID, err := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID())
 			if err != nil || userID == nil {
 				util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
 				return util.JSONResponse{
diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go
index 7d5c061b7..f574781aa 100644
--- a/syncapi/routing/search.go
+++ b/syncapi/routing/search.go
@@ -205,12 +205,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 
 		profileInfos := make(map[string]ProfileInfoResponse)
 		for _, ev := range append(eventsBefore, eventsAfter...) {
-			validRoomID, roomErr := spec.NewRoomID(ev.RoomID())
-			if err != nil {
-				logrus.WithError(roomErr).WithField("room_id", ev.RoomID()).Warn("failed to query userprofile")
-				continue
-			}
-			userID, queryErr := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, ev.SenderID())
+			userID, queryErr := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID())
 			if queryErr != nil {
 				logrus.WithError(queryErr).WithField("sender_id", ev.SenderID()).Warn("failed to query userprofile")
 				continue
@@ -218,7 +213,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 
 			profile, ok := knownUsersProfiles[userID.String()]
 			if !ok {
-				stateEvent, stateErr := snapshot.GetStateEvent(ctx, ev.RoomID(), spec.MRoomMember, string(ev.SenderID()))
+				stateEvent, stateErr := snapshot.GetStateEvent(ctx, ev.RoomID().String(), spec.MRoomMember, string(ev.SenderID()))
 				if stateErr != nil {
 					logrus.WithError(stateErr).WithField("sender_id", event.SenderID()).Warn("failed to query userprofile")
 					continue
@@ -236,19 +231,14 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 		}
 
 		sender := spec.UserID{}
-		validRoomID, roomErr := spec.NewRoomID(event.RoomID())
-		if err != nil {
-			logrus.WithError(roomErr).WithField("room_id", event.RoomID()).Warn("failed to query userprofile")
-			continue
-		}
-		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, event.SenderID())
+		userID, err := rsAPI.QueryUserIDForSender(req.Context(), event.RoomID(), event.SenderID())
 		if err == nil && userID != nil {
 			sender = *userID
 		}
 
 		sk := event.StateKey()
 		if sk != nil && *sk != "" {
-			skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(*event.StateKey()))
+			skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), event.RoomID(), spec.SenderID(*event.StateKey()))
 			if err == nil && skUserID != nil {
 				skString := skUserID.String()
 				sk = &skString
@@ -269,12 +259,12 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 			Rank:   eventScore[event.EventID()].Score,
 			Result: synctypes.ToClientEvent(event, synctypes.FormatAll, sender.String(), sk, event.Unsigned()),
 		})
-		roomGroup := groups[event.RoomID()]
+		roomGroup := groups[event.RoomID().String()]
 		roomGroup.Results = append(roomGroup.Results, event.EventID())
-		groups[event.RoomID()] = roomGroup
-		if _, ok := stateForRooms[event.RoomID()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok {
+		groups[event.RoomID().String()] = roomGroup
+		if _, ok := stateForRooms[event.RoomID().String()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok {
 			stateFilter := synctypes.DefaultStateFilter()
-			state, err := snapshot.CurrentState(ctx, event.RoomID(), &stateFilter, nil)
+			state, err := snapshot.CurrentState(ctx, event.RoomID().String(), &stateFilter, nil)
 			if err != nil {
 				logrus.WithError(err).Error("unable to get current state")
 				return util.JSONResponse{
@@ -282,7 +272,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 					JSON: spec.InternalServerError{},
 				}
 			}
-			stateForRooms[event.RoomID()] = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(state), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			stateForRooms[event.RoomID().String()] = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(state), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 				return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID)
 			})
 		}
@@ -328,19 +318,19 @@ func contextEvents(
 	roomFilter *synctypes.RoomEventFilter,
 	searchReq SearchRequest,
 ) ([]*types.HeaderedEvent, []*types.HeaderedEvent, error) {
-	id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID(), event.EventID())
+	id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID().String(), event.EventID())
 	if err != nil {
 		logrus.WithError(err).Error("failed to query context event")
 		return nil, nil, err
 	}
 	roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.BeforeLimit
-	eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter)
+	eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID().String(), roomFilter)
 	if err != nil {
 		logrus.WithError(err).Error("failed to query before context event")
 		return nil, nil, err
 	}
 	roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.AfterLimit
-	_, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter)
+	_, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID().String(), roomFilter)
 	if err != nil {
 		logrus.WithError(err).Error("failed to query after context event")
 		return nil, nil, err
diff --git a/syncapi/routing/search_test.go b/syncapi/routing/search_test.go
index 905a9a1ac..a983bb7b5 100644
--- a/syncapi/routing/search_test.go
+++ b/syncapi/routing/search_test.go
@@ -238,7 +238,7 @@ func TestSearch(t *testing.T) {
 			}
 			elements = append(elements, fulltext.IndexElement{
 				EventID:        x.EventID(),
-				RoomID:         x.RoomID(),
+				RoomID:         x.RoomID().String(),
 				Content:        string(x.Content()),
 				ContentType:    x.Type(),
 				StreamPosition: int64(sp),
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 112fa9d4a..b0148bef5 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -340,7 +340,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 	stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt)
 	_, err = stmt.ExecContext(
 		ctx,
-		event.RoomID(),
+		event.RoomID().String(),
 		event.EventID(),
 		event.Type(),
 		event.UserID.String(),
diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go
index 7b8d2d733..1f46cd09d 100644
--- a/syncapi/storage/postgres/invites_table.go
+++ b/syncapi/storage/postgres/invites_table.go
@@ -99,7 +99,7 @@ func (s *inviteEventsStatements) InsertInviteEvent(
 
 	err = sqlutil.TxStmt(txn, s.insertInviteEventStmt).QueryRowContext(
 		ctx,
-		inviteEvent.RoomID(),
+		inviteEvent.RoomID().String(),
 		inviteEvent.EventID(),
 		inviteEvent.UserID.String(),
 		headeredJSON,
diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go
index 09b47432b..fcbe14b16 100644
--- a/syncapi/storage/postgres/memberships_table.go
+++ b/syncapi/storage/postgres/memberships_table.go
@@ -108,7 +108,7 @@ func (s *membershipsStatements) UpsertMembership(
 	}
 	_, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext(
 		ctx,
-		event.RoomID(),
+		event.RoomID().String(),
 		event.StateKeyResolved,
 		membership,
 		event.EventID(),
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index b58cf59f0..b2d191111 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -334,7 +334,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		if err := json.Unmarshal(eventBytes, &ev); err != nil {
 			return nil, nil, err
 		}
-		needSet := stateNeeded[ev.RoomID()]
+		needSet := stateNeeded[ev.RoomID().String()]
 		if needSet == nil { // make set if required
 			needSet = make(map[string]bool)
 		}
@@ -344,7 +344,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		for _, id := range addIDs {
 			needSet[id] = true
 		}
-		stateNeeded[ev.RoomID()] = needSet
+		stateNeeded[ev.RoomID().String()] = needSet
 		ev.Visibility = historyVisibility
 
 		eventIDToEvent[eventID] = types.StreamEvent{
@@ -403,7 +403,7 @@ func (s *outputRoomEventsStatements) InsertEvent(
 	stmt := sqlutil.TxStmt(txn, s.insertEventStmt)
 	err = stmt.QueryRowContext(
 		ctx,
-		event.RoomID(),
+		event.RoomID().String(),
 		event.EventID(),
 		headeredJSON,
 		event.Type(),
diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go
index b281f3300..2158d99ec 100644
--- a/syncapi/storage/postgres/output_room_events_topology_table.go
+++ b/syncapi/storage/postgres/output_room_events_topology_table.go
@@ -107,7 +107,7 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology(
 	ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, pos types.StreamPosition,
 ) (topoPos types.StreamPosition, err error) {
 	err = sqlutil.TxStmt(txn, s.insertEventInTopologyStmt).QueryRowContext(
-		ctx, event.EventID(), event.Depth(), event.RoomID(), pos,
+		ctx, event.EventID(), event.Depth(), event.RoomID().String(), pos,
 	).Scan(&topoPos)
 	return
 }
diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go
index 69e64cc79..0f4080d53 100644
--- a/syncapi/storage/shared/storage_consumer.go
+++ b/syncapi/storage/shared/storage_consumer.go
@@ -114,14 +114,7 @@ func (d *Database) StreamEventsToEvents(ctx context.Context, device *userapi.Dev
 				}).WithError(err).Warnf("Failed to add transaction ID to event")
 				continue
 			}
-			roomID, err := spec.NewRoomID(in[i].RoomID())
-			if err != nil {
-				logrus.WithFields(logrus.Fields{
-					"event_id": out[i].EventID(),
-				}).WithError(err).Warnf("Room ID is invalid")
-				continue
-			}
-			deviceSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID)
+			deviceSenderID, err := rsAPI.QuerySenderIDForUser(ctx, in[i].RoomID(), *userID)
 			if err != nil || deviceSenderID == nil {
 				logrus.WithFields(logrus.Fields{
 					"event_id": out[i].EventID(),
@@ -236,7 +229,7 @@ func (d *Database) UpsertAccountData(
 // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such.
 // This function should always be called within a sqlutil.Writer for safety in SQLite.
 func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *rstypes.HeaderedEvent) error {
-	if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil {
+	if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID().String(), ev.EventID()); err != nil {
 		return err
 	}
 
@@ -257,7 +250,7 @@ func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, e
 
 		// If the event is missing, consider it a backward extremity.
 		if !found {
-			if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil {
+			if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID().String(), ev.EventID(), eID); err != nil {
 				return err
 			}
 		}
@@ -426,7 +419,7 @@ func (d *Database) fetchStateEvents(
 		}
 		// we know we got them all otherwise an error would've been returned, so just loop the events
 		for _, ev := range evs {
-			roomID := ev.RoomID()
+			roomID := ev.RoomID().String()
 			stateBetween[roomID] = append(stateBetween[roomID], ev)
 		}
 	}
@@ -522,11 +515,7 @@ func getMembershipFromEvent(ctx context.Context, ev gomatrixserverlib.PDU, userI
 	if err != nil {
 		return "", ""
 	}
-	roomID, err := spec.NewRoomID(ev.RoomID())
-	if err != nil {
-		return "", ""
-	}
-	senderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *fullUser)
+	senderID, err := rsAPI.QuerySenderIDForUser(ctx, ev.RoomID(), *fullUser)
 	if err != nil || senderID == nil {
 		return "", ""
 	}
@@ -626,7 +615,7 @@ func (d *Database) UpdateRelations(ctx context.Context, event *rstypes.HeaderedE
 	default:
 		return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 			return d.Relations.InsertRelation(
-				ctx, txn, event.RoomID(), content.Relations.EventID,
+				ctx, txn, event.RoomID().String(), content.Relations.EventID,
 				event.EventID(), event.Type(), content.Relations.RelationType,
 			)
 		})
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 3bd19b367..78b2e397c 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -339,7 +339,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 	stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt)
 	_, err = stmt.ExecContext(
 		ctx,
-		event.RoomID(),
+		event.RoomID().String(),
 		event.EventID(),
 		event.Type(),
 		event.UserID.String(),
diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go
index 7e0d895f1..ebb469d24 100644
--- a/syncapi/storage/sqlite3/invites_table.go
+++ b/syncapi/storage/sqlite3/invites_table.go
@@ -106,7 +106,7 @@ func (s *inviteEventsStatements) InsertInviteEvent(
 	_, err = stmt.ExecContext(
 		ctx,
 		streamPos,
-		inviteEvent.RoomID(),
+		inviteEvent.RoomID().String(),
 		inviteEvent.EventID(),
 		inviteEvent.UserID.String(),
 		headeredJSON,
diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go
index a9e880d2a..05f756fda 100644
--- a/syncapi/storage/sqlite3/memberships_table.go
+++ b/syncapi/storage/sqlite3/memberships_table.go
@@ -111,7 +111,7 @@ func (s *membershipsStatements) UpsertMembership(
 	}
 	_, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext(
 		ctx,
-		event.RoomID(),
+		event.RoomID().String(),
 		event.StateKeyResolved,
 		membership,
 		event.EventID(),
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 06c65419a..93caee806 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -254,7 +254,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		if err := json.Unmarshal(eventBytes, &ev); err != nil {
 			return nil, nil, err
 		}
-		needSet := stateNeeded[ev.RoomID()]
+		needSet := stateNeeded[ev.RoomID().String()]
 		if needSet == nil { // make set if required
 			needSet = make(map[string]bool)
 		}
@@ -264,7 +264,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		for _, id := range addIDs {
 			needSet[id] = true
 		}
-		stateNeeded[ev.RoomID()] = needSet
+		stateNeeded[ev.RoomID().String()] = needSet
 		ev.Visibility = historyVisibility
 
 		eventIDToEvent[eventID] = types.StreamEvent{
@@ -344,7 +344,7 @@ func (s *outputRoomEventsStatements) InsertEvent(
 	_, err = insertStmt.ExecContext(
 		ctx,
 		streamPos,
-		event.RoomID(),
+		event.RoomID().String(),
 		event.EventID(),
 		headeredJSON,
 		event.Type(),
diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go
index 614e1df9e..36967d1e7 100644
--- a/syncapi/storage/sqlite3/output_room_events_topology_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go
@@ -106,7 +106,7 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology(
 	ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, pos types.StreamPosition,
 ) (types.StreamPosition, error) {
 	_, err := sqlutil.TxStmt(txn, s.insertEventInTopologyStmt).ExecContext(
-		ctx, event.EventID(), event.Depth(), event.RoomID(), pos,
+		ctx, event.EventID(), event.Depth(), event.RoomID().String(), pos,
 	)
 	return types.StreamPosition(event.Depth()), err
 }
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 1ce3346f4..1424dc2e6 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -70,18 +70,14 @@ func (p *InviteStreamProvider) IncrementalSync(
 
 	for roomID, inviteEvent := range invites {
 		user := spec.UserID{}
-		validRoomID, err := spec.NewRoomID(inviteEvent.RoomID())
-		if err != nil {
-			continue
-		}
-		sender, err := p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, inviteEvent.SenderID())
+		sender, err := p.rsAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), inviteEvent.SenderID())
 		if err == nil && sender != nil {
 			user = *sender
 		}
 
 		sk := inviteEvent.StateKey()
 		if sk != nil && *sk != "" {
-			skUserID, err := p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*inviteEvent.StateKey()))
+			skUserID, err := p.rsAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), spec.SenderID(*inviteEvent.StateKey()))
 			if err == nil && skUserID != nil {
 				skString := skUserID.String()
 				sk = &skString
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 3b27acd8a..eb1f0ef2e 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -476,9 +476,8 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 	newPls := make(map[string]int64)
 	var userID *spec.UserID
 	for user, level := range pls.Users {
-		validRoomID, _ := spec.NewRoomID(ev.RoomID())
 		if eventFormat != synctypes.FormatSyncFederation {
-			userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user))
+			userID, err = p.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(user))
 			if err != nil {
 				return nil, err
 			}
@@ -515,9 +514,8 @@ func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstyp
 
 	newPls = make(map[string]int64)
 	for user, level := range pls.Users {
-		validRoomID, _ := spec.NewRoomID(ev.RoomID())
 		if eventFormat != synctypes.FormatSyncFederation {
-			userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user))
+			userID, err = p.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(user))
 			if err != nil {
 				return nil, err
 			}
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index f29719953..ac5268511 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -1401,7 +1401,7 @@ func toNATSMsgs(t *testing.T, cfg *config.Dendrite, input ...*rstypes.HeaderedEv
 		if ev.StateKey() != nil {
 			addsStateIDs = append(addsStateIDs, ev.EventID())
 		}
-		result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID(), api.OutputEvent{
+		result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID().String(), api.OutputEvent{
 			Type: rsapi.OutputTypeNewRoomEvent,
 			NewRoomEvent: &rsapi.OutputNewRoomEvent{
 				Event:             ev,
diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go
index 7e5b1c1bc..e0616e11d 100644
--- a/syncapi/synctypes/clientevent.go
+++ b/syncapi/synctypes/clientevent.go
@@ -84,18 +84,14 @@ func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat,
 		}
 
 		sender := spec.UserID{}
-		validRoomID, err := spec.NewRoomID(se.RoomID())
-		if err != nil {
-			continue
-		}
-		userID, err := userIDForSender(*validRoomID, se.SenderID())
+		userID, err := userIDForSender(se.RoomID(), se.SenderID())
 		if err == nil && userID != nil {
 			sender = *userID
 		}
 
 		sk := se.StateKey()
 		if sk != nil && *sk != "" {
-			skUserID, err := userIDForSender(*validRoomID, spec.SenderID(*sk))
+			skUserID, err := userIDForSender(se.RoomID(), spec.SenderID(*sk))
 			if err == nil && skUserID != nil {
 				skString := skUserID.String()
 				sk = &skString
@@ -105,7 +101,7 @@ func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat,
 		unsigned := se.Unsigned()
 		var prev PrevEventRef
 		if err := json.Unmarshal(se.Unsigned(), &prev); err == nil && prev.PrevSenderID != "" {
-			prevUserID, err := userIDForSender(*validRoomID, spec.SenderID(prev.PrevSenderID))
+			prevUserID, err := userIDForSender(se.RoomID(), spec.SenderID(prev.PrevSenderID))
 			if err == nil && userID != nil {
 				prev.PrevSenderID = prevUserID.String()
 			} else {
@@ -142,10 +138,10 @@ func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, sender st
 
 	switch format {
 	case FormatAll:
-		ce.RoomID = se.RoomID()
+		ce.RoomID = se.RoomID().String()
 	case FormatSync:
 	case FormatSyncFederation:
-		ce.RoomID = se.RoomID()
+		ce.RoomID = se.RoomID().String()
 		ce.AuthEvents = se.AuthEventIDs()
 		ce.PrevEvents = se.PrevEventIDs()
 		ce.Depth = se.Depth()
@@ -164,18 +160,14 @@ func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, sender st
 // It provides default logic for event.SenderID & event.StateKey -> userID conversions.
 func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserverlib.PDU) ClientEvent {
 	sender := spec.UserID{}
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return ClientEvent{}
-	}
-	userID, err := userIDQuery(*validRoomID, event.SenderID())
+	userID, err := userIDQuery(event.RoomID(), event.SenderID())
 	if err == nil && userID != nil {
 		sender = *userID
 	}
 
 	sk := event.StateKey()
 	if sk != nil && *sk != "" {
-		skUserID, err := userIDQuery(*validRoomID, spec.SenderID(*event.StateKey()))
+		skUserID, err := userIDQuery(event.RoomID(), spec.SenderID(*event.StateKey()))
 		if err == nil && skUserID != nil {
 			skString := skUserID.String()
 			sk = &skString
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index 8863d258a..dca09193c 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -108,7 +108,7 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
 	}
 
 	if s.cfg.Matrix.ReportStats.Enabled {
-		go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID())
+		go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID().String())
 	}
 
 	log.WithFields(log.Fields{
@@ -294,7 +294,7 @@ func (s *OutputRoomEventConsumer) copyTags(ctx context.Context, oldRoomID, newRo
 }
 
 func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rstypes.HeaderedEvent, streamPos uint64) error {
-	members, roomSize, err := s.localRoomMembers(ctx, event.RoomID())
+	members, roomSize, err := s.localRoomMembers(ctx, event.RoomID().String())
 	if err != nil {
 		return fmt.Errorf("s.localRoomMembers: %w", err)
 	}
@@ -302,18 +302,14 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst
 	switch {
 	case event.Type() == spec.MRoomMember:
 		sender := spec.UserID{}
-		validRoomID, roomErr := spec.NewRoomID(event.RoomID())
-		if roomErr != nil {
-			return roomErr
-		}
-		userID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+		userID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 		if queryErr == nil && userID != nil {
 			sender = *userID
 		}
 
 		sk := event.StateKey()
 		if sk != nil && *sk != "" {
-			skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*sk))
+			skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*sk))
 			if queryErr == nil && skUserID != nil {
 				skString := skUserID.String()
 				sk = &skString
@@ -334,7 +330,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst
 		}
 	case event.Type() == "m.room.tombstone" && event.StateKeyEquals(""):
 		// Handle room upgrades
-		oldRoomID := event.RoomID()
+		oldRoomID := event.RoomID().String()
 		newRoomID := gjson.GetBytes(event.Content(), "replacement_room").Str
 		if err = s.handleRoomUpgrade(ctx, oldRoomID, newRoomID, members, roomSize); err != nil {
 			// while inconvenient, this shouldn't stop us from sending push notifications
@@ -351,7 +347,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst
 
 	log.WithFields(log.Fields{
 		"event_id":    event.EventID(),
-		"room_id":     event.RoomID(),
+		"room_id":     event.RoomID().String(),
 		"num_members": len(members),
 		"room_size":   roomSize,
 	}).Tracef("Notifying members")
@@ -464,7 +460,7 @@ func (s *OutputRoomEventConsumer) roomName(ctx context.Context, event *rstypes.H
 	}
 
 	req := &rsapi.QueryCurrentStateRequest{
-		RoomID:      event.RoomID(),
+		RoomID:      event.RoomID().String(),
 		StateTuples: []gomatrixserverlib.StateKeyTuple{roomNameTuple, canonicalAliasTuple},
 	}
 	var res rsapi.QueryCurrentStateResponse
@@ -532,7 +528,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 	if a != pushrules.NotifyAction && a != pushrules.CoalesceAction {
 		log.WithFields(log.Fields{
 			"event_id":  event.EventID(),
-			"room_id":   event.RoomID(),
+			"room_id":   event.RoomID().String(),
 			"localpart": mem.Localpart,
 		}).Tracef("Push rule evaluation rejected the event")
 		return nil
@@ -544,18 +540,14 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 	}
 
 	sender := spec.UserID{}
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return err
-	}
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+	userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 	if err == nil && userID != nil {
 		sender = *userID
 	}
 
 	sk := event.StateKey()
 	if sk != nil && *sk != "" {
-		skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*event.StateKey()))
+		skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*event.StateKey()))
 		if queryErr == nil && skUserID != nil {
 			skString := skUserID.String()
 			sk = &skString
@@ -572,14 +564,14 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 		// make sense. What is this supposed to be? Sytests require it
 		// to "work", but they only use a single device.
 		ProfileTag: profileTag,
-		RoomID:     event.RoomID(),
+		RoomID:     event.RoomID().String(),
 		TS:         spec.AsTimestamp(time.Now()),
 	}
 	if err = s.db.InsertNotification(ctx, mem.Localpart, mem.Domain, event.EventID(), streamPos, tweaks, n); err != nil {
 		return fmt.Errorf("s.db.InsertNotification: %w", err)
 	}
 
-	if err = s.syncProducer.GetAndSendNotificationData(ctx, mem.UserID, event.RoomID()); err != nil {
+	if err = s.syncProducer.GetAndSendNotificationData(ctx, mem.UserID, event.RoomID().String()); err != nil {
 		return fmt.Errorf("s.syncProducer.GetAndSendNotificationData: %w", err)
 	}
 
@@ -591,7 +583,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 
 	log.WithFields(log.Fields{
 		"event_id":   event.EventID(),
-		"room_id":    event.RoomID(),
+		"room_id":    event.RoomID().String(),
 		"localpart":  mem.Localpart,
 		"num_urls":   len(devicesByURLAndFormat),
 		"num_unread": userNumUnreadNotifs,
@@ -648,11 +640,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 // user. Returns actions (including dont_notify).
 func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event *rstypes.HeaderedEvent, mem *localMembership, roomSize int) ([]*pushrules.Action, error) {
 	user := ""
-	validRoomID, err := spec.NewRoomID(event.RoomID())
-	if err != nil {
-		return nil, err
-	}
-	sender, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+	sender, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 	if err == nil {
 		user = sender.String()
 	}
@@ -686,7 +674,7 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event *
 		ctx:      ctx,
 		rsAPI:    s.rsAPI,
 		mem:      mem,
-		roomID:   event.RoomID(),
+		roomID:   event.RoomID().String(),
 		roomSize: roomSize,
 	}
 	eval := pushrules.NewRuleSetEvaluator(ec, &ruleSets.Global)
@@ -704,7 +692,7 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event *
 
 	log.WithFields(log.Fields{
 		"event_id":  event.EventID(),
-		"room_id":   event.RoomID(),
+		"room_id":   event.RoomID().String(),
 		"localpart": mem.Localpart,
 		"rule_id":   rule.RuleID,
 	}).Trace("Matched a push rule")
@@ -793,16 +781,12 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes
 				},
 				Devices: devices,
 				EventID: event.EventID(),
-				RoomID:  event.RoomID(),
+				RoomID:  event.RoomID().String(),
 			},
 		}
 
 	default:
-		validRoomID, err := spec.NewRoomID(event.RoomID())
-		if err != nil {
-			return nil, err
-		}
-		sender, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID())
+		sender, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
 		if err != nil {
 			logger.WithError(err).Errorf("Failed to get userID for sender %s", event.SenderID())
 			return nil, err
@@ -816,7 +800,7 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes
 				Devices:  devices,
 				EventID:  event.EventID(),
 				ID:       event.EventID(),
-				RoomID:   event.RoomID(),
+				RoomID:   event.RoomID().String(),
 				RoomName: roomName,
 				Sender:   sender.String(),
 				Type:     event.Type(),
@@ -830,19 +814,13 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes
 			logger.WithError(err).Errorf("Failed to convert local user to userID %s", localpart)
 			return nil, err
 		}
-		roomID, err := spec.NewRoomID(event.RoomID())
+		localSender, err := s.rsAPI.QuerySenderIDForUser(ctx, event.RoomID(), *userID)
 		if err != nil {
-			logger.WithError(err).Errorf("event roomID is invalid %s", event.RoomID())
-			return nil, err
-		}
-
-		localSender, err := s.rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID)
-		if err != nil {
-			logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID())
+			logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID().String())
 			return nil, err
 		} else if localSender == nil {
-			logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID())
-			return nil, fmt.Errorf("no sender ID for user %s in %s", userID.String(), roomID.String())
+			logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID().String())
+			return nil, fmt.Errorf("no sender ID for user %s in %s", userID.String(), event.RoomID().String())
 		}
 		if event.StateKey() != nil && *event.StateKey() == string(*localSender) {
 			req.Notification.UserIsTarget = true

From db83789654ade3cf4f900e8fbcaa742b60c5dc6c Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Fri, 15 Sep 2023 15:25:09 +0000
Subject: [PATCH 24/57] Move pseudoID ClientEvent hotswapping to a common
 location (#3199)

Fixes a variety of issues where clients were receiving pseudoIDs in
places that should be userIDs.
This change makes pseudoIDs work with sliding sync & element x.

---------

Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>
---
 clientapi/routing/state.go            |  21 +-
 internal/eventutil/events.go          |   6 +-
 syncapi/routing/getevent.go           |  18 +-
 syncapi/routing/relations.go          |  21 +-
 syncapi/routing/search.go             |  20 +-
 syncapi/streams/stream_invite.go      |  15 +-
 syncapi/streams/stream_pdu.go         | 131 ---------
 syncapi/synctypes/clientevent.go      | 375 ++++++++++++++++++++------
 syncapi/synctypes/clientevent_test.go |  30 ++-
 syncapi/types/types.go                |  35 ++-
 syncapi/types/types_test.go           |  29 +-
 userapi/consumers/roomserver.go       |  43 +--
 userapi/util/notify_test.go           |  15 +-
 13 files changed, 426 insertions(+), 333 deletions(-)

diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index 6f363349b..18f9a0e9c 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -172,23 +172,16 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 			}
 		}
 		for _, ev := range stateAfterRes.StateEvents {
-			sender := spec.UserID{}
-			userID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID())
-			if err == nil && userID != nil {
-				sender = *userID
-			}
-
-			sk := ev.StateKey()
-			if sk != nil && *sk != "" {
-				skUserID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey()))
-				if err == nil && skUserID != nil {
-					skString := skUserID.String()
-					sk = &skString
-				}
+			clientEvent, err := synctypes.ToClientEvent(ev, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+			})
+			if err != nil {
+				util.GetLogger(ctx).WithError(err).Error("Failed converting to ClientEvent")
+				continue
 			}
 			stateEvents = append(
 				stateEvents,
-				synctypes.ToClientEvent(ev, synctypes.FormatAll, sender.String(), sk, ev.Unsigned()),
+				*clientEvent,
 			)
 		}
 	}
diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go
index b3523e129..40d62fd68 100644
--- a/internal/eventutil/events.go
+++ b/internal/eventutil/events.go
@@ -176,11 +176,13 @@ func RedactEvent(ctx context.Context, redactionEvent, redactedEvent gomatrixserv
 		return fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type())
 	}
 	redactedEvent.Redact()
-	senderID, err := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactionEvent.SenderID())
+	clientEvent, err := synctypes.ToClientEvent(redactionEvent, synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return querier.QueryUserIDForSender(ctx, roomID, senderID)
+	})
 	if err != nil {
 		return err
 	}
-	redactedBecause := synctypes.ToClientEvent(redactionEvent, synctypes.FormatSync, senderID.String(), redactionEvent.StateKey(), redactionEvent.Unsigned())
+	redactedBecause := clientEvent
 	if err := redactedEvent.SetUnsignedField("redacted_because", redactedBecause); err != nil {
 		return err
 	}
diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go
index 886b11675..c089539f0 100644
--- a/syncapi/routing/getevent.go
+++ b/syncapi/routing/getevent.go
@@ -118,25 +118,19 @@ func GetEvent(
 		}
 	}
 
-	senderUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *roomID, events[0].SenderID())
-	if err != nil || senderUserID == nil {
-		util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("QueryUserIDForSender errored or returned nil-user ID when user should be part of a room")
+	clientEvent, err := synctypes.ToClientEvent(events[0], synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+	})
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("Failed converting to ClientEvent")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: spec.Unknown("internal server error"),
 		}
 	}
 
-	sk := events[0].StateKey()
-	if sk != nil && *sk != "" {
-		skUserID, err := rsAPI.QueryUserIDForSender(ctx, events[0].RoomID(), spec.SenderID(*events[0].StateKey()))
-		if err == nil && skUserID != nil {
-			skString := skUserID.String()
-			sk = &skString
-		}
-	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: synctypes.ToClientEvent(events[0], synctypes.FormatAll, senderUserID.String(), sk, events[0].Unsigned()),
+		JSON: *clientEvent,
 	}
 }
diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go
index b451a7e2e..935ba83b3 100644
--- a/syncapi/routing/relations.go
+++ b/syncapi/routing/relations.go
@@ -130,23 +130,16 @@ func Relations(
 	// type if it was specified.
 	res.Chunk = make([]synctypes.ClientEvent, 0, len(filteredEvents))
 	for _, event := range filteredEvents {
-		sender := spec.UserID{}
-		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *roomID, event.SenderID())
-		if err == nil && userID != nil {
-			sender = *userID
-		}
-
-		sk := event.StateKey()
-		if sk != nil && *sk != "" {
-			skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *roomID, spec.SenderID(*event.StateKey()))
-			if err == nil && skUserID != nil {
-				skString := skUserID.String()
-				sk = &skString
-			}
+		clientEvent, err := synctypes.ToClientEvent(event.PDU, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID)
+		})
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("Failed converting to ClientEvent")
+			continue
 		}
 		res.Chunk = append(
 			res.Chunk,
-			synctypes.ToClientEvent(event.PDU, synctypes.FormatAll, sender.String(), sk, event.Unsigned()),
+			*clientEvent,
 		)
 	}
 
diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go
index f574781aa..4a8be9f49 100644
--- a/syncapi/routing/search.go
+++ b/syncapi/routing/search.go
@@ -230,20 +230,14 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 			profileInfos[userID.String()] = profile
 		}
 
-		sender := spec.UserID{}
-		userID, err := rsAPI.QueryUserIDForSender(req.Context(), event.RoomID(), event.SenderID())
-		if err == nil && userID != nil {
-			sender = *userID
+		clientEvent, err := synctypes.ToClientEvent(event, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+		})
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).WithField("senderID", event.SenderID()).Error("Failed converting to ClientEvent")
+			continue
 		}
 
-		sk := event.StateKey()
-		if sk != nil && *sk != "" {
-			skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), event.RoomID(), spec.SenderID(*event.StateKey()))
-			if err == nil && skUserID != nil {
-				skString := skUserID.String()
-				sk = &skString
-			}
-		}
 		results = append(results, Result{
 			Context: SearchContextResponse{
 				Start: startToken.String(),
@@ -257,7 +251,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 				ProfileInfo: profileInfos,
 			},
 			Rank:   eventScore[event.EventID()].Score,
-			Result: synctypes.ToClientEvent(event, synctypes.FormatAll, sender.String(), sk, event.Unsigned()),
+			Result: *clientEvent,
 		})
 		roomGroup := groups[event.RoomID().String()]
 		roomGroup.Results = append(roomGroup.Results, event.EventID())
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 1424dc2e6..a3634c03f 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -75,20 +75,15 @@ func (p *InviteStreamProvider) IncrementalSync(
 			user = *sender
 		}
 
-		sk := inviteEvent.StateKey()
-		if sk != nil && *sk != "" {
-			skUserID, err := p.rsAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), spec.SenderID(*inviteEvent.StateKey()))
-			if err == nil && skUserID != nil {
-				skString := skUserID.String()
-				sk = &skString
-			}
-		}
-
 		// skip ignored user events
 		if _, ok := req.IgnoredUsers.List[user.String()]; ok {
 			continue
 		}
-		ir := types.NewInviteResponse(inviteEvent, user, sk, eventFormat)
+		ir, err := types.NewInviteResponse(ctx, p.rsAPI, inviteEvent, eventFormat)
+		if err != nil {
+			req.Log.WithError(err).Error("failed creating invite response")
+			continue
+		}
 		req.Response.Rooms.Invite[roomID] = ir
 	}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index eb1f0ef2e..3abb0b3c6 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -3,7 +3,6 @@ package streams
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
 	"fmt"
 	"time"
 
@@ -16,8 +15,6 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -359,23 +356,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	// Now that we've filtered the timeline, work out which state events are still
 	// left. Anything that appears in the filtered timeline will be removed from the
 	// "state" section and kept in "timeline".
-
-	// update the powerlevel event for timeline events
-	for i, ev := range events {
-		if ev.Version() != gomatrixserverlib.RoomVersionPseudoIDs {
-			continue
-		}
-		if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") {
-			continue
-		}
-		var newEvent gomatrixserverlib.PDU
-		newEvent, err = p.updatePowerLevelEvent(ctx, ev, eventFormat)
-		if err != nil {
-			return r.From, err
-		}
-		events[i] = &rstypes.HeaderedEvent{PDU: newEvent}
-	}
-
 	sEvents := gomatrixserverlib.HeaderedReverseTopologicalOrdering(
 		gomatrixserverlib.ToPDUs(removeDuplicates(delta.StateEvents, events)),
 		gomatrixserverlib.TopologicalOrderByAuthEvents,
@@ -390,15 +370,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 			continue
 		}
 		delta.StateEvents[i-skipped] = he
-		// update the powerlevel event for state events
-		if ev.Version() == gomatrixserverlib.RoomVersionPseudoIDs && ev.Type() == spec.MRoomPowerLevels && ev.StateKeyEquals("") {
-			var newEvent gomatrixserverlib.PDU
-			newEvent, err = p.updatePowerLevelEvent(ctx, he, eventFormat)
-			if err != nil {
-				return r.From, err
-			}
-			delta.StateEvents[i-skipped] = &rstypes.HeaderedEvent{PDU: newEvent}
-		}
 	}
 	delta.StateEvents = delta.StateEvents[:len(sEvents)-skipped]
 
@@ -468,79 +439,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	return latestPosition, nil
 }
 
-func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstypes.HeaderedEvent, eventFormat synctypes.ClientEventFormat) (gomatrixserverlib.PDU, error) {
-	pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(ev)
-	if err != nil {
-		return nil, err
-	}
-	newPls := make(map[string]int64)
-	var userID *spec.UserID
-	for user, level := range pls.Users {
-		if eventFormat != synctypes.FormatSyncFederation {
-			userID, err = p.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(user))
-			if err != nil {
-				return nil, err
-			}
-			user = userID.String()
-		}
-		newPls[user] = level
-	}
-	var newPlBytes, newEv []byte
-	newPlBytes, err = json.Marshal(newPls)
-	if err != nil {
-		return nil, err
-	}
-	newEv, err = sjson.SetRawBytes(ev.JSON(), "content.users", newPlBytes)
-	if err != nil {
-		return nil, err
-	}
-
-	// do the same for prev content
-	prevContent := gjson.GetBytes(ev.JSON(), "unsigned.prev_content")
-	if !prevContent.Exists() {
-		var evNew gomatrixserverlib.PDU
-		evNew, err = gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSONWithEventID(ev.EventID(), newEv, false)
-		if err != nil {
-			return nil, err
-		}
-
-		return evNew, err
-	}
-	pls = gomatrixserverlib.PowerLevelContent{}
-	err = json.Unmarshal([]byte(prevContent.Raw), &pls)
-	if err != nil {
-		return nil, err
-	}
-
-	newPls = make(map[string]int64)
-	for user, level := range pls.Users {
-		if eventFormat != synctypes.FormatSyncFederation {
-			userID, err = p.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(user))
-			if err != nil {
-				return nil, err
-			}
-			user = userID.String()
-		}
-		newPls[user] = level
-	}
-	newPlBytes, err = json.Marshal(newPls)
-	if err != nil {
-		return nil, err
-	}
-	newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes)
-	if err != nil {
-		return nil, err
-	}
-
-	var evNew gomatrixserverlib.PDU
-	evNew, err = gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSONWithEventID(ev.EventID(), newEv, false)
-	if err != nil {
-		return nil, err
-	}
-
-	return evNew, err
-}
-
 // applyHistoryVisibilityFilter gets the current room state and supplies it to ApplyHistoryVisibilityFilter, to make
 // sure we always return the required events in the timeline.
 func applyHistoryVisibilityFilter(
@@ -690,35 +588,6 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		prevBatch.Decrement()
 	}
 
-	// Update powerlevel events for timeline events
-	for i, ev := range events {
-		if ev.Version() != gomatrixserverlib.RoomVersionPseudoIDs {
-			continue
-		}
-		if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") {
-			continue
-		}
-		newEvent, err := p.updatePowerLevelEvent(ctx, ev, eventFormat)
-		if err != nil {
-			return nil, err
-		}
-		events[i] = &rstypes.HeaderedEvent{PDU: newEvent}
-	}
-	// Update powerlevel events for state events
-	for i, ev := range stateEvents {
-		if ev.Version() != gomatrixserverlib.RoomVersionPseudoIDs {
-			continue
-		}
-		if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") {
-			continue
-		}
-		newEvent, err := p.updatePowerLevelEvent(ctx, ev, eventFormat)
-		if err != nil {
-			return nil, err
-		}
-		stateEvents[i] = &rstypes.HeaderedEvent{PDU: newEvent}
-	}
-
 	jr.Timeline.PrevBatch = prevBatch
 	jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 		return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go
index e0616e11d..6812f8332 100644
--- a/syncapi/synctypes/clientevent.go
+++ b/syncapi/synctypes/clientevent.go
@@ -22,6 +22,8 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
 )
 
 // PrevEventRef represents a reference to a previous event in a state event upgrade
@@ -78,59 +80,62 @@ func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat,
 		if se == nil {
 			continue // TODO: shouldn't happen?
 		}
-		if format == FormatSyncFederation {
-			evs = append(evs, ToClientEvent(se, format, string(se.SenderID()), se.StateKey(), spec.RawJSON(se.Unsigned())))
+		ev, err := ToClientEvent(se, format, userIDForSender)
+		if err != nil {
+			logrus.WithError(err).Warn("Failed converting event to ClientEvent")
 			continue
 		}
-
-		sender := spec.UserID{}
-		userID, err := userIDForSender(se.RoomID(), se.SenderID())
-		if err == nil && userID != nil {
-			sender = *userID
-		}
-
-		sk := se.StateKey()
-		if sk != nil && *sk != "" {
-			skUserID, err := userIDForSender(se.RoomID(), spec.SenderID(*sk))
-			if err == nil && skUserID != nil {
-				skString := skUserID.String()
-				sk = &skString
-			}
-		}
-
-		unsigned := se.Unsigned()
-		var prev PrevEventRef
-		if err := json.Unmarshal(se.Unsigned(), &prev); err == nil && prev.PrevSenderID != "" {
-			prevUserID, err := userIDForSender(se.RoomID(), spec.SenderID(prev.PrevSenderID))
-			if err == nil && userID != nil {
-				prev.PrevSenderID = prevUserID.String()
-			} else {
-				errString := "userID unknown"
-				if err != nil {
-					errString = err.Error()
-				}
-				logrus.Warnf("Failed to find userID for prev_sender in ClientEvent: %s", errString)
-				// NOTE: Not much can be done here, so leave the previous value in place.
-			}
-			unsigned, err = json.Marshal(prev)
-			if err != nil {
-				logrus.Errorf("Failed to marshal unsigned content for ClientEvent: %s", err.Error())
-				continue
-			}
-		}
-		evs = append(evs, ToClientEvent(se, format, sender.String(), sk, spec.RawJSON(unsigned)))
+		evs = append(evs, *ev)
 	}
 	return evs
 }
 
+// ToClientEventDefault converts a single server event to a client event.
+// It provides default logic for event.SenderID & event.StateKey -> userID conversions.
+func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserverlib.PDU) ClientEvent {
+	ev, err := ToClientEvent(event, FormatAll, userIDQuery)
+	if err != nil {
+		return ClientEvent{}
+	}
+	return *ev
+}
+
+// If provided state key is a user ID (state keys beginning with @ are reserved for this purpose)
+// fetch it's associated sender ID and use that instead. Otherwise returns the same state key back.
+//
+// # This function either returns the state key that should be used, or an error
+//
+// TODO: handle failure cases better (e.g. no sender ID)
+func FromClientStateKey(roomID spec.RoomID, stateKey string, senderIDQuery spec.SenderIDForUser) (*string, error) {
+	if len(stateKey) >= 1 && stateKey[0] == '@' {
+		parsedStateKey, err := spec.NewUserID(stateKey, true)
+		if err != nil {
+			// If invalid user ID, then there is no associated state event.
+			return nil, fmt.Errorf("Provided state key begins with @ but is not a valid user ID: %w", err)
+		}
+		senderID, err := senderIDQuery(roomID, *parsedStateKey)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to query sender ID: %w", err)
+		}
+		if senderID == nil {
+			// If no sender ID, then there is no associated state event.
+			return nil, fmt.Errorf("No associated sender ID found.")
+		}
+		newStateKey := string(*senderID)
+		return &newStateKey, nil
+	} else {
+		return &stateKey, nil
+	}
+}
+
 // ToClientEvent converts a single server event to a client event.
-func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, sender string, stateKey *string, unsigned spec.RawJSON) ClientEvent {
+func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, userIDForSender spec.UserIDForSender) (*ClientEvent, error) {
 	ce := ClientEvent{
-		Content:        spec.RawJSON(se.Content()),
-		Sender:         sender,
+		Content:        se.Content(),
+		Sender:         string(se.SenderID()),
 		Type:           se.Type(),
-		StateKey:       stateKey,
-		Unsigned:       unsigned,
+		StateKey:       se.StateKey(),
+		Unsigned:       se.Unsigned(),
 		OriginServerTS: se.OriginServerTS(),
 		EventID:        se.EventID(),
 		Redacts:        se.Redacts(),
@@ -148,58 +153,268 @@ func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, sender st
 		// TODO: Set Signatures & Hashes fields
 	}
 
-	if format != FormatSyncFederation {
-		if se.Version() == gomatrixserverlib.RoomVersionPseudoIDs {
-			ce.SenderKey = se.SenderID()
+	if format != FormatSyncFederation && se.Version() == gomatrixserverlib.RoomVersionPseudoIDs {
+		err := updatePseudoIDs(&ce, se, userIDForSender, format)
+		if err != nil {
+			return nil, err
 		}
 	}
-	return ce
+
+	return &ce, nil
 }
 
-// ToClientEvent converts a single server event to a client event.
-// It provides default logic for event.SenderID & event.StateKey -> userID conversions.
-func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserverlib.PDU) ClientEvent {
-	sender := spec.UserID{}
-	userID, err := userIDQuery(event.RoomID(), event.SenderID())
+func updatePseudoIDs(ce *ClientEvent, se gomatrixserverlib.PDU, userIDForSender spec.UserIDForSender, format ClientEventFormat) error {
+	ce.SenderKey = se.SenderID()
+
+	userID, err := userIDForSender(se.RoomID(), se.SenderID())
 	if err == nil && userID != nil {
-		sender = *userID
+		ce.Sender = userID.String()
 	}
 
-	sk := event.StateKey()
+	sk := se.StateKey()
 	if sk != nil && *sk != "" {
-		skUserID, err := userIDQuery(event.RoomID(), spec.SenderID(*event.StateKey()))
+		skUserID, err := userIDForSender(se.RoomID(), spec.SenderID(*sk))
 		if err == nil && skUserID != nil {
 			skString := skUserID.String()
-			sk = &skString
+			ce.StateKey = &skString
 		}
 	}
-	return ToClientEvent(event, FormatAll, sender.String(), sk, event.Unsigned())
+
+	var prev PrevEventRef
+	if err := json.Unmarshal(se.Unsigned(), &prev); err == nil && prev.PrevSenderID != "" {
+		prevUserID, err := userIDForSender(se.RoomID(), spec.SenderID(prev.PrevSenderID))
+		if err == nil && userID != nil {
+			prev.PrevSenderID = prevUserID.String()
+		} else {
+			errString := "userID unknown"
+			if err != nil {
+				errString = err.Error()
+			}
+			logrus.Warnf("Failed to find userID for prev_sender in ClientEvent: %s", errString)
+			// NOTE: Not much can be done here, so leave the previous value in place.
+		}
+		ce.Unsigned, err = json.Marshal(prev)
+		if err != nil {
+			err = fmt.Errorf("Failed to marshal unsigned content for ClientEvent: %w", err)
+			return err
+		}
+	}
+
+	switch se.Type() {
+	case spec.MRoomCreate:
+		updatedContent, err := updateCreateEvent(se.Content(), userIDForSender, se.RoomID())
+		if err != nil {
+			err = fmt.Errorf("Failed to update m.room.create event for ClientEvent: %w", err)
+			return err
+		}
+		ce.Content = updatedContent
+	case spec.MRoomMember:
+		updatedEvent, err := updateInviteEvent(userIDForSender, se, format)
+		if err != nil {
+			err = fmt.Errorf("Failed to update m.room.member event for ClientEvent: %w", err)
+			return err
+		}
+		if updatedEvent != nil {
+			ce.Unsigned = updatedEvent.Unsigned()
+		}
+	case spec.MRoomPowerLevels:
+		updatedEvent, err := updatePowerLevelEvent(userIDForSender, se, format)
+		if err != nil {
+			err = fmt.Errorf("Failed update m.room.power_levels event for ClientEvent: %w", err)
+			return err
+		}
+		if updatedEvent != nil {
+			ce.Content = updatedEvent.Content()
+			ce.Unsigned = updatedEvent.Unsigned()
+		}
+	}
+
+	return nil
 }
 
-// If provided state key is a user ID (state keys beginning with @ are reserved for this purpose)
-// fetch it's associated sender ID and use that instead. Otherwise returns the same state key back.
-//
-// # This function either returns the state key that should be used, or an error
-//
-// TODO: handle failure cases better (e.g. no sender ID)
-func FromClientStateKey(roomID spec.RoomID, stateKey string, senderIDQuery spec.SenderIDForUser) (*string, error) {
-	if len(stateKey) >= 1 && stateKey[0] == '@' {
-		parsedStateKey, err := spec.NewUserID(stateKey, true)
+func updateCreateEvent(content spec.RawJSON, userIDForSender spec.UserIDForSender, roomID spec.RoomID) (spec.RawJSON, error) {
+	if creator := gjson.GetBytes(content, "creator"); creator.Exists() {
+		oldCreator := creator.Str
+		userID, err := userIDForSender(roomID, spec.SenderID(oldCreator))
 		if err != nil {
-			// If invalid user ID, then there is no associated state event.
-			return nil, fmt.Errorf("Provided state key begins with @ but is not a valid user ID: %s", err.Error())
+			err = fmt.Errorf("Failed to find userID for creator in ClientEvent: %w", err)
+			return nil, err
 		}
-		senderID, err := senderIDQuery(roomID, *parsedStateKey)
-		if err != nil {
-			return nil, fmt.Errorf("Failed to query sender ID: %s", err.Error())
+
+		if userID != nil {
+			var newCreatorBytes, newContent []byte
+			newCreatorBytes, err = json.Marshal(userID.String())
+			if err != nil {
+				err = fmt.Errorf("Failed to marshal new creator for ClientEvent: %w", err)
+				return nil, err
+			}
+
+			newContent, err = sjson.SetRawBytes([]byte(content), "creator", newCreatorBytes)
+			if err != nil {
+				err = fmt.Errorf("Failed to set new creator for ClientEvent: %w", err)
+				return nil, err
+			}
+
+			return newContent, nil
 		}
-		if senderID == nil {
-			// If no sender ID, then there is no associated state event.
-			return nil, fmt.Errorf("No associated sender ID found.")
-		}
-		newStateKey := string(*senderID)
-		return &newStateKey, nil
-	} else {
-		return &stateKey, nil
 	}
+
+	return content, nil
+}
+
+func updateInviteEvent(userIDForSender spec.UserIDForSender, ev gomatrixserverlib.PDU, eventFormat ClientEventFormat) (gomatrixserverlib.PDU, error) {
+	if inviteRoomState := gjson.GetBytes(ev.Unsigned(), "invite_room_state"); inviteRoomState.Exists() {
+		userID, err := userIDForSender(ev.RoomID(), ev.SenderID())
+		if err != nil || userID == nil {
+			if err != nil {
+				err = fmt.Errorf("invalid userID found when updating invite_room_state: %w", err)
+			}
+			return nil, err
+		}
+
+		newState, err := GetUpdatedInviteRoomState(userIDForSender, inviteRoomState, ev, ev.RoomID(), eventFormat)
+		if err != nil {
+			return nil, err
+		}
+
+		var newEv []byte
+		newEv, err = sjson.SetRawBytes(ev.JSON(), "unsigned.invite_room_state", newState)
+		if err != nil {
+			return nil, err
+		}
+
+		return gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSON(newEv, false)
+	}
+
+	return ev, nil
+}
+
+type InviteRoomStateEvent struct {
+	Content  spec.RawJSON `json:"content"`
+	SenderID string       `json:"sender"`
+	StateKey *string      `json:"state_key"`
+	Type     string       `json:"type"`
+}
+
+func GetUpdatedInviteRoomState(userIDForSender spec.UserIDForSender, inviteRoomState gjson.Result, event gomatrixserverlib.PDU, roomID spec.RoomID, eventFormat ClientEventFormat) (spec.RawJSON, error) {
+	var res spec.RawJSON
+	inviteStateEvents := []InviteRoomStateEvent{}
+	err := json.Unmarshal([]byte(inviteRoomState.Raw), &inviteStateEvents)
+	if err != nil {
+		return nil, err
+	}
+
+	if event.Version() == gomatrixserverlib.RoomVersionPseudoIDs && eventFormat != FormatSyncFederation {
+		for i, ev := range inviteStateEvents {
+			userID, userIDErr := userIDForSender(roomID, spec.SenderID(ev.SenderID))
+			if userIDErr != nil {
+				return nil, userIDErr
+			}
+			if userID != nil {
+				inviteStateEvents[i].SenderID = userID.String()
+			}
+
+			if ev.StateKey != nil && *ev.StateKey != "" {
+				userID, senderErr := userIDForSender(roomID, spec.SenderID(*ev.StateKey))
+				if senderErr != nil {
+					return nil, senderErr
+				}
+				if userID != nil {
+					user := userID.String()
+					inviteStateEvents[i].StateKey = &user
+				}
+			}
+
+			updatedContent, updateErr := updateCreateEvent(ev.Content, userIDForSender, roomID)
+			if updateErr != nil {
+				updateErr = fmt.Errorf("Failed to update m.room.create event for ClientEvent: %w", userIDErr)
+				return nil, updateErr
+			}
+			inviteStateEvents[i].Content = updatedContent
+		}
+	}
+
+	res, err = json.Marshal(inviteStateEvents)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func updatePowerLevelEvent(userIDForSender spec.UserIDForSender, se gomatrixserverlib.PDU, eventFormat ClientEventFormat) (gomatrixserverlib.PDU, error) {
+	if !se.StateKeyEquals("") {
+		return se, nil
+	}
+
+	pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se)
+	if err != nil {
+		return nil, err
+	}
+	newPls := make(map[string]int64)
+	var userID *spec.UserID
+	for user, level := range pls.Users {
+		if eventFormat != FormatSyncFederation {
+			userID, err = userIDForSender(se.RoomID(), spec.SenderID(user))
+			if err != nil {
+				return nil, err
+			}
+			user = userID.String()
+		}
+		newPls[user] = level
+	}
+	var newPlBytes, newEv []byte
+	newPlBytes, err = json.Marshal(newPls)
+	if err != nil {
+		return nil, err
+	}
+	newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	// do the same for prev content
+	prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content")
+	if !prevContent.Exists() {
+		var evNew gomatrixserverlib.PDU
+		evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false)
+		if err != nil {
+			return nil, err
+		}
+
+		return evNew, err
+	}
+	pls = gomatrixserverlib.PowerLevelContent{}
+	err = json.Unmarshal([]byte(prevContent.Raw), &pls)
+	if err != nil {
+		return nil, err
+	}
+
+	newPls = make(map[string]int64)
+	for user, level := range pls.Users {
+		if eventFormat != FormatSyncFederation {
+			userID, err = userIDForSender(se.RoomID(), spec.SenderID(user))
+			if err != nil {
+				return nil, err
+			}
+			user = userID.String()
+		}
+		newPls[user] = level
+	}
+	newPlBytes, err = json.Marshal(newPls)
+	if err != nil {
+		return nil, err
+	}
+	newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	var evNew gomatrixserverlib.PDU
+	evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return evNew, err
 }
diff --git a/syncapi/synctypes/clientevent_test.go b/syncapi/synctypes/clientevent_test.go
index 202c185f1..662f9ea43 100644
--- a/syncapi/synctypes/clientevent_test.go
+++ b/syncapi/synctypes/clientevent_test.go
@@ -26,6 +26,14 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
+func queryUserIDForSender(senderID spec.SenderID) (*spec.UserID, error) {
+	if senderID == "" {
+		return nil, nil
+	}
+
+	return spec.NewUserID(string(senderID), true)
+}
+
 const testSenderID = "testSenderID"
 const testUserID = "@test:localhost"
 
@@ -106,7 +114,12 @@ func TestToClientEvent(t *testing.T) { // nolint: gocyclo
 		t.Fatalf("failed to create userID: %s", err)
 	}
 	sk := ""
-	ce := ToClientEvent(ev, FormatAll, userID.String(), &sk, ev.Unsigned())
+	ce, err := ToClientEvent(ev, FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return queryUserIDForSender(senderID)
+	})
+	if err != nil {
+		t.Fatalf("failed to create ClientEvent: %s", err)
+	}
 
 	verifyEventFields(t,
 		EventFieldsToVerify{
@@ -161,12 +174,12 @@ func TestToClientFormatSync(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to create Event: %s", err)
 	}
-	userID, err := spec.NewUserID("@test:localhost", true)
+	ce, err := ToClientEvent(ev, FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return queryUserIDForSender(senderID)
+	})
 	if err != nil {
-		t.Fatalf("failed to create userID: %s", err)
+		t.Fatalf("failed to create ClientEvent: %s", err)
 	}
-	sk := ""
-	ce := ToClientEvent(ev, FormatSync, userID.String(), &sk, ev.Unsigned())
 	if ce.RoomID != "" {
 		t.Errorf("ClientEvent.RoomID: wanted '', got %s", ce.RoomID)
 	}
@@ -206,7 +219,12 @@ func TestToClientEventFormatSyncFederation(t *testing.T) { // nolint: gocyclo
 		t.Fatalf("failed to create userID: %s", err)
 	}
 	sk := ""
-	ce := ToClientEvent(ev, FormatSyncFederation, userID.String(), &sk, ev.Unsigned())
+	ce, err := ToClientEvent(ev, FormatSyncFederation, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return queryUserIDForSender(senderID)
+	})
+	if err != nil {
+		t.Fatalf("failed to create ClientEvent: %s", err)
+	}
 
 	verifyEventFields(t,
 		EventFieldsToVerify{
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index b90c128c3..bca11855c 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -15,6 +15,7 @@
 package types
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -532,7 +533,7 @@ type InviteResponse struct {
 }
 
 // NewInviteResponse creates an empty response with initialised arrays.
-func NewInviteResponse(event *types.HeaderedEvent, userID spec.UserID, stateKey *string, eventFormat synctypes.ClientEventFormat) *InviteResponse {
+func NewInviteResponse(ctx context.Context, rsAPI api.QuerySenderIDAPI, event *types.HeaderedEvent, eventFormat synctypes.ClientEventFormat) (*InviteResponse, error) {
 	res := InviteResponse{}
 	res.InviteState.Events = []json.RawMessage{}
 
@@ -540,18 +541,42 @@ func NewInviteResponse(event *types.HeaderedEvent, userID spec.UserID, stateKey
 	// If there is then unmarshal it into the response. This will contain the
 	// partial room state such as join rules, room name etc.
 	if inviteRoomState := gjson.GetBytes(event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() {
-		_ = json.Unmarshal([]byte(inviteRoomState.Raw), &res.InviteState.Events)
+		if event.Version() == gomatrixserverlib.RoomVersionPseudoIDs && eventFormat != synctypes.FormatSyncFederation {
+			updatedInvite, err := synctypes.GetUpdatedInviteRoomState(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+			}, inviteRoomState, event.PDU, event.RoomID(), eventFormat)
+			if err != nil {
+				return nil, err
+			}
+			_ = json.Unmarshal(updatedInvite, &res.InviteState.Events)
+		} else {
+			_ = json.Unmarshal([]byte(inviteRoomState.Raw), &res.InviteState.Events)
+		}
+	}
+
+	// Clear unsigned so it doesn't have pseudoIDs converted during ToClientEvent
+	eventNoUnsigned, err := event.SetUnsigned(nil)
+	if err != nil {
+		return nil, err
 	}
 
 	// Then we'll see if we can create a partial of the invite event itself.
 	// This is needed for clients to work out *who* sent the invite.
-	inviteEvent := synctypes.ToClientEvent(event.PDU, eventFormat, userID.String(), stateKey, event.Unsigned())
+	inviteEvent, err := synctypes.ToClientEvent(eventNoUnsigned, eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Ensure unsigned field is empty so it isn't marshalled into the final JSON
 	inviteEvent.Unsigned = nil
-	if ev, err := json.Marshal(inviteEvent); err == nil {
+
+	if ev, err := json.Marshal(*inviteEvent); err == nil {
 		res.InviteState.Events = append(res.InviteState.Events, ev)
 	}
 
-	return &res
+	return &res, nil
 }
 
 // LeaveResponse represents a /sync response for a room which is under the 'leave' key.
diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go
index a79b9fc5d..35e1882cb 100644
--- a/syncapi/types/types_test.go
+++ b/syncapi/types/types_test.go
@@ -1,6 +1,7 @@
 package types
 
 import (
+	"context"
 	"encoding/json"
 	"reflect"
 	"testing"
@@ -11,8 +12,19 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/spec"
 )
 
-func UserIDForSender(roomID string, senderID string) (*spec.UserID, error) {
-	return spec.NewUserID(senderID, true)
+type FakeRoomserverAPI struct{}
+
+func (f *FakeRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	if senderID == "" {
+		return nil, nil
+	}
+
+	return spec.NewUserID(string(senderID), true)
+}
+
+func (f *FakeRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	sender := spec.SenderID(userID.String())
+	return &sender, nil
 }
 
 func TestSyncTokens(t *testing.T) {
@@ -61,25 +73,18 @@ func TestNewInviteResponse(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sender, err := spec.NewUserID("@neilalexander:matrix.org", true)
+	rsAPI := FakeRoomserverAPI{}
+	res, err := NewInviteResponse(context.Background(), &rsAPI, &types.HeaderedEvent{PDU: ev}, synctypes.FormatSync)
 	if err != nil {
 		t.Fatal(err)
 	}
-	skUserID, err := spec.NewUserID("@neilalexander:dendrite.neilalexander.dev", true)
-	if err != nil {
-		t.Fatal(err)
-	}
-	skString := skUserID.String()
-	sk := &skString
-
-	res := NewInviteResponse(&types.HeaderedEvent{PDU: ev}, *sender, sk, synctypes.FormatSync)
 	j, err := json.Marshal(res)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	if string(j) != expected {
-		t.Fatalf("Invite response didn't contain correct info")
+		t.Fatalf("Invite response didn't contain correct info, \nexpected: %s \ngot: %s", expected, string(j))
 	}
 }
 
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index dca09193c..047fe9216 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -301,25 +301,14 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst
 
 	switch {
 	case event.Type() == spec.MRoomMember:
-		sender := spec.UserID{}
-		userID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
-		if queryErr == nil && userID != nil {
-			sender = *userID
+		cevent, clientEvErr := synctypes.ToClientEvent(event, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+		})
+		if clientEvErr != nil {
+			return clientEvErr
 		}
-
-		sk := event.StateKey()
-		if sk != nil && *sk != "" {
-			skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*sk))
-			if queryErr == nil && skUserID != nil {
-				skString := skUserID.String()
-				sk = &skString
-			} else {
-				return fmt.Errorf("queryUserIDForSender: userID unknown for %s", *sk)
-			}
-		}
-		cevent := synctypes.ToClientEvent(event, synctypes.FormatAll, sender.String(), sk, event.Unsigned())
 		var member *localMembership
-		member, err = newLocalMembership(&cevent)
+		member, err = newLocalMembership(cevent)
 		if err != nil {
 			return fmt.Errorf("newLocalMembership: %w", err)
 		}
@@ -538,27 +527,19 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 	if err != nil {
 		return fmt.Errorf("s.localPushDevices: %w", err)
 	}
-
-	sender := spec.UserID{}
-	userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID())
-	if err == nil && userID != nil {
-		sender = *userID
+	clientEvent, err := synctypes.ToClientEvent(event, synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+		return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
+	})
+	if err != nil {
+		return err
 	}
 
-	sk := event.StateKey()
-	if sk != nil && *sk != "" {
-		skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*event.StateKey()))
-		if queryErr == nil && skUserID != nil {
-			skString := skUserID.String()
-			sk = &skString
-		}
-	}
 	n := &api.Notification{
 		Actions: actions,
 		// UNSPEC: the spec doesn't say this is a ClientEvent, but the
 		// fields seem to match. room_id should be missing, which
 		// matches the behaviour of FormatSync.
-		Event: synctypes.ToClientEvent(event, synctypes.FormatSync, sender.String(), sk, event.Unsigned()),
+		Event: *clientEvent,
 		// TODO: this is per-device, but it's not part of the primary
 		// key. So inserting one notification per profile tag doesn't
 		// make sense. What is this supposed to be? Sytests require it
diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go
index 27e77cf7a..2ea978d69 100644
--- a/userapi/util/notify_test.go
+++ b/userapi/util/notify_test.go
@@ -23,6 +23,14 @@ import (
 	userUtil "github.com/matrix-org/dendrite/userapi/util"
 )
 
+func queryUserIDForSender(senderID spec.SenderID) (*spec.UserID, error) {
+	if senderID == "" {
+		return nil, nil
+	}
+
+	return spec.NewUserID(string(senderID), true)
+}
+
 func TestNotifyUserCountsAsync(t *testing.T) {
 	alice := test.NewUser(t)
 	aliceLocalpart, serverName, err := gomatrixserverlib.SplitID('@', alice.ID)
@@ -100,13 +108,14 @@ func TestNotifyUserCountsAsync(t *testing.T) {
 		}
 
 		// Insert a dummy event
-		sender, err := spec.NewUserID(alice.ID, true)
+		ev, err := synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+			return queryUserIDForSender(senderID)
+		})
 		if err != nil {
 			t.Error(err)
 		}
-		sk := ""
 		if err := db.InsertNotification(ctx, aliceLocalpart, serverName, dummyEvent.EventID(), 0, nil, &api.Notification{
-			Event: synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll, sender.String(), &sk, dummyEvent.Unsigned()),
+			Event: *ev,
 		}); err != nil {
 			t.Error(err)
 		}

From d065219de1c2860c9cf28e8d8169fabfb71dcc31 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 26 Sep 2023 15:47:37 +0200
Subject: [PATCH 25/57] Fix invitations not sending push notifications (#3207)

The tests added in https://github.com/matrix-org/sytest/pull/1356
uncovered that we don't consider invitations as events the userapi
should handle and thus just don't notify the client about any new
invitations received over federation.
---
 sytest-whitelist                |  6 +++-
 userapi/consumers/roomserver.go | 60 +++++++++++++++++++++++----------
 2 files changed, 48 insertions(+), 18 deletions(-)

diff --git a/sytest-whitelist b/sytest-whitelist
index c61e0bc3c..60ba02302 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -783,4 +783,8 @@ Invited user can reject invite for empty room
 Invited user can reject local invite after originator leaves
 Guest users can join guest_access rooms
 Forgotten room messages cannot be paginated
-Local device key changes get to remote servers with correct prev_id
\ No newline at end of file
+Local device key changes get to remote servers with correct prev_id
+HS provides query metadata
+HS can provide query metadata on a single protocol
+Invites over federation are correctly pushed
+Invites over federation are correctly pushed with name
\ No newline at end of file
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index 047fe9216..d5baa074c 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -92,18 +92,36 @@ func (s *OutputRoomEventConsumer) Start() error {
 func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// Only handle events we care about
-	if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent {
-		return true
-	}
-	var output rsapi.OutputEvent
-	if err := json.Unmarshal(msg.Data, &output); err != nil {
-		// If the message was invalid, log it and move on to the next message in the stream
-		log.WithError(err).Errorf("roomserver output log: message parse failure")
-		return true
-	}
-	event := output.NewRoomEvent.Event
-	if event == nil {
-		log.Errorf("userapi consumer: expected event")
+
+	var event *rstypes.HeaderedEvent
+	var isNewRoomEvent bool
+	switch rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) {
+	case rsapi.OutputTypeNewRoomEvent:
+		isNewRoomEvent = true
+		fallthrough
+	case rsapi.OutputTypeNewInviteEvent:
+		var output rsapi.OutputEvent
+		if err := json.Unmarshal(msg.Data, &output); err != nil {
+			// If the message was invalid, log it and move on to the next message in the stream
+			log.WithError(err).Errorf("roomserver output log: message parse failure")
+			return true
+		}
+		if isNewRoomEvent {
+			event = output.NewRoomEvent.Event
+		} else {
+			event = output.NewInviteEvent.Event
+		}
+
+		if event == nil {
+			log.Errorf("userapi consumer: expected event")
+			return true
+		}
+
+		log.WithFields(log.Fields{
+			"event_id":   event.EventID(),
+			"event_type": event.Type(),
+		}).Tracef("Received message from roomserver: %#v", output)
+	default:
 		return true
 	}
 
@@ -111,11 +129,6 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
 		go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID().String())
 	}
 
-	log.WithFields(log.Fields{
-		"event_id":   event.EventID(),
-		"event_type": event.Type(),
-	}).Tracef("Received message from roomserver: %#v", output)
-
 	metadata, err := msg.Metadata()
 	if err != nil {
 		return true
@@ -448,6 +461,19 @@ func (s *OutputRoomEventConsumer) roomName(ctx context.Context, event *rstypes.H
 		}
 	}
 
+	// Special case for invites, as we don't store any "current state" for these events,
+	// we need to make sure that, if present, the m.room.name is sent as well.
+	if event.Type() == spec.MRoomMember &&
+		gjson.GetBytes(event.Content(), "membership").Str == "invite" {
+		invState := gjson.GetBytes(event.JSON(), "unsigned.invite_room_state")
+		for _, ev := range invState.Array() {
+			if ev.Get("type").Str == spec.MRoomName {
+				name := ev.Get("content.name").Str
+				return name, nil
+			}
+		}
+	}
+
 	req := &rsapi.QueryCurrentStateRequest{
 		RoomID:      event.RoomID().String(),
 		StateTuples: []gomatrixserverlib.StateKeyTuple{roomNameTuple, canonicalAliasTuple},

From 16d922de701ca28966127b2e1160d65527f65e8e Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Tue, 26 Sep 2023 17:44:49 +0000
Subject: [PATCH 26/57] Complement fixes for pseudoIDs (#3206)

---
 clientapi/routing/sendevent.go               |   9 +-
 go.mod                                       |   4 +-
 go.sum                                       |   8 +-
 roomserver/internal/perform/perform_leave.go |  22 +++-
 syncapi/synctypes/clientevent.go             | 114 ++++++++++---------
 5 files changed, 91 insertions(+), 66 deletions(-)

diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index fc9c05ba9..69131966b 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -263,7 +263,11 @@ func SendEvent(
 }
 
 func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error {
-	userMap := r["users"].(map[string]interface{})
+	users, ok := r["users"]
+	if !ok {
+		return nil
+	}
+	userMap := users.(map[string]interface{})
 	validRoomID, err := spec.NewRoomID(roomID)
 	if err != nil {
 		return err
@@ -277,7 +281,8 @@ func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID strin
 		if err != nil {
 			return err
 		} else if senderID == nil {
-			return fmt.Errorf("sender ID not found for %s in %s", uID, *validRoomID)
+			util.GetLogger(req.Context()).Warnf("sender ID not found for %s in %s", uID, *validRoomID)
+			continue
 		}
 		userMap[string(*senderID)] = level
 		delete(userMap, user)
diff --git a/go.mod b/go.mod
index 564b4250a..78c1058e7 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
@@ -36,7 +36,7 @@ require (
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.2
-	github.com/tidwall/gjson v1.16.0
+	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/sjson v1.2.5
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
diff --git a/go.sum b/go.sum
index f7d21d96b..3cf569497 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a h1:+RC9Ddmt5v4y58qmdz5WuEEWCJ9gBWuYLyndnWkGfXU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230915142004-095d10f3a87a/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49 h1:o4mdKYYIYCi/QplAjBAJ5kvu3NXXkutZF88gTTpZjj4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -318,8 +318,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
-github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
+github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index 5c63a6684..5bea00445 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -93,11 +93,21 @@ func (r *Leaver) performLeaveRoomByID(
 	isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, *leaver)
 	if err == nil && isInvitePending {
 		sender, serr := r.RSAPI.QueryUserIDForSender(ctx, *roomID, senderUser)
-		if serr != nil || sender == nil {
-			return nil, fmt.Errorf("sender %q has no matching userID", senderUser)
+		if serr != nil {
+			return nil, fmt.Errorf("failed looking up userID for sender %q: %w", senderUser, serr)
 		}
-		if !r.Cfg.Matrix.IsLocalServerName(sender.Domain()) {
-			return r.performFederatedRejectInvite(ctx, req, res, *sender, eventID, *leaver)
+
+		var domain spec.ServerName
+		if sender == nil {
+			// TODO: Currently a federated invite has no way of knowing the mxid_mapping of the inviter.
+			// Should we add the inviter's m.room.member event (with mxid_mapping) to invite_room_state to allow
+			// the invited user to leave via the inviter's server?
+			domain = roomID.Domain()
+		} else {
+			domain = sender.Domain()
+		}
+		if !r.Cfg.Matrix.IsLocalServerName(domain) {
+			return r.performFederatedRejectInvite(ctx, req, res, domain, eventID, *leaver)
 		}
 		// check that this is not a "server notice room"
 		accData := &userapi.QueryAccountDataResponse{}
@@ -219,14 +229,14 @@ func (r *Leaver) performFederatedRejectInvite(
 	ctx context.Context,
 	req *api.PerformLeaveRequest,
 	res *api.PerformLeaveResponse, // nolint:unparam
-	inviteSender spec.UserID, eventID string,
+	inviteDomain spec.ServerName, eventID string,
 	leaver spec.SenderID,
 ) ([]api.OutputEvent, error) {
 	// Ask the federation sender to perform a federated leave for us.
 	leaveReq := fsAPI.PerformLeaveRequest{
 		RoomID:      req.RoomID,
 		UserID:      req.Leaver.String(),
-		ServerNames: []spec.ServerName{inviteSender.Domain()},
+		ServerNames: []spec.ServerName{inviteDomain},
 	}
 	leaveRes := fsAPI.PerformLeaveResponse{}
 	if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil {
diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go
index 6812f8332..fe4f6c07f 100644
--- a/syncapi/synctypes/clientevent.go
+++ b/syncapi/synctypes/clientevent.go
@@ -347,71 +347,81 @@ func updatePowerLevelEvent(userIDForSender spec.UserIDForSender, se gomatrixserv
 		return se, nil
 	}
 
-	pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se)
-	if err != nil {
-		return nil, err
-	}
-	newPls := make(map[string]int64)
-	var userID *spec.UserID
-	for user, level := range pls.Users {
-		if eventFormat != FormatSyncFederation {
-			userID, err = userIDForSender(se.RoomID(), spec.SenderID(user))
-			if err != nil {
-				return nil, err
-			}
-			user = userID.String()
-		}
-		newPls[user] = level
-	}
-	var newPlBytes, newEv []byte
-	newPlBytes, err = json.Marshal(newPls)
-	if err != nil {
-		return nil, err
-	}
-	newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes)
-	if err != nil {
-		return nil, err
-	}
+	newEv := se.JSON()
 
-	// do the same for prev content
-	prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content")
-	if !prevContent.Exists() {
-		var evNew gomatrixserverlib.PDU
-		evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false)
+	usersField := gjson.GetBytes(se.JSON(), "content.users")
+	if usersField.Exists() {
+		pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se)
 		if err != nil {
 			return nil, err
 		}
 
-		return evNew, err
-	}
-	pls = gomatrixserverlib.PowerLevelContent{}
-	err = json.Unmarshal([]byte(prevContent.Raw), &pls)
-	if err != nil {
-		return nil, err
+		newPls := make(map[string]int64)
+		var userID *spec.UserID
+		for user, level := range pls.Users {
+			if eventFormat != FormatSyncFederation {
+				userID, err = userIDForSender(se.RoomID(), spec.SenderID(user))
+				if err != nil {
+					return nil, err
+				}
+				user = userID.String()
+			}
+			newPls[user] = level
+		}
+
+		var newPlBytes []byte
+		newPlBytes, err = json.Marshal(newPls)
+		if err != nil {
+			return nil, err
+		}
+		newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	newPls = make(map[string]int64)
-	for user, level := range pls.Users {
-		if eventFormat != FormatSyncFederation {
-			userID, err = userIDForSender(se.RoomID(), spec.SenderID(user))
+	// do the same for prev content
+	prevUsersField := gjson.GetBytes(se.JSON(), "unsigned.prev_content.users")
+	if prevUsersField.Exists() {
+		prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content")
+		if !prevContent.Exists() {
+			evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false)
 			if err != nil {
 				return nil, err
 			}
-			user = userID.String()
+
+			return evNew, err
+		}
+		pls := gomatrixserverlib.PowerLevelContent{}
+		err := json.Unmarshal([]byte(prevContent.Raw), &pls)
+		if err != nil {
+			return nil, err
+		}
+
+		newPls := make(map[string]int64)
+		for user, level := range pls.Users {
+			if eventFormat != FormatSyncFederation {
+				userID, userErr := userIDForSender(se.RoomID(), spec.SenderID(user))
+				if userErr != nil {
+					return nil, userErr
+				}
+				user = userID.String()
+			}
+			newPls[user] = level
+		}
+
+		var newPlBytes []byte
+		newPlBytes, err = json.Marshal(newPls)
+		if err != nil {
+			return nil, err
+		}
+		newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes)
+		if err != nil {
+			return nil, err
 		}
-		newPls[user] = level
-	}
-	newPlBytes, err = json.Marshal(newPls)
-	if err != nil {
-		return nil, err
-	}
-	newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes)
-	if err != nil {
-		return nil, err
 	}
 
-	var evNew gomatrixserverlib.PDU
-	evNew, err = gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false)
+	evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false)
 	if err != nil {
 		return nil, err
 	}

From 05a8f1ede3ca0ccd0f9a7d327ef3cb30b6fd2798 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 27 Sep 2023 08:27:08 +0200
Subject: [PATCH 27/57] Support for room version v11 (#3204)

Fixes #3203
---
 are-we-synapse-yet.list                            | 10 +++++++++-
 clientapi/routing/redaction.go                     |  8 +++++++-
 go.mod                                             |  2 +-
 go.sum                                             |  4 ++--
 roomserver/internal/perform/perform_create_room.go | 11 ++++++++++-
 sytest-whitelist                                   | 10 +++++++++-
 6 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list
index 585374738..80c0dbff6 100644
--- a/are-we-synapse-yet.list
+++ b/are-we-synapse-yet.list
@@ -944,4 +944,12 @@ rmv remote user can join room with version 10
 rmv User can invite remote user to room with version 10
 rmv Remote user can backfill in a room with version 10
 rmv Can reject invites over federation for rooms with version 10
-rmv Can receive redactions from regular users over federation in room version 10
\ No newline at end of file
+rmv Can receive redactions from regular users over federation in room version 10
+rmv User can create and send/receive messages in a room with version 11
+rmv local user can join room with version 11
+rmv User can invite local user to room with version 11
+rmv remote user can join room with version 11
+rmv User can invite remote user to room with version 11
+rmv Remote user can backfill in a room with version 11
+rmv Can reject invites over federation for rooms with version 11
+rmv Can receive redactions from regular users over federation in room version 11
\ No newline at end of file
diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go
index aa579db64..f331a73c7 100644
--- a/clientapi/routing/redaction.go
+++ b/clientapi/routing/redaction.go
@@ -34,7 +34,8 @@ import (
 )
 
 type redactionContent struct {
-	Reason string `json:"reason"`
+	Reason  string `json:"reason"`
+	Redacts string `json:"redacts"`
 }
 
 type redactionResponse struct {
@@ -151,6 +152,11 @@ func SendRedaction(
 		Type:     spec.MRoomRedaction,
 		Redacts:  eventID,
 	}
+
+	// Room version 11 expects the "redacts" field on the
+	// content field, so add it here as well
+	r.Redacts = eventID
+
 	err = proto.SetContent(r)
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed")
diff --git a/go.mod b/go.mod
index 78c1058e7..3176a8b7b 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
diff --git a/go.sum b/go.sum
index 3cf569497..7f0e98838 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49 h1:o4mdKYYIYCi/QplAjBAJ5kvu3NXXkutZF88gTTpZjj4=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230926023021-d4830c9bfa49/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 h1:UuXfC7b29RBDfMdLmggeF3opu3XuGi8bNT9SKZtZc3I=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go
index cd6629d28..eb8de7811 100644
--- a/roomserver/internal/perform/perform_create_room.go
+++ b/roomserver/internal/perform/perform_create_room.go
@@ -90,7 +90,16 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo
 	} else {
 		senderID = spec.SenderID(userID.String())
 	}
-	createContent["creator"] = senderID
+
+	// TODO: Maybe, at some point, GMSL should return the events to create, so we can define the version
+	// entirely there.
+	switch createRequest.RoomVersion {
+	case gomatrixserverlib.RoomVersionV11:
+		// RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175
+	default:
+		createContent["creator"] = senderID
+	}
+
 	createContent["room_version"] = createRequest.RoomVersion
 	powerLevelContent := eventutil.InitialPowerLevelsContent(string(senderID))
 	joinRuleContent := gomatrixserverlib.JoinRuleContent{
diff --git a/sytest-whitelist b/sytest-whitelist
index 60ba02302..492c756ba 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -787,4 +787,12 @@ Local device key changes get to remote servers with correct prev_id
 HS provides query metadata
 HS can provide query metadata on a single protocol
 Invites over federation are correctly pushed
-Invites over federation are correctly pushed with name
\ No newline at end of file
+Invites over federation are correctly pushed with name
+User can create and send/receive messages in a room with version 11
+local user can join room with version 11
+User can invite local user to room with version 11
+remote user can join room with version 11
+User can invite remote user to room with version 11
+Remote user can backfill in a room with version 11
+Can reject invites over federation for rooms with version 11
+Can receive redactions from regular users over federation in room version 11
\ No newline at end of file

From 10b4fbc66d4dca20faa6cc56a272d5303871094b Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Sep 2023 07:36:34 +0200
Subject: [PATCH 28/57] Fix m.direct only being partially upgraded (#3209)

Previously we would update `m.direct` once we found the old room ID. If
the roomID is found somewhere in the middle, we would never add the rest
of the users, resulting in only partially upgraded `m.direct` and chats
loosing their 1:1 flag.
---
 userapi/consumers/roomserver.go | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index d5baa074c..6da41f8a1 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -266,8 +266,8 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID,
 	directChats := gjson.ParseBytes(directChatsRaw)
 	newDirectChats := make(map[string][]string)
 	// iterate over all userID -> roomIDs
+	var found bool
 	directChats.ForEach(func(userID, roomIDs gjson.Result) bool {
-		var found bool
 		for _, roomID := range roomIDs.Array() {
 			newDirectChats[userID.Str] = append(newDirectChats[userID.Str], roomID.Str)
 			// add the new roomID to m.direct
@@ -276,22 +276,21 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID,
 				newDirectChats[userID.Str] = append(newDirectChats[userID.Str], newRoomID)
 			}
 		}
-		// Only hit the database if we found the old room as a DM for this user
-		if found {
-			var data []byte
-			data, err = json.Marshal(newDirectChats)
-			if err != nil {
-				return true
-			}
-			if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil {
-				return true
-			}
-		}
 		return true
 	})
-	if err != nil {
-		return fmt.Errorf("failed to update m.direct state")
+
+	// Only hit the database if we found the old room as a DM for this user
+	if found {
+		var data []byte
+		data, err = json.Marshal(newDirectChats)
+		if err != nil {
+			return err
+		}
+		if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil {
+			return fmt.Errorf("failed to update m.direct state: %w", err)
+		}
 	}
+
 	return nil
 }
 

From f02d998253a6fb2b4e4563f7bd66ac5f00aa97f9 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Sep 2023 07:36:57 +0200
Subject: [PATCH 29/57] Remove the creator field when upgrading to v11 (#3210)

Minor oversight
---
 roomserver/internal/perform/perform_upgrade.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go
index c32e10d53..9d7c7b567 100644
--- a/roomserver/internal/perform/perform_upgrade.go
+++ b/roomserver/internal/perform/perform_upgrade.go
@@ -368,7 +368,16 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query
 	// in the create event (such as for the room types MSC).
 	newCreateContent := map[string]interface{}{}
 	_ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent)
-	newCreateContent["creator"] = string(senderID)
+
+	switch newVersion {
+	case gomatrixserverlib.RoomVersionV11:
+		// RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175
+		// So if we are upgrading from pre v11, we need to remove the field.
+		delete(newCreateContent, "creator")
+	default:
+		newCreateContent["creator"] = senderID
+	}
+
 	newCreateContent["room_version"] = newVersion
 	newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{
 		EventID: tombstoneEvent.EventID(),

From f1db57c7f80093d4b9727d72dff93f1fe6d9f829 Mon Sep 17 00:00:00 2001
From: jahway603 <64485701+jahway603@users.noreply.github.com>
Date: Thu, 28 Sep 2023 01:38:29 -0400
Subject: [PATCH 30/57] Updated minimum required go version in README.md
 (#3194)

Updated minimum required go version in README.md

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `jahway603 <jahway603@protonmail.com>`

Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 34604eff9..bde19b07e 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
 more information on requirements.
 
-To build Dendrite, you will need Go 1.18 or later.
+To build Dendrite, you will need Go 1.20 or later.
 
 For a usable federating Dendrite deployment, you will also need:
 

From 4d344b65b2132b1c4ce7fe4f142295b9716c9ca5 Mon Sep 17 00:00:00 2001
From: Tracker-Friendly <109815155+Tracker-Friendly@users.noreply.github.com>
Date: Thu, 28 Sep 2023 05:40:12 +0000
Subject: [PATCH 31/57] Fixed typo in documentation (#3212)

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [ ] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests

This PR doesn't need tests because it's a documentation update

* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed off privately

Co-authored-by: Tracker-Friendly <jliwin98@pm.me>
---
 docs/administration/5_optimisation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/administration/5_optimisation.md b/docs/administration/5_optimisation.md
index b327171eb..57b7924d3 100644
--- a/docs/administration/5_optimisation.md
+++ b/docs/administration/5_optimisation.md
@@ -95,7 +95,7 @@ Consider enabling the DNS cache by modifying the `global` section of your config
 ## Time synchronisation
 
 Matrix relies heavily on TLS which requires the system time to be correct. If the clock
-drifts then you may find that federation no works reliably (or at all) and clients may
+drifts then you may find that federation will not work reliably (or at all) and clients may
 struggle to connect to your Dendrite server.
 
 Ensure that the time is synchronised on your system by enabling NTP sync.

From b341a66152269ea8c031955a6654dfb4b07e56e3 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Sep 2023 12:06:21 +0200
Subject: [PATCH 32/57] Version 0.13.3 (#3213)

---
 .github/workflows/helm.yml            |  2 +-
 CHANGES.md                            | 19 +++++++++++++++++++
 cmd/dendrite-demo-yggdrasil/README.md |  2 +-
 cmd/dendrite-upgrade-tests/main.go    |  3 +--
 docs/installation/1_planning.md       |  2 +-
 go.mod                                |  2 +-
 helm/dendrite/Chart.yaml              |  4 ++--
 helm/dendrite/README.md               |  5 +----
 internal/version.go                   |  2 +-
 9 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml
index bf62a1c19..9a5eb2b62 100644
--- a/.github/workflows/helm.yml
+++ b/.github/workflows/helm.yml
@@ -32,7 +32,7 @@ jobs:
           version: v3.10.0
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.4.1
+        uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         with:
diff --git a/CHANGES.md b/CHANGES.md
index f4a814566..bdb6a796e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,24 @@
 # Changelog
 
+## Dendrite 0.13.3 (2023-09-28)
+
+### Fixes:
+
+- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir))
+- Invitations are now correctly pushed to devices
+- A bug which could result in the corruption of `m.direct` account data has been fixed 
+
+### Features
+
+- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response
+- Room version 11 is now supported
+- Clients can request the `federation` `event_format` when creating filters
+- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+
+### Other
+
+- Dendrite now requires Go 1.20 if building from source
+
 ## Dendrite 0.13.2 (2023-08-23)
 
 ### Fixes:
diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md
index 14fc3a2db..23304c214 100644
--- a/cmd/dendrite-demo-yggdrasil/README.md
+++ b/cmd/dendrite-demo-yggdrasil/README.md
@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run:
 
diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index 68919e525..b78c5f605 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -515,7 +514,7 @@ func testCreateAccount(dockerClient *client.Client, version *semver.Version, con
 	}
 	defer response.Close()
 
-	data, err := ioutil.ReadAll(response.Reader)
+	data, err := io.ReadAll(response.Reader)
 	if err != nil {
 		return err
 	}
diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md
index 354003aef..37ca5702a 100644
--- a/docs/installation/1_planning.md
+++ b/docs/installation/1_planning.md
@@ -59,7 +59,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie
 
 ### Go
 
-At this time, Dendrite supports being built with Go 1.18 or later. We do not support building
+At this time, Dendrite supports being built with Go 1.20 or later. We do not support building
 Dendrite with older versions of Go than this. If you are installing Go using a package manager,
 you should check (by running `go version`) that you are using a suitable version before you start.
 
diff --git a/go.mod b/go.mod
index 3176a8b7b..c2ab105b1 100644
--- a/go.mod
+++ b/go.mod
@@ -143,4 +143,4 @@ require (
 	modernc.org/token v1.0.1 // indirect
 )
 
-go 1.18
+go 1.20
diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index 5590a39b1..674152051 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.3"
-appVersion: "0.13.2"
+version: "0.13.4"
+appVersion: "0.13.3"
 description: Dendrite Matrix Homeserver
 type: application
 keywords:
diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index 7f7ea484a..cb78a54f2 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.2](https://img.shields.io/badge/Version-0.13.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.2](https://img.shields.io/badge/AppVersion-0.13.2-informational?style=flat-square)
+![Version: 0.13.4](https://img.shields.io/badge/Version-0.13.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.3](https://img.shields.io/badge/AppVersion-0.13.3-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -63,9 +63,6 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
 | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process |
 | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods |
-| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
-| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process |
-| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods |
 | dendrite_config.version | int | `2` |  |
 | dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. |
 | dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) |
diff --git a/internal/version.go b/internal/version.go
index 81e0fc529..1f8a62bc2 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -18,7 +18,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 13
-	VersionPatch = 2
+	VersionPatch = 3
 	VersionTag   = "" // example: "rc1"
 
 	gitRevLen = 7 // 7 matches the displayed characters on github.com

From 1853f58cb492c423641636562c6aa6eb701426a2 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Sep 2023 12:38:53 +0200
Subject: [PATCH 33/57] Add missing sliding sync config

---
 dendrite-sample.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml
index 8abc23011..7affc2599 100644
--- a/dendrite-sample.yaml
+++ b/dendrite-sample.yaml
@@ -72,6 +72,10 @@ global:
   # The base URL to delegate client-server communications to e.g. https://localhost
   well_known_client_name: ""
 
+  # The server name to delegate sliding sync communications to, with optional port.
+  # Requires `well_known_client_name` to also be configured.
+  well_known_sliding_sync_proxy: ""
+
   # Lists of domains that the server will trust as identity servers to verify third
   # party identifiers such as phone numbers and email addresses.
   trusted_third_party_id_servers:

From 3d02c8103152fbb9a5401a091ef0f681a32bfeee Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Sep 2023 14:50:31 +0200
Subject: [PATCH 34/57] Fix tests for x86 (#3214)

---
 syncapi/internal/history_visibility_test.go   | 2 +-
 syncapi/storage/interface.go                  | 2 +-
 syncapi/storage/postgres/memberships_table.go | 2 +-
 syncapi/storage/shared/storage_consumer.go    | 2 +-
 syncapi/storage/sqlite3/memberships_table.go  | 2 +-
 syncapi/storage/tables/interface.go           | 2 +-
 syncapi/storage/tables/memberships_test.go    | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/syncapi/internal/history_visibility_test.go b/syncapi/internal/history_visibility_test.go
index 984f90edd..24515bbb2 100644
--- a/syncapi/internal/history_visibility_test.go
+++ b/syncapi/internal/history_visibility_test.go
@@ -58,7 +58,7 @@ type mockDB struct {
 	roomID            string
 }
 
-func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int, error) {
+func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int64, error) {
 	if roomID == s.roomID {
 		membership, ok := s.currentMembership[userID]
 		if !ok {
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index dca5d1a14..97c781b9b 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -107,7 +107,7 @@ type DatabaseTransaction interface {
 	// SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found
 	// returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty
 	// string as the membership.
-	SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
+	SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error)
 	// getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms
 	GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error)
 	GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error)
diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go
index fcbe14b16..4fe4260da 100644
--- a/syncapi/storage/postgres/memberships_table.go
+++ b/syncapi/storage/postgres/memberships_table.go
@@ -131,7 +131,7 @@ func (s *membershipsStatements) SelectMembershipCount(
 // string as the membership.
 func (s *membershipsStatements) SelectMembershipForUser(
 	ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64,
-) (membership string, topologyPos int, err error) {
+) (membership string, topologyPos int64, err error) {
 	stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt)
 	err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos)
 	if err != nil {
diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go
index 0f4080d53..923ead9bd 100644
--- a/syncapi/storage/shared/storage_consumer.go
+++ b/syncapi/storage/shared/storage_consumer.go
@@ -583,7 +583,7 @@ func (d *Database) GetPresences(ctx context.Context, userIDs []string) ([]*types
 	return d.Presence.GetPresenceForUsers(ctx, nil, userIDs)
 }
 
-func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) {
+func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) {
 	return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos)
 }
 
diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go
index 05f756fda..a1b16306c 100644
--- a/syncapi/storage/sqlite3/memberships_table.go
+++ b/syncapi/storage/sqlite3/memberships_table.go
@@ -134,7 +134,7 @@ func (s *membershipsStatements) SelectMembershipCount(
 // string as the membership.
 func (s *membershipsStatements) SelectMembershipForUser(
 	ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64,
-) (membership string, topologyPos int, err error) {
+) (membership string, topologyPos int64, err error) {
 	stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt)
 	err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos)
 	if err != nil {
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index f5c66c42d..45117d6d3 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -194,7 +194,7 @@ type Receipts interface {
 type Memberships interface {
 	UpsertMembership(ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error
 	SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error)
-	SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
+	SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error)
 	PurgeMemberships(ctx context.Context, txn *sql.Tx, roomID string) error
 	SelectMemberships(
 		ctx context.Context, txn *sql.Tx,
diff --git a/syncapi/storage/tables/memberships_test.go b/syncapi/storage/tables/memberships_test.go
index a421a9772..0a36f5887 100644
--- a/syncapi/storage/tables/memberships_test.go
+++ b/syncapi/storage/tables/memberships_test.go
@@ -124,7 +124,7 @@ func testUpsert(t *testing.T, ctx context.Context, table tables.Memberships, mem
 		if err != nil {
 			t.Fatalf("failed to select membership: %s", err)
 		}
-		expectedPos := 1
+		var expectedPos int64 = 1
 		if pos != expectedPos {
 			t.Fatalf("expected pos to be %d, got %d", expectedPos, pos)
 		}

From 2259e71c0cc7d154eefd85a6466b08e7afb7f69a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 5 Oct 2023 10:33:04 +0200
Subject: [PATCH 35/57] Fix `resolve-state` (#3229)

Previously we would "start" the roomserver API, which isn't the best
idea, given it also starts processing Jetstream events. We now use a
`dummyQuerier` to implement the needed interface for "converting"
userID/senderIDs. As per the comment, this **DOES NOT** do any magic for
pseudoID rooms.
---
 cmd/resolve-state/main.go | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index 3ffcac9e6..5be449097 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -11,13 +11,11 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -35,6 +33,19 @@ var roomVersion = flag.String("roomversion", "5", "the room version to parse eve
 var filterType = flag.String("filtertype", "", "the event types to filter on")
 var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots")
 
+// dummyQuerier implements QuerySenderIDAPI. Does **NOT** do any "magic" for pseudoID rooms
+// to avoid having to "start" a full roomserver API.
+type dummyQuerier struct{}
+
+func (d dummyQuerier) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	s := spec.SenderIDFromUserID(userID)
+	return &s, nil
+}
+
+func (d dummyQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	return senderID.ToUserID(), nil
+}
+
 // nolint:gocyclo
 func main() {
 	ctx := context.Background()
@@ -56,27 +67,32 @@ func main() {
 		}
 	}
 
-	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
-
 	processCtx := process.NewProcessContext()
 	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+
+	dbOpts := cfg.RoomServer.Database
+	if dbOpts.ConnectionString == "" {
+		dbOpts = cfg.Global.DatabaseOptions
+	}
+
+	fmt.Println("Opening database")
 	roomserverDB, err := storage.Open(
-		processCtx.Context(), cm, &cfg.RoomServer.Database,
-		caching.NewRistrettoCache(128*1024*1024, time.Hour, true),
+		processCtx.Context(), cm, &dbOpts,
+		caching.NewRistrettoCache(8*1024*1024, time.Minute*5, caching.DisableMetrics),
 	)
 	if err != nil {
 		panic(err)
 	}
 
-	natsInstance := &jetstream.NATSInstance{}
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm,
-		natsInstance, caching.NewRistrettoCache(128*1024*1024, time.Hour, true), false)
+	rsAPI := dummyQuerier{}
 
 	roomInfo := &types.RoomInfo{
 		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
 	}
 	stateres := state.NewStateResolution(roomserverDB, roomInfo, rsAPI)
 
+	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
+
 	if *difference {
 		if len(snapshotNIDs) != 2 {
 			panic("need exactly two state snapshot NIDs to calculate difference")

From 5888329b133468b23bb6d84ab9f244c667c9ac17 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Wed, 11 Oct 2023 17:41:12 +0100
Subject: [PATCH 36/57] Update Complement to match new public API shape (#3232)

Sister PR to matrix-org/complement#666

Context:
https://github.com/matrix-org/complement/issues/654#issuecomment-1746613495
---
 .github/workflows/dendrite.yml | 2 +-
 build/scripts/complement.sh    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 772b45cb2..ac40f06b0 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -440,7 +440,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt -hide all
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:
diff --git a/build/scripts/complement.sh b/build/scripts/complement.sh
index 29feff304..8608d8fa5 100755
--- a/build/scripts/complement.sh
+++ b/build/scripts/complement.sh
@@ -15,5 +15,5 @@ tar -xzf master.tar.gz
 
 # Run the tests!
 cd complement-master
-COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests
+COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi
 

From 933ae2db91c4c65dfc7b8a395567cd39f4c366c8 Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Thu, 12 Oct 2023 18:03:06 +0000
Subject: [PATCH 37/57] Update bug report to reflect current team members
 (#3234)

---
 .github/ISSUE_TEMPLATE/BUG_REPORT.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
index a3b8c0754..e18968baf 100644
--- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md
+++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
@@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include
 those (please be careful to remove any personal or private data). Please surround them with
 ``` (three backticks, on a line on their own), so that they are formatted legibly.
 
-Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org
+Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host
 with a link to the respective Github issue, thanks!
 -->

From fe2955a4dbf40562374d79bc991e0d7028f0f239 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Oct 2023 09:40:21 +0200
Subject: [PATCH 38/57] Bump golang.org/x/net from 0.14.0 to 0.17.0 (#3233)

Bumps [golang.org/x/net](https://github.com/golang/net) from 0.14.0 to
0.17.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/net/commit/b225e7ca6dde1ef5a5ae5ce922861bda011cfabd"><code>b225e7c</code></a>
http2: limit maximum handler goroutines to MaxConcurrentStreams</li>
<li><a
href="https://github.com/golang/net/commit/88194ad8ab44a02ea952c169883c3f57db6cf9f4"><code>88194ad</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/net/commit/2b60a61f1e4cf3a5ecded0bd7e77ea168289e6de"><code>2b60a61</code></a>
quic: fix several bugs in flow control accounting</li>
<li><a
href="https://github.com/golang/net/commit/73d82efb96cacc0c378bc150b56675fc191894b9"><code>73d82ef</code></a>
quic: handle DATA_BLOCKED frames</li>
<li><a
href="https://github.com/golang/net/commit/5d5a036a503f8accd748f7453c0162115187be13"><code>5d5a036</code></a>
quic: handle streams moving from the data queue to the meta queue</li>
<li><a
href="https://github.com/golang/net/commit/350aad2603e57013fafb1a9e2089a382fe67dc80"><code>350aad2</code></a>
quic: correctly extend peer's flow control window after MAX_DATA</li>
<li><a
href="https://github.com/golang/net/commit/21814e71db756f39b69fb1a3e06350fa555a79b1"><code>21814e7</code></a>
quic: validate connection id transport parameters</li>
<li><a
href="https://github.com/golang/net/commit/a600b3518eed7a9a4e24380b4b249cb986d9b64d"><code>a600b35</code></a>
quic: avoid redundant MAX_DATA updates</li>
<li><a
href="https://github.com/golang/net/commit/ea633599b58dc6a50d33c7f5438edfaa8bc313df"><code>ea63359</code></a>
http2: check stream body is present on read timeout</li>
<li><a
href="https://github.com/golang/net/commit/ddd8598e5694aa5e966e44573a53e895f6fa5eb2"><code>ddd8598</code></a>
quic: version negotiation</li>
<li>Additional commits viewable in <a
href="https://github.com/golang/net/compare/v0.14.0...v0.17.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/net&package-manager=go_modules&previous-version=0.14.0&new-version=0.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  8 ++++----
 go.sum | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/go.mod b/go.mod
index c2ab105b1..839e23f4c 100644
--- a/go.mod
+++ b/go.mod
@@ -42,12 +42,12 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
 	go.uber.org/atomic v1.10.0
-	golang.org/x/crypto v0.13.0
+	golang.org/x/crypto v0.14.0
 	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
 	golang.org/x/image v0.5.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
 	golang.org/x/sync v0.3.0
-	golang.org/x/term v0.12.0
+	golang.org/x/term v0.13.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
@@ -123,8 +123,8 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.12.0 // indirect
diff --git a/go.sum b/go.sum
index 7f0e98838..e036f7b2b 100644
--- a/go.sum
+++ b/go.sum
@@ -354,8 +354,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -386,8 +386,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -418,12 +418,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

From 8c23c1150c37a88e078037b8c4b47f4efecab727 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 23 Oct 2023 11:09:05 +0200
Subject: [PATCH 39/57] Tweaks around the device list updater (#3227)

I hope the comments explain the changes.

`notifyWorkers` notifies a worker which then calls `processServer`,
which in turn gets all users and calls `processServerUser`. There is no
need to call `processServer` for the same domain on startup.
---
 .golangci.yml                               |  2 +-
 userapi/internal/device_list_update.go      | 45 ++++++++++++++++----
 userapi/internal/device_list_update_test.go | 46 +++++++++++++++++++++
 3 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index bb8d38a8b..5bee0a885 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -180,7 +180,6 @@ linters-settings:
 linters:
   enable:
     - errcheck
-    - goconst
     - gocyclo
     - goimports # Does everything gofmt does
     - gosimple
@@ -211,6 +210,7 @@ linters:
     - stylecheck
     - typecheck # Should turn back on soon
     - unconvert # Should turn back on soon
+    - goconst # Slightly annoying, as it reports "issues" in SQL statements
   disable-all: false
   presets:
   fast: false
diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go
index 3fccf56bb..2f33589fe 100644
--- a/userapi/internal/device_list_update.go
+++ b/userapi/internal/device_list_update.go
@@ -180,11 +180,13 @@ func (u *DeviceListUpdater) Start() error {
 	if err != nil {
 		return err
 	}
+
+	newStaleLists := dedupeStaleLists(staleLists)
 	offset, step := time.Second*10, time.Second
-	if max := len(staleLists); max > 120 {
+	if max := len(newStaleLists); max > 120 {
 		step = (time.Second * 120) / time.Duration(max)
 	}
-	for _, userID := range staleLists {
+	for _, userID := range newStaleLists {
 		userID := userID // otherwise we are only sending the last entry
 		time.AfterFunc(offset, func() {
 			u.notifyWorkers(userID)
@@ -416,6 +418,12 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) {
 
 func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Duration, bool) {
 	ctx := u.process.Context()
+	// If the process.Context is canceled, there is no need to go further.
+	// This avoids spamming the logs when shutting down
+	if errors.Is(ctx.Err(), context.Canceled) {
+		return defaultWaitTime, false
+	}
+
 	logger := util.GetLogger(ctx).WithField("server_name", serverName)
 	deviceListUpdateCount.WithLabelValues(string(serverName)).Inc()
 
@@ -428,13 +436,6 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura
 		return waitTime, true
 	}
 
-	defer func() {
-		for _, userID := range userIDs {
-			// always clear the channel to unblock Update calls regardless of success/failure
-			u.clearChannel(userID)
-		}
-	}()
-
 	for _, userID := range userIDs {
 		userWait, err := u.processServerUser(ctx, serverName, userID)
 		if err != nil {
@@ -461,6 +462,11 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura
 func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName spec.ServerName, userID string) (time.Duration, error) {
 	ctx, cancel := context.WithTimeout(ctx, requestTimeout)
 	defer cancel()
+
+	// If we are processing more than one user per server, this unblocks further calls to Update
+	// immediately instead of just after **all** users have been processed.
+	defer u.clearChannel(userID)
+
 	logger := util.GetLogger(ctx).WithFields(logrus.Fields{
 		"server_name": serverName,
 		"user_id":     userID,
@@ -579,3 +585,24 @@ func (u *DeviceListUpdater) updateDeviceList(res *fclient.RespUserDevices) error
 	}
 	return nil
 }
+
+// dedupeStaleLists de-duplicates the stateList entries using the domain.
+// This is used on startup, processServer is getting all users anyway, so
+// there is no need to send every user to the workers.
+func dedupeStaleLists(staleLists []string) []string {
+	seenDomains := make(map[spec.ServerName]struct{})
+	newStaleLists := make([]string, 0, len(staleLists))
+	for _, userID := range staleLists {
+		_, domain, err := gomatrixserverlib.SplitID('@', userID)
+		if err != nil {
+			// non-fatal and should not block starting up
+			continue
+		}
+		if _, ok := seenDomains[domain]; ok {
+			continue
+		}
+		newStaleLists = append(newStaleLists, userID)
+		seenDomains[domain] = struct{}{}
+	}
+	return newStaleLists
+}
diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go
index 10b9c6521..38fd8b583 100644
--- a/userapi/internal/device_list_update_test.go
+++ b/userapi/internal/device_list_update_test.go
@@ -428,3 +428,49 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) {
 		}
 	})
 }
+
+func Test_dedupeStateList(t *testing.T) {
+	alice := "@alice:localhost"
+	bob := "@bob:localhost"
+	charlie := "@charlie:notlocalhost"
+	invalidUserID := "iaminvalid:localhost"
+
+	tests := []struct {
+		name       string
+		staleLists []string
+		want       []string
+	}{
+		{
+			name:       "empty stateLists",
+			staleLists: []string{},
+			want:       []string{},
+		},
+		{
+			name:       "single entry",
+			staleLists: []string{alice},
+			want:       []string{alice},
+		},
+		{
+			name:       "multiple entries without dupe servers",
+			staleLists: []string{alice, charlie},
+			want:       []string{alice, charlie},
+		},
+		{
+			name:       "multiple entries with dupe servers",
+			staleLists: []string{alice, bob, charlie},
+			want:       []string{alice, charlie},
+		},
+		{
+			name:       "list with invalid userID",
+			staleLists: []string{alice, bob, invalidUserID},
+			want:       []string{alice},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := dedupeStaleLists(tt.staleLists); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("dedupeStaleLists() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

From 8b3adaf244a699e81577d81f1e0a1a38d42e5f51 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 23 Oct 2023 15:17:21 +0200
Subject: [PATCH 40/57] Fix state resets (#3231)

Needs https://github.com/matrix-org/gomatrixserverlib/pull/419

May fix: https://github.com/matrix-org/dendrite/issues/2508,
https://github.com/matrix-org/dendrite/issues/1760
---
 clientapi/routing/profile.go  |   2 +-
 go.mod                        |   2 +-
 go.sum                        |   4 +-
 roomserver/roomserver_test.go | 112 +++++++++++++++++++++++++++++++++-
 4 files changed, 114 insertions(+), 6 deletions(-)

diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go
index 564cd588a..9959144c8 100644
--- a/clientapi/routing/profile.go
+++ b/clientapi/routing/profile.go
@@ -300,7 +300,7 @@ func updateProfile(
 		}, e
 	}
 
-	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil {
+	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
diff --git a/go.mod b/go.mod
index 839e23f4c..7d4c3677b 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
diff --git a/go.sum b/go.sum
index e036f7b2b..c1392cbd4 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4 h1:UuXfC7b29RBDfMdLmggeF3opu3XuGi8bNT9SKZtZc3I=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230926165653-79fcff283fc4/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be h1:bZP16ydP8uRoRBo1p/7WHMexjg7JJGj81fKzZ1FULb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index 47626b30a..90e67b699 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -407,7 +407,9 @@ type fledglingEvent struct {
 	RoomID     string
 	Redacts    string
 	Depth      int64
-	PrevEvents []interface{}
+	PrevEvents []any
+	AuthEvents []any
+	Content    map[string]any
 }
 
 func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEvent) {
@@ -424,7 +426,13 @@ func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEve
 		Depth:      ev.Depth,
 		PrevEvents: ev.PrevEvents,
 	})
-	err := eb.SetContent(map[string]interface{}{})
+	if ev.Content == nil {
+		ev.Content = map[string]any{}
+	}
+	if ev.AuthEvents != nil {
+		eb.AuthEvents = ev.AuthEvents
+	}
+	err := eb.SetContent(ev.Content)
 	if err != nil {
 		t.Fatalf("mustCreateEvent: failed to marshal event content %v", err)
 	}
@@ -1077,3 +1085,103 @@ func TestUpgrade(t *testing.T) {
 		}
 	})
 }
+
+func TestStateReset(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t)
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		// Prepare APIs
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := jetstream.NATSInstance{}
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		// create a new room
+		room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat))
+
+		// join with Bob and Charlie
+		bobJoinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(bob.ID))
+		charlieJoinEv := room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(charlie.ID))
+
+		// Send and create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Errorf("failed to send events: %v", err)
+		}
+
+		// send a message
+		bobMsg := room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"})
+		charlieMsg := room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"})
+
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil {
+			t.Errorf("failed to send events: %v", err)
+		}
+
+		// Bob changes his name
+		expectedDisplayname := "Bob!"
+		bobDisplayname := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join", "displayname": expectedDisplayname}, test.WithStateKey(bob.ID))
+
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobDisplayname}, "test", "test", "test", nil, false); err != nil {
+			t.Errorf("failed to send events: %v", err)
+		}
+
+		// Change another state event
+		jrEv := room.CreateAndInsert(t, alice, spec.MRoomJoinRules, gomatrixserverlib.JoinRuleContent{JoinRule: "invite"}, test.WithStateKey(""))
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{jrEv}, "test", "test", "test", nil, false); err != nil {
+			t.Errorf("failed to send events: %v", err)
+		}
+
+		// send a message
+		bobMsg = room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"})
+		charlieMsg = room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"})
+
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil {
+			t.Errorf("failed to send events: %v", err)
+		}
+
+		// Craft the state reset message, which is using Bobs initial join event and the
+		// last message Charlie sent as the prev_events. This should trigger the recalculation
+		// of the "current" state, since the message event does not have state and no missing events in the DB.
+		stateResetMsg := mustCreateEvent(t, fledglingEvent{
+			Type:     "m.room.message",
+			SenderID: charlie.ID,
+			RoomID:   room.ID,
+			Depth:    charlieMsg.Depth() + 1,
+			PrevEvents: []any{
+				bobJoinEv.EventID(),
+				charlieMsg.EventID(),
+			},
+			AuthEvents: []any{
+				room.Events()[0].EventID(), // create event
+				room.Events()[2].EventID(), // PL event
+				charlieJoinEv.EventID(),    // Charlie join event
+			},
+		})
+
+		// Send the state reset message
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{stateResetMsg}, "test", "test", "test", nil, false); err != nil {
+			t.Errorf("failed to send events: %v", err)
+		}
+
+		// Validate that there is a membership event for Bob
+		bobMembershipEv := api.GetStateEvent(ctx, rsAPI, room.ID, gomatrixserverlib.StateKeyTuple{
+			EventType: spec.MRoomMember,
+			StateKey:  bob.ID,
+		})
+
+		if bobMembershipEv == nil {
+			t.Fatalf("Membership event for Bob does not exist. State reset?")
+		} else {
+			// Validate it's the correct membership event
+			if dn := gjson.GetBytes(bobMembershipEv.Content(), "displayname").Str; dn != expectedDisplayname {
+				t.Fatalf("Expected displayname to be %q, got %q", expectedDisplayname, dn)
+			}
+		}
+	})
+}

From c1d6b9aa8ef29a76b61b1aaf60572d3493adf306 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Oct 2023 09:11:58 +0200
Subject: [PATCH 41/57] Bump github.com/nats-io/nats-server/v2 from 2.9.19 to
 2.9.23 (#3238)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps
[github.com/nats-io/nats-server/v2](https://github.com/nats-io/nats-server)
from 2.9.19 to 2.9.23.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/nats-io/nats-server/releases">github.com/nats-io/nats-server/v2's
releases</a>.</em></p>
<blockquote>
<h2>Release v2.9.23</h2>
<h2>Changelog</h2>
<h3>Go Version</h3>
<ul>
<li>1.20.10</li>
</ul>
<h3>Fixed</h3>
<p>Accounts</p>
<ul>
<li>Prevent bypassing authorization block when enabling system account
access in accounts block (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4605">#4605</a>).
Backport from v2.10.2</li>
</ul>
<p>Leafnodes</p>
<ul>
<li>Prevent a leafnode cluster from receiving a message multiple times
in a queue subscription (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4578">#4578</a>).
Backport from v2.10.2</li>
</ul>
<p>JetStream</p>
<ul>
<li>Hold lock when calculating the first message for subject in a
message block (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4531">#4531</a>).
Backport from v2.10.0</li>
<li>Add self-healing mechanism to detect and delete orphaned Raft groups
(<a
href="https://redirect.github.com/nats-io/nats-server/issues/4647">#4647</a>).
Backport from v2.10.0</li>
<li>Prevent forward proposals in consumers after scaling down a stream
(<a
href="https://redirect.github.com/nats-io/nats-server/issues/4647">#4647</a>).
Backport from v2.10.0</li>
<li>Fix race condition during leader failover scenarios resulting in
potential duplicate messages being sourced (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4592">#4592</a>).
Backport from v2.10.2</li>
</ul>
<h3>Complete Changes</h3>
<p><a
href="https://github.com/nats-io/nats-server/compare/v2.9.22...v2.9.23">https://github.com/nats-io/nats-server/compare/v2.9.22...v2.9.23</a></p>
<h2>Release v2.9.22</h2>
<h2>Changelog</h2>
<h3>Go Version</h3>
<ul>
<li>1.20.8 (updated out-of-cycle since Go 1.19 is now EOL)</li>
</ul>
<h3>Dependencies</h3>
<ul>
<li>github.com/nats-io/jwt/v2 v2.5.0</li>
<li>golang.org/x/crypto v0.12.0</li>
<li>golang.org/x/sys v0.11.0</li>
</ul>
<h3>Improved</h3>
<p>Monitoring</p>
<ul>
<li>CORS Allow-Origin passthrough for monitoring server (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4423">#4423</a>)
Thanks to <a href="https://github.com/mdawar"><code>@​mdawar</code></a>
for the contribution!</li>
</ul>
<p>JetStream</p>
<ul>
<li>Improve consumer scaling reliability with filters and cluster
restart (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4404">#4404</a>)</li>
<li>Send event on lame duck mode (LDM) to avoid placing assets on
shutting down nodes (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4405">#4405</a>)</li>
<li>Skip filestore tombstones if downgrade from 2.10 occurs (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4452">#4452</a>)</li>
<li>Adjust delivered and waiting count when consumer message delivery
fails (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4472">#4472</a>)</li>
</ul>
<h3>Fixed</h3>
<p>Config</p>
<ul>
<li>Allow empty configs and fix JSON compatibility (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4394">#4394</a>,
<a
href="https://redirect.github.com/nats-io/nats-server/issues/4418">#4418</a>)</li>
<li>Remove TLS OCSP debug log on reload (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4453">#4453</a>)</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/nats-io/nats-server/commit/45436e1e5021106cebc9d76e6af0779c908b0f7a"><code>45436e1</code></a>
Release v2.9.23 (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4652">#4652</a>)</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/72ffa38b05811e13228dcac3b9cc16e7ca420321"><code>72ffa38</code></a>
Release v2.9.23</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/05fe77fd083936392534c81f609ca9ad7e39011a"><code>05fe77f</code></a>
Backport <a
href="https://redirect.github.com/nats-io/nats-server/issues/4592">#4592</a>
to 2.9 (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4651">#4651</a>)</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/6a73e6824a4cfbc187727cad522879d7464878e4"><code>6a73e68</code></a>
[2.9.x] Bump Travis Go version to 1.20.10 (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4650">#4650</a>)</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/8b981a26216eb0d8b06fdc65e3012d668510d547"><code>8b981a2</code></a>
Backports from v2.10 for v2.9.23 release (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4647">#4647</a>)</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/28eb7c0ac2fec792c9223001445f3befc5de55c3"><code>28eb7c0</code></a>
Only setup auto no-auth for $G account iff no authorization block was
defined.</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/9f16edd4314c5f469791929cb3949c4a81ef24d3"><code>9f16edd</code></a>
Make sure to not forward a message across a route for dq sub when we are
a sp...</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/0ac7895b983a4dbb12f28bd680abbc028a643439"><code>0ac7895</code></a>
Add in utility to detect and delete any NRG orphans.</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/50722e9ec10de8d3cdafda12c8aadd724ff8e23b"><code>50722e9</code></a>
When scaling a consumer down make sure to pop the
loopAndForwardProposals go ...</li>
<li><a
href="https://github.com/nats-io/nats-server/commit/770cf2edd6b82877f98becdf4324b93f051fe136"><code>770cf2e</code></a>
Backport JetStream benchmarks improvements to 2.9.x (<a
href="https://redirect.github.com/nats-io/nats-server/issues/4644">#4644</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/nats-io/nats-server/compare/v2.9.19...v2.9.23">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/nats-io/nats-server/v2&package-manager=go_modules&previous-version=2.9.19&new-version=2.9.23)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>
---
 go.mod |  8 ++++----
 go.sum | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/go.mod b/go.mod
index 7d4c3677b..e8fbbb81e 100644
--- a/go.mod
+++ b/go.mod
@@ -26,8 +26,8 @@ require (
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
-	github.com/nats-io/nats-server/v2 v2.9.19
-	github.com/nats-io/nats.go v1.27.0
+	github.com/nats-io/nats-server/v2 v2.9.23
+	github.com/nats-io/nats.go v1.28.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/opentracing/opentracing-go v1.2.0
@@ -94,7 +94,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@@ -104,7 +104,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/nats-io/jwt/v2 v2.4.1 // indirect
+	github.com/nats-io/jwt/v2 v2.5.0 // indirect
 	github.com/nats-io/nkeys v0.4.4 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
diff --git a/go.sum b/go.sum
index c1392cbd4..102ff3aa0 100644
--- a/go.sum
+++ b/go.sum
@@ -190,8 +190,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
-github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -242,12 +242,12 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
-github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
-github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
-github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA=
-github.com/nats-io/nats-server/v2 v2.9.19/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw=
-github.com/nats-io/nats.go v1.27.0 h1:3o9fsPhmoKm+yK7rekH2GtWoE+D9jFbw8N3/ayI1C00=
-github.com/nats-io/nats.go v1.27.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
+github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
+github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
+github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
+github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
+github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
+github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
 github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
 github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=

From 1b124fe9cba8b4699a1f65382e4ea1a5674d12fa Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 24 Oct 2023 11:51:08 +0200
Subject: [PATCH 42/57] Implement MSC3987, fix setting Element Android
 notifications (#3242)

Should fix https://github.com/matrix-org/dendrite/issues/3183, since
Element Android already implements
[MSC3987](https://github.com/vector-im/element-android/pull/8530)

This is also part of https://github.com/matrix-org/dendrite/issues/3225
---
 clientapi/clientapi_test.go                  |  2 +-
 internal/pushrules/action_test.go            |  2 --
 internal/pushrules/default_override.go       | 11 +++++------
 internal/pushrules/default_pushrules_test.go |  6 +++---
 internal/pushrules/util.go                   |  5 +----
 internal/pushrules/util_test.go              |  5 ++---
 internal/pushrules/validate.go               |  2 +-
 userapi/consumers/roomserver.go              |  4 ++--
 userapi/consumers/roomserver_test.go         | 10 +++-------
 9 files changed, 18 insertions(+), 29 deletions(-)

diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index 82ec9fea2..f2d617cb9 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -1418,7 +1418,7 @@ func TestPushRules(t *testing.T) {
 				validateFunc: func(t *testing.T, respBody *bytes.Buffer) {
 					actions := gjson.GetBytes(respBody.Bytes(), "actions").Array()
 					// only a basic check
-					assert.Equal(t, 1, len(actions))
+					assert.Equal(t, 0, len(actions))
 				},
 			},
 			{
diff --git a/internal/pushrules/action_test.go b/internal/pushrules/action_test.go
index 72db9c998..d1eb16ef1 100644
--- a/internal/pushrules/action_test.go
+++ b/internal/pushrules/action_test.go
@@ -13,8 +13,6 @@ func TestActionJSON(t *testing.T) {
 		Want Action
 	}{
 		{Action{Kind: NotifyAction}},
-		{Action{Kind: DontNotifyAction}},
-		{Action{Kind: CoalesceAction}},
 		{Action{Kind: SetTweakAction}},
 
 		{Action{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}},
diff --git a/internal/pushrules/default_override.go b/internal/pushrules/default_override.go
index f97427b71..cb825af5f 100644
--- a/internal/pushrules/default_override.go
+++ b/internal/pushrules/default_override.go
@@ -10,6 +10,7 @@ func defaultOverrideRules(userID string) []*Rule {
 		&mRuleRoomNotifDefinition,
 		&mRuleTombstoneDefinition,
 		&mRuleReactionDefinition,
+		&mRuleACLsDefinition,
 	}
 }
 
@@ -30,7 +31,7 @@ var (
 		RuleID:  MRuleMaster,
 		Default: true,
 		Enabled: false,
-		Actions: []*Action{{Kind: DontNotifyAction}},
+		Actions: []*Action{},
 	}
 	mRuleSuppressNoticesDefinition = Rule{
 		RuleID:  MRuleSuppressNotices,
@@ -43,7 +44,7 @@ var (
 				Pattern: pointer("m.notice"),
 			},
 		},
-		Actions: []*Action{{Kind: DontNotifyAction}},
+		Actions: []*Action{},
 	}
 	mRuleMemberEventDefinition = Rule{
 		RuleID:  MRuleMemberEvent,
@@ -56,7 +57,7 @@ var (
 				Pattern: pointer("m.room.member"),
 			},
 		},
-		Actions: []*Action{{Kind: DontNotifyAction}},
+		Actions: []*Action{},
 	}
 	mRuleContainsDisplayNameDefinition = Rule{
 		RuleID:     MRuleContainsDisplayName,
@@ -152,9 +153,7 @@ var (
 				Pattern: pointer("m.reaction"),
 			},
 		},
-		Actions: []*Action{
-			{Kind: DontNotifyAction},
-		},
+		Actions: []*Action{},
 	}
 )
 
diff --git a/internal/pushrules/default_pushrules_test.go b/internal/pushrules/default_pushrules_test.go
index dea829842..506fe3cc8 100644
--- a/internal/pushrules/default_pushrules_test.go
+++ b/internal/pushrules/default_pushrules_test.go
@@ -21,12 +21,12 @@ func TestDefaultRules(t *testing.T) {
 		// Default override rules
 		{
 			name:       ".m.rule.master",
-			inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":["dont_notify"]}`),
+			inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":[]}`),
 			want:       mRuleMasterDefinition,
 		},
 		{
 			name:       ".m.rule.suppress_notices",
-			inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":["dont_notify"]}`),
+			inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":[]}`),
 			want:       mRuleSuppressNoticesDefinition,
 		},
 		{
@@ -36,7 +36,7 @@ func TestDefaultRules(t *testing.T) {
 		},
 		{
 			name:       ".m.rule.member_event",
-			inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":["dont_notify"]}`),
+			inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":[]}`),
 			want:       mRuleMemberEventDefinition,
 		},
 		{
diff --git a/internal/pushrules/util.go b/internal/pushrules/util.go
index de8fe5cd0..e2821b57a 100644
--- a/internal/pushrules/util.go
+++ b/internal/pushrules/util.go
@@ -16,10 +16,7 @@ func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) {
 
 	for _, a := range as {
 		switch a.Kind {
-		case DontNotifyAction:
-			// Don't bother processing any further
-			return DontNotifyAction, nil, nil
-
+		case DontNotifyAction: // Ignored
 		case SetTweakAction:
 			if tweaks == nil {
 				tweaks = map[string]interface{}{}
diff --git a/internal/pushrules/util_test.go b/internal/pushrules/util_test.go
index 89f8243d9..83eee7ede 100644
--- a/internal/pushrules/util_test.go
+++ b/internal/pushrules/util_test.go
@@ -17,17 +17,16 @@ func TestActionsToTweaks(t *testing.T) {
 		{"empty", nil, UnknownAction, nil},
 		{"zero", []*Action{{}}, UnknownAction, nil},
 		{"onlyPrimary", []*Action{{Kind: NotifyAction}}, NotifyAction, nil},
-		{"onlyPrimaryDontNotify", []*Action{{Kind: DontNotifyAction}}, DontNotifyAction, nil},
+		{"onlyPrimaryDontNotify", []*Action{}, UnknownAction, nil},
 		{"onlyTweak", []*Action{{Kind: SetTweakAction, Tweak: HighlightTweak}}, UnknownAction, map[string]interface{}{"highlight": nil}},
 		{"onlyTweakWithValue", []*Action{{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, UnknownAction, map[string]interface{}{"sound": "default"}},
 		{
 			"all",
 			[]*Action{
-				{Kind: CoalesceAction},
 				{Kind: SetTweakAction, Tweak: HighlightTweak},
 				{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"},
 			},
-			CoalesceAction,
+			UnknownAction,
 			map[string]interface{}{"highlight": nil, "sound": "default"},
 		},
 	}
diff --git a/internal/pushrules/validate.go b/internal/pushrules/validate.go
index b54ec3fb0..4cc479345 100644
--- a/internal/pushrules/validate.go
+++ b/internal/pushrules/validate.go
@@ -18,7 +18,7 @@ func ValidateRule(kind Kind, rule *Rule) []error {
 		errs = append(errs, fmt.Errorf("invalid rule ID: %s", rule.RuleID))
 	}
 
-	if len(rule.Actions) == 0 {
+	if rule.Actions == nil {
 		errs = append(errs, fmt.Errorf("missing actions"))
 	}
 	for _, action := range rule.Actions {
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index 6da41f8a1..fca741298 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -538,8 +538,8 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype
 	if err != nil {
 		return fmt.Errorf("pushrules.ActionsToTweaks: %w", err)
 	}
-	// TODO: support coalescing.
-	if a != pushrules.NotifyAction && a != pushrules.CoalesceAction {
+
+	if a != pushrules.NotifyAction {
 		log.WithFields(log.Fields{
 			"event_id":  event.EventID(),
 			"room_id":   event.RoomID().String(),
diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go
index 49dd5b238..7b7c08618 100644
--- a/userapi/consumers/roomserver_test.go
+++ b/userapi/consumers/roomserver_test.go
@@ -81,12 +81,8 @@ func Test_evaluatePushRules(t *testing.T) {
 			{
 				name:         "m.reaction doesn't notify",
 				eventContent: `{"type":"m.reaction","room_id":"!room:example.com"}`,
-				wantAction:   pushrules.DontNotifyAction,
-				wantActions: []*pushrules.Action{
-					{
-						Kind: pushrules.DontNotifyAction,
-					},
-				},
+				wantAction:   pushrules.UnknownAction,
+				wantActions:  []*pushrules.Action{},
 			},
 			{
 				name:         "m.room.message notifies",
@@ -136,7 +132,7 @@ func Test_evaluatePushRules(t *testing.T) {
 					t.Fatalf("expected action to be '%s', got '%s'", tc.wantAction, gotAction)
 				}
 				// this is taken from `notifyLocal`
-				if tc.wantNotify && gotAction != pushrules.NotifyAction && gotAction != pushrules.CoalesceAction {
+				if tc.wantNotify && gotAction != pushrules.NotifyAction {
 					t.Fatalf("expected to notify but didn't")
 				}
 			})

From 4fa8512d576ca2a425e6cc546264934d959bced5 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 25 Oct 2023 09:47:21 +0200
Subject: [PATCH 43/57] Check event is not rejected  (#3243)

Companion PR to https://github.com/matrix-org/gomatrixserverlib/pull/421
---
 cmd/resolve-state/main.go                  | 13 +++++++++++++
 go.mod                                     |  2 +-
 go.sum                                     |  4 ++--
 roomserver/internal/input/input_missing.go |  7 +++++++
 roomserver/internal/query/query.go         | 14 ++++++++++++++
 roomserver/state/state.go                  |  8 ++++++++
 roomserver/storage/shared/room_updater.go  |  4 ++++
 7 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index 5be449097..d6db72436 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -202,12 +202,25 @@ func main() {
 		authEvents[i] = authEventEntries[i].PDU
 	}
 
+	// Get the roomNID
+	roomInfo, err = roomserverDB.RoomInfo(ctx, authEvents[0].RoomID().String())
+	if err != nil {
+		panic(err)
+	}
+
 	fmt.Println("Resolving state")
 	var resolved Events
 	resolved, err = gomatrixserverlib.ResolveConflicts(
 		gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		},
+		func(eventID string) bool {
+			isRejected, rejectedErr := roomserverDB.IsEventRejected(ctx, roomInfo.RoomNID, eventID)
+			if rejectedErr != nil {
+				return true
+			}
+			return isRejected
+		},
 	)
 	if err != nil {
 		panic(err)
diff --git a/go.mod b/go.mod
index e8fbbb81e..de60d9b4e 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
diff --git a/go.sum b/go.sum
index 102ff3aa0..955cdc11c 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be h1:bZP16ydP8uRoRBo1p/7WHMexjg7JJGj81fKzZ1FULb4=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231023121512-16e7431168be/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca h1:JCP72vU4Vcmur2071RwYVOSoekR+ZjbC03wZD5lAAK0=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index d9ab291e9..21493287e 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -498,6 +498,13 @@ func (t *missingStateReq) resolveStatesAndCheck(ctx context.Context, roomVersion
 		roomVersion, gomatrixserverlib.ToPDUs(stateEventList), gomatrixserverlib.ToPDUs(authEventList), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID)
 		},
+		func(eventID string) bool {
+			isRejected, err := t.db.IsEventRejected(ctx, t.roomInfo.RoomNID, eventID)
+			if err != nil {
+				return true
+			}
+			return isRejected
+		},
 	)
 	if err != nil {
 		return nil, err
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index f87a3f7ed..74b010281 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -165,6 +165,13 @@ func (r *Queryer) QueryStateAfterEvents(
 			info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 				return r.QueryUserIDForSender(ctx, roomID, senderID)
 			},
+			func(eventID string) bool {
+				isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID)
+				if rejectedErr != nil {
+					return true
+				}
+				return isRejected
+			},
 		)
 		if err != nil {
 			return fmt.Errorf("state.ResolveConflictsAdhoc: %w", err)
@@ -676,6 +683,13 @@ func (r *Queryer) QueryStateAndAuthChain(
 			info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 				return r.QueryUserIDForSender(ctx, roomID, senderID)
 			},
+			func(eventID string) bool {
+				isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID)
+				if rejectedErr != nil {
+					return true
+				}
+				return isRejected
+			},
 		)
 		if err != nil {
 			return err
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index 1e776ff6c..dfd439a22 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -45,6 +45,7 @@ type StateResolutionStorage interface {
 	AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error)
 	Events(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, eventNIDs []types.EventNID) ([]types.Event, error)
 	EventsFromIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]types.Event, error)
+	IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error)
 }
 
 type StateResolution struct {
@@ -1066,6 +1067,13 @@ func (v *StateResolution) resolveConflictsV2(
 			func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 				return v.Querier.QueryUserIDForSender(ctx, roomID, senderID)
 			},
+			func(eventID string) bool {
+				isRejected, err := v.db.IsEventRejected(ctx, v.roomInfo.RoomNID, eventID)
+				if err != nil {
+					return true
+				}
+				return isRejected
+			},
 		)
 	}()
 
diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go
index 70672a33e..06284d2e3 100644
--- a/roomserver/storage/shared/room_updater.go
+++ b/roomserver/storage/shared/room_updater.go
@@ -250,3 +250,7 @@ func (u *RoomUpdater) MarkEventAsSent(eventNID types.EventNID) error {
 func (u *RoomUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (*MembershipUpdater, error) {
 	return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomInfo.RoomNID, targetUserNID, targetLocal)
 }
+
+func (u *RoomUpdater) IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) {
+	return u.d.IsEventRejected(ctx, roomNID, eventID)
+}

From e02a7948d8556398ceb345a241c175b5ca1d011f Mon Sep 17 00:00:00 2001
From: WrenIX <133280015+wrenix@users.noreply.github.com>
Date: Wed, 25 Oct 2023 10:08:54 +0200
Subject: [PATCH 44/57] fix(helm): empty storage class in pvcs (#3191)

fix #3103

---

not yet tested

[skip ci]
---
 helm/dendrite/templates/pvc.yaml | 27 ++++++++++++++++++++++++---
 helm/dendrite/values.yaml        | 32 ++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml
index 88eff3bed..70b1ce563 100644
--- a/helm/dendrite/templates/pvc.yaml
+++ b/helm/dendrite/templates/pvc.yaml
@@ -12,7 +12,14 @@ spec:
   resources:
     requests:
       storage: {{ .Values.persistence.media.capacity }}
-  storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.media.storageClass }}
+  {{ $storageClass := .Values.persistence.media.storageClass | default .Values.persistence.storageClass }}
+  {{- if $storageClass }}
+  {{- if (eq "-" $storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ $storageClass }}"
+  {{- end }}
+  {{- end }}
 {{ end }}
 {{ if not .Values.persistence.jetstream.existingClaim }}
 ---
@@ -28,7 +35,14 @@ spec:
   resources:
     requests:
       storage: {{ .Values.persistence.jetstream.capacity }}
-  storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.jetstream.storageClass }}
+  {{ $storageClass := .Values.persistence.jetstream.storageClass | default .Values.persistence.storageClass }}
+  {{- if $storageClass }}
+  {{- if (eq "-" $storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ $storageClass }}"
+  {{- end }}
+  {{- end }}
 {{ end }}
 {{ if not .Values.persistence.search.existingClaim }}
 ---
@@ -44,5 +58,12 @@ spec:
   resources:
     requests:
       storage: {{ .Values.persistence.search.capacity }}
-  storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.search.storageClass }}
+  {{ $storageClass := .Values.persistence.search.storageClass | default .Values.persistence.storageClass }}
+  {{- if $storageClass }}
+  {{- if (eq "-" $storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ $storageClass }}"
+  {{- end }}
+  {{- end }}
 {{ end }}
diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml
index 8a72f6693..afce1d930 100644
--- a/helm/dendrite/values.yaml
+++ b/helm/dendrite/values.yaml
@@ -26,7 +26,13 @@ persistence:
   # -- The storage class to use for volume claims.
   # Used unless specified at the specific component.
   # Defaults to the cluster default storage class.
-  storageClass: ""
+  ## If defined, storageClassName: <storageClass>
+  ## If set to "-", storageClassName: "", which disables dynamic provisioning
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+  ##   GKE, AWS & OpenStack)
+  ##
+  # storageClass: ""
   jetstream:
     # -- Use an existing volume claim for jetstream
     existingClaim: ""
@@ -34,7 +40,13 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    storageClass: ""
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: ""
   media:
     # -- Use an existing volume claim for media files
     existingClaim: ""
@@ -42,7 +54,13 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    storageClass: ""
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: ""
   search:
     # -- Use an existing volume claim for the fulltext search index
     existingClaim: ""
@@ -50,7 +68,13 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    storageClass: ""
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    # storageClass: ""
 
 # -- Add additional volumes to the Dendrite Pod
 extraVolumes: []

From a0375d41fbbdabd98df743d2e7fa77b4d0c44d4b Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Wed, 25 Oct 2023 08:13:18 +0000
Subject: [PATCH 45/57] Add simple test for one time keys (#3239)

---
 userapi/storage/storage_test.go | 51 +++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index a46ee9ebb..5a789dfd7 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -1,6 +1,7 @@
 package storage_test
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -758,3 +759,53 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) {
 		}
 	})
 }
+
+func TestOneTimeKeys(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, clean := mustCreateKeyDatabase(t, dbType)
+		defer clean()
+		userID := "@alice:localhost"
+		deviceID := "alice_device"
+		otk := api.OneTimeKeys{
+			UserID:   userID,
+			DeviceID: deviceID,
+			KeyJSON:  map[string]json.RawMessage{"curve25519:KEY1": []byte(`{"key":"v1"}`)},
+		}
+
+		// Add a one time key to the DB
+		_, err := db.StoreOneTimeKeys(ctx, otk)
+		MustNotError(t, err)
+
+		// Check the count of one time keys is correct
+		count, err := db.OneTimeKeysCount(ctx, userID, deviceID)
+		MustNotError(t, err)
+		if count.KeyCount["curve25519"] != 1 {
+			t.Fatalf("Expected 1 key, got %d", count.KeyCount["curve25519"])
+		}
+
+		// Check the actual key contents are correct
+		keysJSON, err := db.ExistingOneTimeKeys(ctx, userID, deviceID, []string{"curve25519:KEY1"})
+		MustNotError(t, err)
+		keyJSON, err := keysJSON["curve25519:KEY1"].MarshalJSON()
+		MustNotError(t, err)
+		if !bytes.Equal(keyJSON, []byte(`{"key":"v1"}`)) {
+			t.Fatalf("Existing keys do not match expected. Got %v", keysJSON["curve25519:KEY1"])
+		}
+
+		// Claim a one time key from the database. This should remove it from the database.
+		claimedKeys, err := db.ClaimKeys(ctx, map[string]map[string]string{userID: {deviceID: "curve25519"}})
+		MustNotError(t, err)
+
+		// Check the claimed key contents are correct
+		if !reflect.DeepEqual(claimedKeys[0], otk) {
+			t.Fatalf("Expected to claim stored key %v. Got %v", otk, claimedKeys[0])
+		}
+
+		// Check the count of one time keys is now zero
+		count, err = db.OneTimeKeysCount(ctx, userID, deviceID)
+		MustNotError(t, err)
+		if count.KeyCount["curve25519"] != 0 {
+			t.Fatalf("Expected 0 keys, got %d", count.KeyCount["curve25519"])
+		}
+	})
+}

From 89482ad7902d6cacddc29f8eac62ab66ff4ebf24 Mon Sep 17 00:00:00 2001
From: CicadaCinema <52425971+CicadaCinema@users.noreply.github.com>
Date: Wed, 25 Oct 2023 09:24:06 +0100
Subject: [PATCH 46/57] clean up dead links, fix typo (#3130)

I fixed any dead links beginning https://matrix.org/speculator and some
issues I found along the way.


https://web.archive.org/web/20190329152312/https://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
is now found at

https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api


https://web.archive.org/web/20170620093435/https://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
is now found at

https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register


https://github.com/matrix-org/matrix-spec/blob/2a8d64fef7a40717ef9f5748ee0551b2117be037/specification/intro.rst?plain=1#L443
is now found at
https://spec.matrix.org/v1.7/appendices/#user-identifiers
---
 clientapi/routing/register.go    |  6 +++---
 docs/development/CONTRIBUTING.md |  2 +-
 internal/validate.go             |  2 +-
 setup/config/config_clientapi.go | 20 +++++++++-----------
 4 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go
index 565c41533..090a2fc20 100644
--- a/clientapi/routing/register.go
+++ b/clientapi/routing/register.go
@@ -236,7 +236,7 @@ type authDict struct {
 	// TODO: Lots of custom keys depending on the type
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
+// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api
 type userInteractiveResponse struct {
 	Flows     []authtypes.Flow       `json:"flows"`
 	Completed []authtypes.LoginType  `json:"completed"`
@@ -256,7 +256,7 @@ func newUserInteractiveResponse(
 	}
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 type registerResponse struct {
 	UserID      string `json:"user_id"`
 	AccessToken string `json:"access_token,omitempty"`
@@ -462,7 +462,7 @@ func validateApplicationService(
 }
 
 // Register processes a /register request.
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 func Register(
 	req *http.Request,
 	userAPI userapi.ClientUserAPI,
diff --git a/docs/development/CONTRIBUTING.md b/docs/development/CONTRIBUTING.md
index 71e7516a2..caab1e749 100644
--- a/docs/development/CONTRIBUTING.md
+++ b/docs/development/CONTRIBUTING.md
@@ -109,7 +109,7 @@ To configure the connection to a remote Postgres, you can use the following envi
 
 ```bash
 POSTGRES_USER=postgres
-POSTGERS_PASSWORD=yourPostgresPassword
+POSTGRES_PASSWORD=yourPostgresPassword
 POSTGRES_HOST=localhost
 POSTGRES_DB=postgres # the superuser database to use
 ```
diff --git a/internal/validate.go b/internal/validate.go
index 99088f240..7f0d8b9e6 100644
--- a/internal/validate.go
+++ b/internal/validate.go
@@ -25,7 +25,7 @@ import (
 )
 
 const (
-	maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain
+	maxUsernameLength = 254 // https://spec.matrix.org/v1.7/appendices/#user-identifiers TODO account for domain
 
 	minPasswordLength = 8   // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
 	maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go
index 44136e2a0..85dfe0beb 100644
--- a/setup/config/config_clientapi.go
+++ b/setup/config/config_clientapi.go
@@ -93,17 +93,15 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors) {
 		checkNotEmpty(configErrs, "client_api.recaptcha_sitekey_class", c.RecaptchaSitekeyClass)
 	}
 	// Ensure there is any spam counter measure when enabling registration
-	if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled {
-		if !c.RecaptchaEnabled {
-			configErrs.Add(
-				"You have tried to enable open registration without any secondary verification methods " +
-					"(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " +
-					"increasing the risk that your server will be used to send spam or abuse, and may result in " +
-					"your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " +
-					"start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " +
-					"should set the registration_disabled option in your Dendrite config.",
-			)
-		}
+	if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled && !c.RecaptchaEnabled {
+		configErrs.Add(
+			"You have tried to enable open registration without any secondary verification methods " +
+				"(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " +
+				"increasing the risk that your server will be used to send spam or abuse, and may result in " +
+				"your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " +
+				"start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " +
+				"should set the registration_disabled option in your Dendrite config.",
+		)
 	}
 }
 

From 317b1018a38fbbc573f095b880beba5b1fab60be Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 25 Oct 2023 13:53:40 +0200
Subject: [PATCH 47/57] Version 0.13.4 (#3244)

If I didn't mess up the workflow, this should remove some ugliness from
the version string (e.g. 0.13.2+57ddbe0.57ddbe0, dupe commit hash, as a
result of https://github.com/matrix-org/dendrite/pull/3147)
---
 .github/workflows/docker.yml           | 18 ------------------
 CHANGES.md                             | 18 ++++++++++++++++++
 Dockerfile                             |  5 ++---
 build/docker/Dockerfile.demo-yggdrasil |  2 +-
 helm/dendrite/Chart.yaml               |  4 ++--
 helm/dendrite/README.md                |  7 ++-----
 internal/version.go                    |  2 +-
 7 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 8448d8e23..8d3a8d674 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -32,10 +32,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -60,7 +56,6 @@ jobs:
           cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache
           cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -75,7 +70,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -109,10 +103,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -137,7 +127,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -153,7 +142,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -176,10 +164,6 @@ jobs:
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -204,7 +188,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -220,7 +203,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true
diff --git a/CHANGES.md b/CHANGES.md
index bdb6a796e..57e3a3d4f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,23 @@
 # Changelog
 
+## Dendrite 0.13.4 (2023-10-25)
+
+Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution
+algorithm.
+
+### Fixes:
+
+- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also 
+  avoids spamming the logs when shutting down.)
+- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset"
+  out of rooms and other missing state events due to calculating the wrong state.
+- A bug when setting notifications from Element Android has been fixed by implementing MSC3987
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.19 to v2.9.23
+
 ## Dendrite 0.13.3 (2023-09-28)
 
 ### Fixes:
diff --git a/Dockerfile b/Dockerfile
index 4ee20933a..8c8f1588f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,7 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base
 RUN apk --update --no-cache add bash build-base curl git
 
 #
@@ -13,7 +13,6 @@ FROM --platform=${BUILDPLATFORM} base AS build
 WORKDIR /src
 ARG TARGETOS
 ARG TARGETARCH
-ARG FLAGS
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
@@ -21,7 +20,7 @@ RUN --mount=target=. \
     GOARCH="$TARGETARCH" \
     GOOS="linux" \
     CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
-    go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
+    go build -v -trimpath -o /out/ ./cmd/...
 
 
 #
diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil
index efae5496c..b9e387666 100644
--- a/build/docker/Dockerfile.demo-yggdrasil
+++ b/build/docker/Dockerfile.demo-yggdrasil
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.19-alpine AS base
+FROM docker.io/golang:1.21-alpine AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,
diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index 674152051..32f479960 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.4"
-appVersion: "0.13.3"
+version: "0.13.5"
+appVersion: "0.13.4"
 description: Dendrite Matrix Homeserver
 type: application
 keywords:
diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index cb78a54f2..22daa1813 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.4](https://img.shields.io/badge/Version-0.13.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.3](https://img.shields.io/badge/AppVersion-0.13.3-informational?style=flat-square)
+![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.4](https://img.shields.io/badge/AppVersion-0.13.4-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -48,16 +48,13 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | signing_key.create | bool | `true` | Create a new signing key, if not exists |
 | signing_key.existingSecret | string | `""` | Use an existing secret |
 | resources | object | sets some sane default values | Default resource requests/limits. |
-| persistence.storageClass | string | `""` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. |
+| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: <storageClass> # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is #   set, choosing the default provisioner.  (gp2 on AWS, standard on #   GKE, AWS & OpenStack) # storageClass: "" |
 | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream |
 | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume |
-| persistence.jetstream.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass |
 | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files |
 | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume |
-| persistence.media.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass |
 | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index |
 | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume |
-| persistence.search.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass |
 | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod |
 | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod |
 | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
diff --git a/internal/version.go b/internal/version.go
index 1f8a62bc2..55726ddcb 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -18,7 +18,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 13
-	VersionPatch = 3
+	VersionPatch = 4
 	VersionTag   = "" // example: "rc1"
 
 	gitRevLen = 7 // 7 matches the displayed characters on github.com

From 32f7c4b166c5e74ef34973a1d6a30e5e2d75c3ed Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 31 Oct 2023 07:59:19 +0100
Subject: [PATCH 48/57] Bump github.com/docker/docker from 24.0.5+incompatible
 to 24.0.7+incompatible (#3250)

Bumps [github.com/docker/docker](https://github.com/docker/docker) from
24.0.5+incompatible to 24.0.7+incompatible.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/docker/docker/releases">github.com/docker/docker's
releases</a>.</em></p>
<blockquote>
<h2>v24.0.7</h2>
<h2>24.0.7</h2>
<p>For a full list of pull requests and changes in this release, refer
to the relevant GitHub milestones:</p>
<ul>
<li><a
href="https://github.com/docker/cli/issues?q=is%3Aclosed+milestone%3A24.0.7">docker/cli,
24.0.7 milestone</a></li>
<li><a
href="https://github.com/moby/moby/issues?q=is%3Aclosed+milestone%3A24.0.7">moby/moby,
24.0.7 milestone</a></li>
</ul>
<h3>Bug fixes and enhancements</h3>
<ul>
<li>Write overlay2 layer metadata atomically. <a
href="https://redirect.github.com/moby/moby/pull/46703">moby/moby#46703</a></li>
<li>Fix &quot;Rootful-in-Rootless&quot; Docker-in-Docker on systemd
version 250 and later. <a
href="https://redirect.github.com/moby/moby/pull/46626">moby/moby#46626</a></li>
<li>Fix <code>dockerd-rootless-setuptools.sh</code> when username
contains a backslash. <a
href="https://redirect.github.com/moby/moby/pull/46407">moby/moby#46407</a></li>
<li>Fix a bug that would prevent network sandboxes to be fully deleted
when stopping containers with no network attachments and when
<code>dockerd --bridge=none</code> is used. <a
href="https://redirect.github.com/moby/moby/pull/46702">moby/moby#46702</a></li>
<li>Fix a bug where cancelling an API request could interrupt container
restart. <a
href="https://redirect.github.com/moby/moby/pull/46697">moby/moby#46697</a></li>
<li>Fix an issue where containers would fail to start when providing
<code>--ip-range</code> with a range larger than the subnet. <a
href="https://redirect.github.com/docker/for-mac/issues/6870">docker/for-mac#6870</a></li>
<li>Fix data corruption with zstd output. <a
href="https://redirect.github.com/moby/moby/pull/46709">moby/moby#46709</a></li>
<li>Fix the conditions under which the container's MAC address is
applied. <a
href="https://redirect.github.com/moby/moby/pull/46478">moby/moby#46478</a></li>
<li>Improve the performance of the stats collector. <a
href="https://redirect.github.com/moby/moby/pull/46448">moby/moby#46448</a></li>
<li>Fix an issue with source policy rules ending up in the wrong order.
<a
href="https://redirect.github.com/moby/moby/pull/46441">moby/moby#46441</a></li>
</ul>
<h3>Packaging updates</h3>
<ul>
<li>Add support for Fedora 39 and Ubuntu 23.10. <a
href="https://redirect.github.com/docker/docker-ce-packaging/pull/940">docker/docker-ce-packaging#940</a>,
<a
href="https://redirect.github.com/docker/docker-ce-packaging/pull/955">docker/docker-ce-packaging#955</a></li>
<li>Fix <code>docker.socket</code> not getting disabled when
uninstalling the <code>docker-ce</code> RPM package. <a
href="https://redirect.github.com/docker/docker-ce-packaging/pull/852">docker/docker-ce-packaging#852</a></li>
<li>Upgrade Go to <code>go1.20.10</code>. <a
href="https://redirect.github.com/docker/docker-ce-packaging/pull/951">docker/docker-ce-packaging#951</a></li>
<li>Upgrade containerd to <code>v1.7.6</code> (static binaries only). <a
href="https://redirect.github.com/moby/moby/pull/46103">moby/moby#46103</a></li>
<li>Upgrade the <code>containerd.io</code> package to <a
href="https://github.com/containerd/containerd/releases/tag/v1.6.24"><code>v1.6.24</code></a>.</li>
</ul>
<h3>Security</h3>
<ul>
<li>Deny containers access to <code>/sys/devices/virtual/powercap</code>
by default. This change hardens against <a
href="https://scout.docker.com/v/CVE-2020-8694">CVE-2020-8694</a>, <a
href="https://scout.docker.com/v/CVE-2020-8695">CVE-2020-8695</a>, and
<a href="https://scout.docker.com/v/CVE-2020-12912">CVE-2020-12912</a>,
and an attack known as <a href="https://platypusattack.com/">the
PLATYPUS attack</a>. For more details, see <a
href="https://github.com/moby/moby/security/advisories/GHSA-jq35-85cj-fj4p">advisory</a>,
<a
href="https://github.com/moby/moby/commit/c9ccbfad11a60e703e91b6cca4f48927828c7e35">commit</a>.</li>
</ul>
<h2>v24.0.6</h2>
<h2>24.0.6</h2>
<p>For a full list of pull requests and changes in this release, refer
to the relevant GitHub milestones:</p>
<ul>
<li><a
href="https://github.com/docker/cli/issues?q=is%3Aclosed+milestone%3A24.0.6">docker/cli,
24.0.6 milestone</a></li>
<li><a
href="https://github.com/moby/moby/issues?q=is%3Aclosed+milestone%3A24.0.6">moby/moby,
24.0.6 milestone</a></li>
</ul>
<h3>Bug fixes and enhancements</h3>
<ul>
<li>containerd storage backend: Fix <code>docker ps</code> failing when
a container image is no longer present in the content store. <a
href="https://redirect.github.com/moby/moby/pull/46095">moby/moby#46095</a></li>
<li>containerd storage backend: Fix <code>docker ps -s -a</code> and
<code>docker container prune</code> failing when a container image
config is no longer present in the content store. <a
href="https://redirect.github.com/moby/moby/pull/46097">moby/moby#46097</a></li>
<li>containerd storage backend: Fix <code>docker inspect</code> failing
when a container image config is no longer (or was never) present in the
content store. <a
href="https://redirect.github.com/moby/moby/pull/46244">moby/moby#46244</a></li>
<li>containerd storage backend: Fix diff and export with the
<code>overlayfs</code> snapshotter by using reference-counted rootfs
mounts. <a
href="https://redirect.github.com/moby/moby/pull/46266">moby/moby#46266</a></li>
<li>containerd storage backend: Fix a misleading error message when the
image platforms available locally do not match the desired platform. <a
href="https://redirect.github.com/moby/moby/pull/46300">moby/moby#46300</a></li>
<li>containerd storage backend: Fix the <code>FROM scratch</code>
Dockerfile instruction with the classic builder. <a
href="https://redirect.github.com/moby/moby/pull/46302">moby/moby#46302</a></li>
<li>containerd storage backend: Fix <code>mismatched image rootfs and
manifest layers</code> errors with the classic builder. <a
href="https://redirect.github.com/moby/moby/pull/46310">moby/moby#46310</a></li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/moby/moby/commit/311b9ff0aa93aa55880e1e5f8871c4fb69583426"><code>311b9ff</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/46697">#46697</a>
from thaJeztah/24.0_backport_restart_nocancel</li>
<li><a
href="https://github.com/moby/moby/commit/af608045eef0b87f31a24d21fb7af80de76134aa"><code>af60804</code></a>
Merge pull request from GHSA-jq35-85cj-fj4p</li>
<li><a
href="https://github.com/moby/moby/commit/3cf363e1ee33fe00dbedfdb7d6caf299990d5568"><code>3cf363e</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/46709">#46709</a>
from thaJeztah/24.0_backport_bump_compress</li>
<li><a
href="https://github.com/moby/moby/commit/05d7386665793b7f8398eb80b4e85adff5486035"><code>05d7386</code></a>
daemon: daemon.containerRestart: don't cancel restart on context
cancel</li>
<li><a
href="https://github.com/moby/moby/commit/649c9440f28c7334ee5c9f17889448a81dcc8729"><code>649c944</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/46703">#46703</a>
from thaJeztah/24.0_backport_atomic-layer-data-write</li>
<li><a
href="https://github.com/moby/moby/commit/9b20b1a5fe0919a79cc15f6a3f331f2cdae0a37a"><code>9b20b1a</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/46702">#46702</a>
from thaJeztah/24.0_backport_releaseNetwork_Network...</li>
<li><a
href="https://github.com/moby/moby/commit/dd37b0b960ec4d3da0ca2efe78fa47484d4c6380"><code>dd37b0b</code></a>
vendor: github.com/klauspost/compress v1.17.2</li>
<li><a
href="https://github.com/moby/moby/commit/7058c0d24da8ac9267e52224b6a3beaa24ce5e9f"><code>7058c0d</code></a>
vendor: github.com/klauspost/compress v1.16.5</li>
<li><a
href="https://github.com/moby/moby/commit/57bd38858262922b86ceea37770536ff535fa2af"><code>57bd388</code></a>
daemon: overlay2: Write layer metadata atomically</li>
<li><a
href="https://github.com/moby/moby/commit/05d95fd5038a8a56ff69294a3bdd33b2d2769ba3"><code>05d95fd</code></a>
daemon: release sandbox even when NetworkDisabled</li>
<li>Additional commits viewable in <a
href="https://github.com/docker/docker/compare/v24.0.5...v24.0.7">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/docker/docker&package-manager=go_modules&previous-version=24.0.5+incompatible&new-version=24.0.7+incompatible)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index de60d9b4e..9e7d278fe 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/blevesearch/bleve/v2 v2.3.8
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1
-	github.com/docker/docker v24.0.5+incompatible
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
 	github.com/gologme/log v1.3.0
diff --git a/go.sum b/go.sum
index 955cdc11c..e9da63f12 100644
--- a/go.sum
+++ b/go.sum
@@ -89,8 +89,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY=
-github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=

From da7bca0224760a4fe0e10876a9c11da333513a29 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 31 Oct 2023 16:39:45 +0100
Subject: [PATCH 49/57] Some tweaks for the device list updater (#3251)

This makes the following changes:
- Adds two new metrics observing the usage of the `DeviceListUpdater`
workers
- Makes the number of workers configurable
- Adds a 30s timeout for DB requests when receiving a device list update
over federation
---
 appservice/appservice_test.go                 |  6 ++--
 build/dendritejs-pinecone/main.go             |  3 +-
 build/gobind-yggdrasil/monolith.go            |  2 +-
 clientapi/admin_test.go                       | 20 +++++------
 clientapi/clientapi_test.go                   | 26 +++++++--------
 clientapi/routing/joinroom_test.go            |  2 +-
 clientapi/routing/login_test.go               |  2 +-
 clientapi/routing/register_test.go            |  6 ++--
 .../monolith/monolith.go                      |  2 +-
 cmd/dendrite-demo-yggdrasil/main.go           |  2 +-
 cmd/dendrite/main.go                          |  2 +-
 dendrite-sample.yaml                          |  4 +++
 roomserver/roomserver_test.go                 |  6 ++--
 setup/config/config_userapi.go                |  5 +++
 userapi/consumers/devicelistupdate.go         |  6 +++-
 userapi/internal/device_list_update.go        | 33 +++++++++++++++++--
 userapi/internal/device_list_update_test.go   |  9 ++---
 userapi/userapi.go                            |  3 +-
 18 files changed, 92 insertions(+), 47 deletions(-)

diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go
index ddc24477b..bbdeb47df 100644
--- a/appservice/appservice_test.go
+++ b/appservice/appservice_test.go
@@ -144,7 +144,7 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 		runCases(t, asAPI)
@@ -239,7 +239,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
 	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
-	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 	t.Run("UserIDExists", func(t *testing.T) {
@@ -378,7 +378,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		// Create required internal APIs
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics)
 		// start the consumer
 		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
 
diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go
index 61baed902..d3d5f59fe 100644
--- a/build/dendritejs-pinecone/main.go
+++ b/build/dendritejs-pinecone/main.go
@@ -38,6 +38,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -190,7 +191,7 @@ func startup() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics)
 
 	asQuery := appservice.NewInternalAPI(
 		processCtx, cfg, &natsInstance, userAPI, rsAPI,
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 720ce37eb..791ad261b 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() {
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go
index 66667b03c..b228dd93c 100644
--- a/clientapi/admin_test.go
+++ b/clientapi/admin_test.go
@@ -45,7 +45,7 @@ func TestAdminCreateToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -196,7 +196,7 @@ func TestAdminListRegistrationTokens(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -314,7 +314,7 @@ func TestAdminGetRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -415,7 +415,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -509,7 +509,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -693,7 +693,7 @@ func TestAdminResetPassword(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for changing the password/login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -791,7 +791,7 @@ func TestPurgeRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
 
 		// Create the room
@@ -863,7 +863,7 @@ func TestAdminEvacuateRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
@@ -964,7 +964,7 @@ func TestAdminEvacuateUser(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
@@ -1055,7 +1055,7 @@ func TestAdminMarkAsStale(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index f2d617cb9..2bb15fba3 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -121,7 +121,7 @@ func TestGetPutDevices(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -170,7 +170,7 @@ func TestDeleteDevice(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -275,7 +275,7 @@ func TestDeleteDevices(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -442,7 +442,7 @@ func TestSetDisplayname(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -554,7 +554,7 @@ func TestSetAvatarURL(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -632,7 +632,7 @@ func TestTyping(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -716,7 +716,7 @@ func TestMembership(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		rsAPI.SetUserAPI(userAPI)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -955,7 +955,7 @@ func TestCapabilities(t *testing.T) {
 		// Needed to create accounts
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1002,7 +1002,7 @@ func TestTurnserver(t *testing.T) {
 	// Needed to create accounts
 	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 	//rsAPI.SetUserAPI(userAPI)
 	// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 	AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1100,7 +1100,7 @@ func Test3PID(t *testing.T) {
 		// Needed to create accounts
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1276,7 +1276,7 @@ func TestPushRules(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1663,7 +1663,7 @@ func TestKeys(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -2125,7 +2125,7 @@ func TestKeyBackup(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go
index 933ea8d3a..be3fb429f 100644
--- a/clientapi/routing/joinroom_test.go
+++ b/clientapi/routing/joinroom_test.go
@@ -36,7 +36,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 		// Create the users in the userapi
diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go
index 252017db2..4c4fc3533 100644
--- a/clientapi/routing/login_test.go
+++ b/clientapi/routing/login_test.go
@@ -49,7 +49,7 @@ func TestLogin(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for /login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go
index 0a1986cf7..69b29e9c2 100644
--- a/clientapi/routing/register_test.go
+++ b/clientapi/routing/register_test.go
@@ -416,7 +416,7 @@ func Test_register(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -596,7 +596,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
@@ -637,7 +637,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go
index 41af568a6..5222d0b8e 100644
--- a/cmd/dendrite-demo-pinecone/monolith/monolith.go
+++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go
@@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite(
 	)
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 3ec550113..b07687538 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -213,7 +213,7 @@ func main() {
 	natsInstance := jetstream.NATSInstance{}
 	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go
index f3140b4e2..f38263c6c 100644
--- a/cmd/dendrite/main.go
+++ b/cmd/dendrite/main.go
@@ -162,7 +162,7 @@ func main() {
 	// dependency. Other components also need updating after their dependencies are up.
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics)
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 	rsAPI.SetAppserviceAPI(asAPI)
diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml
index 7affc2599..e143a7398 100644
--- a/dendrite-sample.yaml
+++ b/dendrite-sample.yaml
@@ -325,6 +325,10 @@ user_api:
   auto_join_rooms:
   #  - "#main:matrix.org"
 
+  # The number of workers to start for the DeviceListUpdater. Defaults to 8.
+  # This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely.
+  # worker_count: 8
+
 # Configuration for Opentracing.
 # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
 # how this works and how to set it up.
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index 90e67b699..22d27ba05 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -58,7 +58,7 @@ func TestUsers(t *testing.T) {
 		})
 
 		t.Run("kick users", func(t *testing.T) {
-			usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+			usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 			rsAPI.SetUserAPI(usrAPI)
 			testKickUsers(t, rsAPI, usrAPI)
 		})
@@ -258,7 +258,7 @@ func TestPurgeRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
 
 		// Create the room
@@ -1050,7 +1050,7 @@ func TestUpgrade(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
 		rsAPI.SetUserAPI(userAPI)
 
 		for _, tc := range testCases {
diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go
index e64a3910c..559de72ac 100644
--- a/setup/config/config_userapi.go
+++ b/setup/config/config_userapi.go
@@ -21,6 +21,10 @@ type UserAPI struct {
 	// Users who register on this homeserver will automatically
 	// be joined to the rooms listed under this option.
 	AutoJoinRooms []string `yaml:"auto_join_rooms"`
+
+	// The number of workers to start for the DeviceListUpdater. Defaults to 8.
+	// This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely.
+	WorkerCount int `yaml:"worker_count"`
 }
 
 const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
@@ -28,6 +32,7 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
 func (c *UserAPI) Defaults(opts DefaultOpts) {
 	c.BCryptCost = bcrypt.DefaultCost
 	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
+	c.WorkerCount = 8
 	if opts.Generate {
 		if !opts.SingleDatabase {
 			c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
diff --git a/userapi/consumers/devicelistupdate.go b/userapi/consumers/devicelistupdate.go
index 3389bb808..b3ccb573b 100644
--- a/userapi/consumers/devicelistupdate.go
+++ b/userapi/consumers/devicelistupdate.go
@@ -17,6 +17,7 @@ package consumers
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	"github.com/matrix-org/dendrite/userapi/internal"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -82,7 +83,10 @@ func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M
 		return true
 	}
 
-	err := t.updater.Update(ctx, m)
+	timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+
+	err := t.updater.Update(timeoutCtx, m)
 	if err != nil {
 		logrus.WithFields(logrus.Fields{
 			"user_id":   m.UserID,
diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go
index 2f33589fe..a4d28188a 100644
--- a/userapi/internal/device_list_update.go
+++ b/userapi/internal/device_list_update.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net"
+	"strconv"
 	"sync"
 	"time"
 
@@ -142,13 +143,36 @@ type KeyChangeProducer interface {
 	ProduceKeyChanges(keys []api.DeviceMessage) error
 }
 
+var deviceListUpdaterBackpressure = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: "dendrite",
+		Subsystem: "keyserver",
+		Name:      "worker_backpressure",
+		Help:      "How many device list updater requests are queued",
+	},
+	[]string{"worker_id"},
+)
+var deviceListUpdaterServersRetrying = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: "dendrite",
+		Subsystem: "keyserver",
+		Name:      "worker_servers_retrying",
+		Help:      "How many servers are queued for retry",
+	},
+	[]string{"worker_id"},
+)
+
 // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale.
 func NewDeviceListUpdater(
 	process *process.ProcessContext, db DeviceListUpdaterDatabase,
 	api DeviceListUpdaterAPI, producer KeyChangeProducer,
 	fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int,
 	rsAPI rsapi.KeyserverRoomserverAPI, thisServer spec.ServerName,
+	enableMetrics bool,
 ) *DeviceListUpdater {
+	if enableMetrics {
+		prometheus.MustRegister(deviceListUpdaterBackpressure, deviceListUpdaterServersRetrying)
+	}
 	return &DeviceListUpdater{
 		process:        process,
 		userIDToMutex:  make(map[string]*sync.Mutex),
@@ -173,7 +197,7 @@ func (u *DeviceListUpdater) Start() error {
 		// to stop (in this transaction) until key requests can be made.
 		ch := make(chan spec.ServerName, 10)
 		u.workerChans[i] = ch
-		go u.worker(ch)
+		go u.worker(ch, i)
 	}
 
 	staleLists, err := u.db.StaleDeviceLists(u.process.Context(), []spec.ServerName{})
@@ -343,6 +367,8 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) {
 	index := int(int64(hash.Sum32()) % int64(len(u.workerChans)))
 
 	ch := u.assignChannel(userID)
+	deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Inc()
+	defer deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Dec()
 	u.workerChans[index] <- remoteServer
 	select {
 	case <-ch:
@@ -372,7 +398,7 @@ func (u *DeviceListUpdater) clearChannel(userID string) {
 	}
 }
 
-func (u *DeviceListUpdater) worker(ch chan spec.ServerName) {
+func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) {
 	retries := make(map[spec.ServerName]time.Time)
 	retriesMu := &sync.Mutex{}
 	// restarter goroutine which will inject failed servers into ch when it is time
@@ -391,9 +417,12 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) {
 			for _, srv := range serversToRetry {
 				delete(retries, srv)
 			}
+			deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries)))
 			retriesMu.Unlock()
 			for _, srv := range serversToRetry {
+				deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Inc()
 				ch <- srv
+				deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec()
 			}
 		}
 	}()
diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go
index 38fd8b583..14a49bc54 100644
--- a/userapi/internal/device_list_update_test.go
+++ b/userapi/internal/device_list_update_test.go
@@ -27,6 +27,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
@@ -161,7 +162,7 @@ func TestUpdateHavePrevID(t *testing.T) {
 	}
 	ap := &mockDeviceListUpdaterAPI{}
 	producer := &mockKeyChangeProducer{}
-	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost")
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics)
 	event := gomatrixserverlib.DeviceListUpdateEvent{
 		DeviceDisplayName: "Foo Bar",
 		Deleted:           false,
@@ -233,7 +234,7 @@ func TestUpdateNoPrevID(t *testing.T) {
 			`)),
 		}, nil
 	})
-	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test")
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics)
 	if err := updater.Start(); err != nil {
 		t.Fatalf("failed to start updater: %s", err)
 	}
@@ -303,7 +304,7 @@ func TestDebounce(t *testing.T) {
 		close(incomingFedReq)
 		return <-fedCh, nil
 	})
-	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost")
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics)
 	if err := updater.Start(); err != nil {
 		t.Fatalf("failed to start updater: %s", err)
 	}
@@ -406,7 +407,7 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) {
 
 		updater := NewDeviceListUpdater(processCtx, db, nil,
 			nil, nil,
-			0, rsAPI, "test")
+			0, rsAPI, "test", caching.DisableMetrics)
 		if err := updater.CleanUp(); err != nil {
 			t.Error(err)
 		}
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 6b6dac884..34bf119a0 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -46,6 +46,7 @@ func NewInternalAPI(
 	natsInstance *jetstream.NATSInstance,
 	rsAPI rsapi.UserRoomserverAPI,
 	fedClient fedsenderapi.KeyserverFederationAPI,
+	enableMetrics bool,
 ) *internal.UserInternalAPI {
 	js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream)
 	appServices := dendriteCfg.Derived.ApplicationServices
@@ -99,7 +100,7 @@ func NewInternalAPI(
 		FedClient:            fedClient,
 	}
 
-	updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, dendriteCfg.Global.ServerName) // 8 workers TODO: configurable
+	updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics)
 	userAPI.Updater = updater
 	// Remove users which we don't share a room with anymore
 	if err := updater.CleanUp(); err != nil {

From 8b4043473c824b9f6233f19fa9cacf53dc5f1496 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 1 Nov 2023 12:00:59 +0100
Subject: [PATCH 50/57] Bump github.com/nats-io/nkeys from 0.4.4 to 0.4.6
 (#3252)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [github.com/nats-io/nkeys](https://github.com/nats-io/nkeys) from
0.4.4 to 0.4.6.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/nats-io/nkeys/releases">github.com/nats-io/nkeys's
releases</a>.</em></p>
<blockquote>
<h2>v0.4.5</h2>
<h2>What's Changed</h2>
<ul>
<li>[CI] bump staticcheck GHAction by <a
href="https://github.com/philpennock"><code>@​philpennock</code></a> in
<a
href="https://redirect.github.com/nats-io/nkeys/pull/49">nats-io/nkeys#49</a></li>
<li>[FIX] added windows binary by <a
href="https://github.com/aricart"><code>@​aricart</code></a> in <a
href="https://redirect.github.com/nats-io/nkeys/pull/51">nats-io/nkeys#51</a></li>
<li>[FIX] YAML Enginering: quote go-version string by <a
href="https://github.com/philpennock"><code>@​philpennock</code></a> in
<a
href="https://redirect.github.com/nats-io/nkeys/pull/53">nats-io/nkeys#53</a></li>
<li>[FEAT] Use readKeyFile to read both seed file and public key file by
<a href="https://github.com/nanjj"><code>@​nanjj</code></a> in <a
href="https://redirect.github.com/nats-io/nkeys/pull/54">nats-io/nkeys#54</a></li>
<li>[FEAT] Made <code>decode</code> a little fast by <a
href="https://github.com/nanjj"><code>@​nanjj</code></a> in <a
href="https://redirect.github.com/nats-io/nkeys/pull/55">nats-io/nkeys#55</a></li>
<li>[REPO] Add issue forms by <a
href="https://github.com/bruth"><code>@​bruth</code></a> in <a
href="https://redirect.github.com/nats-io/nkeys/pull/56">nats-io/nkeys#56</a></li>
<li>[FIX] added binaries to match nats-server by <a
href="https://github.com/aricart"><code>@​aricart</code></a> in <a
href="https://redirect.github.com/nats-io/nkeys/pull/58">nats-io/nkeys#58</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/nats-io/nkeys/compare/v0.4.4...v0.4.5">https://github.com/nats-io/nkeys/compare/v0.4.4...v0.4.5</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/nats-io/nkeys/commit/62e5d8c7c4af84283b6535bcbf1300ab25b45e2d"><code>62e5d8c</code></a>
Merge pull request <a
href="https://redirect.github.com/nats-io/nkeys/issues/60">#60</a> from
nats-io/0_4_6</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/f63761b84d5972c999c240c4326a13fac17f0249"><code>f63761b</code></a>
[BUMP] release version and dependencies</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/d2e442ebad85f339de307598dab4f461e0eb1603"><code>d2e442e</code></a>
Merge pull request <a
href="https://redirect.github.com/nats-io/nkeys/issues/59">#59</a> from
nats-io/empty</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/58fb9d69f42ea73fffad1d14e5914dc666f3daa1"><code>58fb9d6</code></a>
Make sure to use byte slice to receive proper copy, otherwise empty
public ke...</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/3e454c8ca12e8e8a15d4c058d380e1ec31399597"><code>3e454c8</code></a>
Merge pull request <a
href="https://redirect.github.com/nats-io/nkeys/issues/58">#58</a> from
nats-io/arch-bins</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/53c07776673181060a391c9c7571a59f9c9412ac"><code>53c0777</code></a>
bump go to 1.21.x</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/d9358349661a09625246f07ba1c740fc41b63335"><code>d935834</code></a>
bump version number</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/6b488b30789ff698e14eee0c6e3d1ab9c33638ec"><code>6b488b3</code></a>
[FIX] added binaries to match nats-server</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/9fb41511a902119995e7bc5df543d8091133be68"><code>9fb4151</code></a>
Merge pull request <a
href="https://redirect.github.com/nats-io/nkeys/issues/56">#56</a> from
nats-io/add-issue-forms</li>
<li><a
href="https://github.com/nats-io/nkeys/commit/4647ec0912596d816de574fdf641f94b576601e0"><code>4647ec0</code></a>
Fix issue config discussions link</li>
<li>Additional commits viewable in <a
href="https://github.com/nats-io/nkeys/compare/v0.4.4...v0.4.6">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/nats-io/nkeys&package-manager=go_modules&previous-version=0.4.4&new-version=0.4.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9e7d278fe..e4febed8a 100644
--- a/go.mod
+++ b/go.mod
@@ -105,7 +105,7 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/nats-io/jwt/v2 v2.5.0 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index e9da63f12..ff44d5a1f 100644
--- a/go.sum
+++ b/go.sum
@@ -248,8 +248,8 @@ github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ
 github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
 github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
 github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
-github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
-github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
+github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
+github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=

From 5c67eb99b3387ddbcf8646b1d6d3d9f2cf8ad03a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 3 Nov 2023 08:13:34 +0100
Subject: [PATCH 51/57] Bump golang.org/x/image from 0.5.0 to 0.10.0 (#3257)

Bumps [golang.org/x/image](https://github.com/golang/image) from 0.5.0
to 0.10.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/image/commit/cb227cd2c919b27c6206fe0c1041a8bcc677949d"><code>cb227cd</code></a>
tiff: limit work when decoding malicious images</li>
<li><a
href="https://github.com/golang/image/commit/a5392f068b20c5126e356d1987f3eb74fffe1af2"><code>a5392f0</code></a>
bmp: support to decode 8-bit format with up to 256 color palette</li>
<li><a
href="https://github.com/golang/image/commit/f9550b04a5344792f1e5e5f9fbe8f5e87423f19e"><code>f9550b0</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/image/commit/81c166c49c1d18a6e9a5f659b646eb300013ccd0"><code>81c166c</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/image/commit/ed5dba0ea28f9438e4dac0320f7d9bb2fddd9737"><code>ed5dba0</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/image/commit/08ca817286cef4a50486ff2dc212ec148ff956ae"><code>08ca817</code></a>
font: have Glyph return !ok for U+FFFD substitute</li>
<li><a
href="https://github.com/golang/image/commit/b6ac75bc5918c3a0a2200faa20aedebc76d5b349"><code>b6ac75b</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/image/commit/1b7441254c9a43adda43ffcf12d7add0f1df0191"><code>1b74412</code></a>
font/sfnt: set type for all NameID constants</li>
<li><a
href="https://github.com/golang/image/commit/f632f7f87ca2653b091bcaab6d048f5799b841c9"><code>f632f7f</code></a>
tiff, tiff/lzw, vector: use single space in comments</li>
<li>See full diff in <a
href="https://github.com/golang/image/compare/v0.5.0...v0.10.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/image&package-manager=go_modules&previous-version=0.5.0&new-version=0.10.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  2 +-
 go.sum | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index e4febed8a..8dad4ebf0 100644
--- a/go.mod
+++ b/go.mod
@@ -44,7 +44,7 @@ require (
 	go.uber.org/atomic v1.10.0
 	golang.org/x/crypto v0.14.0
 	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
-	golang.org/x/image v0.5.0
+	golang.org/x/image v0.10.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
 	golang.org/x/sync v0.3.0
 	golang.org/x/term v0.13.0
diff --git a/go.sum b/go.sum
index ff44d5a1f..b173f6b85 100644
--- a/go.sum
+++ b/go.sum
@@ -366,8 +366,8 @@ golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N0
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI=
-golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4=
+golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M=
+golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4=
 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY=
@@ -376,6 +376,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -386,6 +387,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -394,6 +396,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -418,10 +421,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -429,6 +434,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -445,6 +451,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
 golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From 5f872f4a828eef28c6332d3fb7e8368380261f27 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 8 Nov 2023 14:22:02 +0100
Subject: [PATCH 52/57] Fix panic in `QueryNextRoomHierarchyPage` (#3253)

Sentry reported the following panic:
```
time="2023-11-01T01:33:56.220583478Z" level=error msg="Request panicked!
goroutine 43763845 [running]:
runtime/debug.Stack()
	runtime/debug/stack.go:24 +0x5e
github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.MakeJSONAPI.Protect.func3.1()
	github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:98 +0x13e
panic({0x15b5540?, 0x2453560?})
	runtime/panic.go:914 +0x21f
github.com/matrix-org/dendrite/internal/httputil.MakeAuthAPI.func1.1()
	github.com/matrix-org/dendrite/internal/httputil/httpapi.go:91 +0x4a
panic({0x15b5540?, 0x2453560?})
	runtime/panic.go:914 +0x21f
github.com/matrix-org/dendrite/roomserver/internal/query.(*Queryer).QueryNextRoomHierarchyPage(0x413185?, {0x1a576e0, 0xc0436705a0}, {{{0xc01e5fd260, 0x1f}, {0xc01e5fd261, 0x12}, {0xc01e5fd274, 0xb}}, {0xc145cb5200, ...}, ...}, ...)
	github.com/matrix-org/dendrite/roomserver/internal/query/query_room_hierarchy.go:116 +0xbfe
github.com/matrix-org/dendrite/clientapi/routing.QueryRoomHierarchy(0xc0be13b200, 0xc144e65dd0, {0xc01e5fd260?, 0x6?}, {0x7faf140639c8, 0xc00059af20}, 0xc08adca000?)
	github.com/matrix-org/dendrite/clientapi/routing/room_hierarchy.go:141 +0x68b
github.com/matrix-org/dendrite/clientapi/routing.Setup.func35(0xc03e7d5c20?, 0x17c3a57?)
	github.com/matrix-org/dendrite/clientapi/routing/routing.go:534 +0xbe
github.com/matrix-org/dendrite/internal/httputil.MakeAuthAPI.func1(0xc0bd097300)
	github.com/matrix-org/dendrite/internal/httputil/httpapi.go:108 +0x5ed
github.com/matrix-org/util.(*jsonRequestHandlerWrapper).OnIncomingRequest(0xc0bd097200?, 0xc13b7d6fc0?)
	github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:79 +0x19
github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.MakeJSONAPI.func2({0x1a54880, 0xc138f28b60}, 0xc0bd097200?)
	github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:141 +0xaa
github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.MakeJSONAPI.Protect.func3({0x1a54880?, 0xc138f28b60?}, 0x17c01d9?)
	github.com/matrix-org/util@v0.0.0-20221111132719-399730281e66/json.go:103 +0x63
net/http.HandlerFunc.ServeHTTP(...)
	net/http/server.go:2136
github.com/matrix-org/dendrite/internal/httputil.MakeExternalAPI.func1({0x1a54880?, 0xc138f28b60?}, 0xc0bd097100)
	github.com/matrix-org/dendrite/internal/httputil/httpapi.go:191 +0x411
net/http.HandlerFunc.ServeHTTP(0xc0bd097000?, {0x1a54880?, 0xc138f28b60?}, 0xbe1348905308878e?)
	net/http/server.go:2136 +0x29
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000000000, {0x1a54880, 0xc138f28b60}, 0xc0bd096f00)
	github.com/gorilla/mux@v1.8.0/mux.go:210 +0x1c5
github.com/matrix-org/dendrite/setup/base.SetupAndServeHTTP.(*Handler).Handle.(*Handler).handle.func5({0x1a54880, 0xc138f28b60}, 0xc0bd096e00)
	github.com/getsentry/sentry-go@v0.14.0/http/sentryhttp.go:103 +0x298
net/http.HandlerFunc.ServeHTTP(0xc0bd096a00?, {0x1a54880?, 0xc138f28b60?}, 0x7fae6812f5d0?)
	net/http/server.go:2136 +0x29
github.com/gorilla/mux.(*Router).ServeHTTP(0xc000000a80, {0x1a54880, 0xc138f28b60}, 0xc0bd096900)
	github.com/gorilla/mux@v1.8.0/mux.go:210 +0x1c5
net/http.serverHandler.ServeHTTP({0xc02884c4e0?}, {0x1a54880?, 0xc138f28b60?}, 0x6?)
	net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc1926922d0, {0x1a576e0, 0xc024a6ec90})
	net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 16979
	net/http/server.go:3086 +0x5cb
" context=missing panic="runtime error: invalid memory address or nil pointer dereference"
```

[skip ci]
---
 roomserver/internal/query/query_room_hierarchy.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go
index 76eba12be..5f55980f0 100644
--- a/roomserver/internal/query/query_room_hierarchy.go
+++ b/roomserver/internal/query/query_room_hierarchy.go
@@ -17,6 +17,7 @@ package query
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sort"
 
@@ -56,6 +57,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 			break
 		}
 
+		// If the context is canceled, we might still have discovered rooms
+		// return them to the client and let the client know there _may_ be more rooms.
+		if errors.Is(ctx.Err(), context.Canceled) {
+			break
+		}
+
 		// pop the stack
 		queuedRoom := unvisited[len(unvisited)-1]
 		unvisited = unvisited[:len(unvisited)-1]
@@ -112,6 +119,11 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 
 			pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID)
 
+			if pubRoom == nil {
+				util.GetLogger(ctx).WithField("room_id", queuedRoom.RoomID).Debug("unable to get publicRoomsChunk")
+				continue
+			}
+
 			discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{
 				PublicRoom:    *pubRoom,
 				RoomType:      roomType,

From ee73a90aea8349e2695ce555ac868bbf45c11d02 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 8 Nov 2023 14:22:20 +0100
Subject: [PATCH 53/57] Fix potential connection leak (#3247)

We didn't rollback/commit after getting events, now we're rolling back
since we didn't change anything.
---
 syncapi/routing/getevent.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go
index c089539f0..d0227f4ea 100644
--- a/syncapi/routing/getevent.go
+++ b/syncapi/routing/getevent.go
@@ -44,7 +44,7 @@ func GetEvent(
 	rsAPI api.SyncRoomserverAPI,
 ) util.JSONResponse {
 	ctx := req.Context()
-	db, err := syncDB.NewDatabaseTransaction(ctx)
+	db, err := syncDB.NewDatabaseSnapshot(ctx)
 	logger := util.GetLogger(ctx).WithFields(logrus.Fields{
 		"event_id": eventID,
 		"room_id":  rawRoomID,
@@ -56,6 +56,7 @@ func GetEvent(
 			JSON: spec.InternalServerError{},
 		}
 	}
+	defer db.Rollback() // nolint: errcheck
 
 	roomID, err := spec.NewRoomID(rawRoomID)
 	if err != nil {

From 699f5ca8c1f73ff7e4b70f0f9273ffcb1c195cdc Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 9 Nov 2023 08:42:33 +0100
Subject: [PATCH 54/57] More `rows.Close()` and `rows.Err()` (#3262)

Looks like we missed some `rows.Close()`

Even though `rows.Err()` is mostly not necessary, we should be more
consistent in the DB layer.

[skip ci]
---
 .../notary_server_keys_metadata_table.go       |  2 +-
 .../storage/postgres/queue_json_table.go       |  2 +-
 .../storage/postgres/relay_servers_table.go    |  2 +-
 .../storage/sqlite3/joined_hosts_table.go      |  2 +-
 .../notary_server_keys_metadata_table.go       |  2 +-
 .../storage/sqlite3/queue_json_table.go        |  2 +-
 .../storage/sqlite3/relay_servers_table.go     |  2 +-
 .../storage/postgres/relay_queue_json_table.go |  2 +-
 .../storage/sqlite3/relay_queue_json_table.go  |  2 +-
 roomserver/storage/postgres/events_table.go    |  3 ++-
 .../storage/postgres/membership_table.go       |  2 +-
 roomserver/storage/postgres/rooms_table.go     |  8 ++++----
 .../storage/postgres/user_room_keys_table.go   |  3 ++-
 roomserver/storage/sqlite3/event_json_table.go |  2 +-
 .../storage/sqlite3/event_state_keys_table.go  |  4 ++--
 .../storage/sqlite3/event_types_table.go       |  2 +-
 roomserver/storage/sqlite3/events_table.go     | 18 +++++++++++++++---
 roomserver/storage/sqlite3/invite_table.go     |  5 ++++-
 roomserver/storage/sqlite3/membership_table.go |  4 +++-
 .../storage/sqlite3/room_aliases_table.go      |  2 +-
 roomserver/storage/sqlite3/rooms_table.go      |  8 ++++----
 .../storage/sqlite3/state_snapshot_table.go    |  7 +++++--
 .../storage/sqlite3/user_room_keys_table.go    |  3 ++-
 setup/mscs/msc2836/storage.go                  |  2 +-
 .../postgres/current_room_state_table.go       |  2 +-
 syncapi/storage/postgres/memberships_table.go  |  2 ++
 syncapi/storage/postgres/peeks_table.go        |  2 +-
 syncapi/storage/postgres/presence_table.go     |  2 +-
 .../sqlite3/current_room_state_table.go        |  6 +++---
 syncapi/storage/sqlite3/invites_table.go       |  2 +-
 syncapi/storage/sqlite3/memberships_table.go   |  2 ++
 .../sqlite3/output_room_events_table.go        |  4 ++--
 .../output_room_events_topology_table.go       |  4 +++-
 syncapi/storage/sqlite3/peeks_table.go         |  2 +-
 syncapi/storage/sqlite3/presence_table.go      |  2 +-
 .../postgres/cross_signing_keys_table.go       |  3 ++-
 .../postgres/cross_signing_sigs_table.go       |  3 ++-
 userapi/storage/postgres/key_backup_table.go   |  2 +-
 userapi/storage/postgres/key_changes_table.go  |  3 ++-
 .../storage/postgres/one_time_keys_table.go    |  2 +-
 userapi/storage/postgres/profile_table.go      |  2 +-
 userapi/storage/postgres/threepid_table.go     |  4 +++-
 userapi/storage/sqlite3/account_data_table.go  |  4 +++-
 .../sqlite3/cross_signing_keys_table.go        |  3 ++-
 .../sqlite3/cross_signing_sigs_table.go        |  3 ++-
 userapi/storage/sqlite3/devices_table.go       |  3 ++-
 userapi/storage/sqlite3/key_backup_table.go    |  2 +-
 userapi/storage/sqlite3/key_changes_table.go   |  3 ++-
 userapi/storage/sqlite3/one_time_keys_table.go |  2 +-
 userapi/storage/sqlite3/profile_table.go       |  2 +-
 50 files changed, 101 insertions(+), 61 deletions(-)

diff --git a/federationapi/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go
index 7a1ec4122..47aa82b48 100644
--- a/federationapi/storage/postgres/notary_server_keys_metadata_table.go
+++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go
@@ -151,7 +151,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn
 		}
 		results = append(results, sk)
 	}
-	return results, nil
+	return results, rows.Err()
 }
 
 func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error {
diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go
index 563738dd5..f92e33d5e 100644
--- a/federationapi/storage/postgres/queue_json_table.go
+++ b/federationapi/storage/postgres/queue_json_table.go
@@ -109,5 +109,5 @@ func (s *queueJSONStatements) SelectQueueJSON(
 		}
 		blobs[nid] = blob
 	}
-	return blobs, err
+	return blobs, rows.Err()
 }
diff --git a/federationapi/storage/postgres/relay_servers_table.go b/federationapi/storage/postgres/relay_servers_table.go
index 9e1bc5d40..1a47816e2 100644
--- a/federationapi/storage/postgres/relay_servers_table.go
+++ b/federationapi/storage/postgres/relay_servers_table.go
@@ -110,7 +110,7 @@ func (s *relayServersStatements) SelectRelayServers(
 		}
 		result = append(result, spec.ServerName(relayServer))
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *relayServersStatements) DeleteRelayServers(
diff --git a/federationapi/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go
index 2412cacdb..418194312 100644
--- a/federationapi/storage/sqlite3/joined_hosts_table.go
+++ b/federationapi/storage/sqlite3/joined_hosts_table.go
@@ -216,5 +216,5 @@ func joinedHostsFromStmt(
 		})
 	}
 
-	return result, nil
+	return result, rows.Err()
 }
diff --git a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go
index 2fd9ef211..d9b98fc4f 100644
--- a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go
+++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go
@@ -154,7 +154,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn
 		}
 		results = append(results, sk)
 	}
-	return results, nil
+	return results, rows.Err()
 }
 
 func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error {
diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go
index 0e2806d56..33ae06131 100644
--- a/federationapi/storage/sqlite3/queue_json_table.go
+++ b/federationapi/storage/sqlite3/queue_json_table.go
@@ -135,5 +135,5 @@ func (s *queueJSONStatements) SelectQueueJSON(
 		}
 		blobs[nid] = blob
 	}
-	return blobs, err
+	return blobs, rows.Err()
 }
diff --git a/federationapi/storage/sqlite3/relay_servers_table.go b/federationapi/storage/sqlite3/relay_servers_table.go
index 36cabeb4d..232db32af 100644
--- a/federationapi/storage/sqlite3/relay_servers_table.go
+++ b/federationapi/storage/sqlite3/relay_servers_table.go
@@ -109,7 +109,7 @@ func (s *relayServersStatements) SelectRelayServers(
 		}
 		result = append(result, spec.ServerName(relayServer))
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *relayServersStatements) DeleteRelayServers(
diff --git a/relayapi/storage/postgres/relay_queue_json_table.go b/relayapi/storage/postgres/relay_queue_json_table.go
index 74410fc88..94ae41407 100644
--- a/relayapi/storage/postgres/relay_queue_json_table.go
+++ b/relayapi/storage/postgres/relay_queue_json_table.go
@@ -109,5 +109,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON(
 		}
 		blobs[nid] = blob
 	}
-	return blobs, err
+	return blobs, rows.Err()
 }
diff --git a/relayapi/storage/sqlite3/relay_queue_json_table.go b/relayapi/storage/sqlite3/relay_queue_json_table.go
index 502da3b00..a1af82aa0 100644
--- a/relayapi/storage/sqlite3/relay_queue_json_table.go
+++ b/relayapi/storage/sqlite3/relay_queue_json_table.go
@@ -133,5 +133,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON(
 		}
 		blobs[nid] = blob
 	}
-	return blobs, err
+	return blobs, rows.Err()
 }
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index a00b4b1d7..1c9cd1599 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -249,6 +249,7 @@ func (s *eventStatements) BulkSelectSnapshotsFromEventIDs(
 	if err != nil {
 		return nil, err
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "BulkSelectSnapshotsFromEventIDs: rows.close() failed")
 
 	var eventID string
 	var stateNID types.StateSnapshotNID
@@ -563,7 +564,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs(
 		}
 		result[eventNID] = roomNID
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array {
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index 835a43b2d..1a96e3527 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -363,7 +363,7 @@ func (s *membershipStatements) SelectRoomsWithMembership(
 		}
 		roomNIDs = append(roomNIDs, roomNID)
 	}
-	return roomNIDs, nil
+	return roomNIDs, rows.Err()
 }
 
 func (s *membershipStatements) SelectJoinedUsersSetForRooms(
diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go
index c8346733d..bc3820b2c 100644
--- a/roomserver/storage/postgres/rooms_table.go
+++ b/roomserver/storage/postgres/rooms_table.go
@@ -137,7 +137,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T
 		}
 		roomIDs = append(roomIDs, roomID)
 	}
-	return roomIDs, nil
+	return roomIDs, rows.Err()
 }
 func (s *roomStatements) InsertRoomNID(
 	ctx context.Context, txn *sql.Tx,
@@ -255,7 +255,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
 		}
 		result[roomNID] = roomVersion
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) {
@@ -277,7 +277,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
 		}
 		roomIDs = append(roomIDs, roomID)
 	}
-	return roomIDs, nil
+	return roomIDs, rows.Err()
 }
 
 func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) {
@@ -299,7 +299,7 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
 		}
 		roomNIDs = append(roomNIDs, roomNID)
 	}
-	return roomNIDs, nil
+	return roomNIDs, rows.Err()
 }
 
 func roomNIDsAsArray(roomNIDs []types.RoomNID) pq.Int64Array {
diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go
index 217ee957f..57e8f213b 100644
--- a/roomserver/storage/postgres/user_room_keys_table.go
+++ b/roomserver/storage/postgres/user_room_keys_table.go
@@ -162,6 +162,7 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context,
 	if errors.Is(err, sql.ErrNoRows) {
 		return nil, nil
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows")
 
 	resultMap := make(map[types.RoomNID]ed25519.PublicKey)
 
@@ -173,5 +174,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context,
 		}
 		resultMap[roomNID] = pubkey
 	}
-	return resultMap, err
+	return resultMap, rows.Err()
 }
diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go
index dc26885bb..325951c7f 100644
--- a/roomserver/storage/sqlite3/event_json_table.go
+++ b/roomserver/storage/sqlite3/event_json_table.go
@@ -109,5 +109,5 @@ func (s *eventJSONStatements) BulkSelectEventJSON(
 		}
 		result.EventNID = types.EventNID(eventNID)
 	}
-	return results[:i], nil
+	return results[:i], rows.Err()
 }
diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go
index 347524a81..a052d69ed 100644
--- a/roomserver/storage/sqlite3/event_state_keys_table.go
+++ b/roomserver/storage/sqlite3/event_state_keys_table.go
@@ -136,7 +136,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID(
 		}
 		result[stateKey] = types.EventStateKeyNID(stateKeyNID)
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *eventStateKeyStatements) BulkSelectEventStateKey(
@@ -167,5 +167,5 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey(
 		}
 		result[types.EventStateKeyNID(stateKeyNID)] = stateKey
 	}
-	return result, nil
+	return result, rows.Err()
 }
diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go
index 0581ec194..c030fffea 100644
--- a/roomserver/storage/sqlite3/event_types_table.go
+++ b/roomserver/storage/sqlite3/event_types_table.go
@@ -147,5 +147,5 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID(
 		}
 		result[eventType] = types.EventTypeNID(eventTypeNID)
 	}
-	return result, nil
+	return result, rows.Err()
 }
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index c49c6dc38..2c269bced 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -310,6 +310,9 @@ func (s *eventStatements) BulkSelectStateEventByID(
 		}
 		results = append(results, result)
 	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
 	if !excludeRejected && i != len(eventIDs) {
 		// If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have.
 		// We don't know which ones were missing because we don't return the string IDs in the query.
@@ -377,7 +380,7 @@ func (s *eventStatements) BulkSelectStateEventByNID(
 			return nil, err
 		}
 	}
-	return results[:i], err
+	return results[:i], rows.Err()
 }
 
 // bulkSelectStateAtEventByID lookups the state at a list of events by event ID.
@@ -425,6 +428,9 @@ func (s *eventStatements) BulkSelectStateAtEventByID(
 			)
 		}
 	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
 	if i != len(eventIDs) {
 		return nil, types.MissingEventError(
 			fmt.Sprintf("storage: event IDs missing from the database (%d != %d)", i, len(eventIDs)),
@@ -507,6 +513,9 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference(
 		result.BeforeStateSnapshotNID = types.StateSnapshotNID(stateSnapshotNID)
 		result.EventID = eventID
 	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
 	if i != len(eventNIDs) {
 		return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs))
 	}
@@ -544,6 +553,9 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, txn *sql.Tx, ev
 		}
 		results[types.EventNID(eventNID)] = eventID
 	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
 	if i != len(eventNIDs) {
 		return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs))
 	}
@@ -602,7 +614,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e
 			RoomNID:  types.RoomNID(roomNID),
 		}
 	}
-	return results, nil
+	return results, rows.Err()
 }
 
 func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) {
@@ -652,7 +664,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs(
 		}
 		result[eventNID] = roomNID
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func eventNIDsAsArray(eventNIDs []types.EventNID) string {
diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go
index ca6e7c511..b678d8add 100644
--- a/roomserver/storage/sqlite3/invite_table.go
+++ b/roomserver/storage/sqlite3/invite_table.go
@@ -126,6 +126,9 @@ func (s *inviteStatements) UpdateInviteRetired(
 		}
 		eventIDs = append(eventIDs, inviteEventID)
 	}
+	if err = rows.Err(); err != nil {
+		return
+	}
 	// now retire the invites
 	stmt = sqlutil.TxStmt(txn, s.updateInviteRetiredStmt)
 	_, err = stmt.ExecContext(ctx, roomNID, targetUserNID)
@@ -157,5 +160,5 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 		result = append(result, types.EventStateKeyNID(senderUserNID))
 		eventIDs = append(eventIDs, eventID)
 	}
-	return result, eventIDs, eventJSON, nil
+	return result, eventIDs, eventJSON, rows.Err()
 }
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index 977788d50..1012c074a 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -250,6 +250,7 @@ func (s *membershipStatements) SelectMembershipsFromRoom(
 		}
 		eventNIDs = append(eventNIDs, eNID)
 	}
+	err = rows.Err()
 	return
 }
 
@@ -277,6 +278,7 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership(
 		}
 		eventNIDs = append(eventNIDs, eNID)
 	}
+	err = rows.Err()
 	return
 }
 
@@ -313,7 +315,7 @@ func (s *membershipStatements) SelectRoomsWithMembership(
 		}
 		roomNIDs = append(roomNIDs, roomNID)
 	}
-	return roomNIDs, nil
+	return roomNIDs, rows.Err()
 }
 
 func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) {
diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go
index 3bdbbaa35..815b42a27 100644
--- a/roomserver/storage/sqlite3/room_aliases_table.go
+++ b/roomserver/storage/sqlite3/room_aliases_table.go
@@ -121,7 +121,7 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID(
 
 		aliases = append(aliases, alias)
 	}
-
+	err = rows.Err()
 	return
 }
 
diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go
index 7556b3461..22700a710 100644
--- a/roomserver/storage/sqlite3/rooms_table.go
+++ b/roomserver/storage/sqlite3/rooms_table.go
@@ -128,7 +128,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T
 		}
 		roomIDs = append(roomIDs, roomID)
 	}
-	return roomIDs, nil
+	return roomIDs, rows.Err()
 }
 
 func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
@@ -265,7 +265,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
 		}
 		result[roomNID] = roomVersion
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) {
@@ -293,7 +293,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
 		}
 		roomIDs = append(roomIDs, roomID)
 	}
-	return roomIDs, nil
+	return roomIDs, rows.Err()
 }
 
 func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) {
@@ -321,5 +321,5 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
 		}
 		roomNIDs = append(roomNIDs, roomNID)
 	}
-	return roomNIDs, nil
+	return roomNIDs, rows.Err()
 }
diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go
index 2edff0ba8..dcac0b07c 100644
--- a/roomserver/storage/sqlite3/state_snapshot_table.go
+++ b/roomserver/storage/sqlite3/state_snapshot_table.go
@@ -133,13 +133,16 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 	var stateBlockNIDsJSON string
 	for ; rows.Next(); i++ {
 		result := &results[i]
-		if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil {
+		if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil {
 			return nil, err
 		}
-		if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil {
+		if err = json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil {
 			return nil, err
 		}
 	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
 	if i != len(stateNIDs) {
 		return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)))
 	}
diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go
index 434bad295..13906f771 100644
--- a/roomserver/storage/sqlite3/user_room_keys_table.go
+++ b/roomserver/storage/sqlite3/user_room_keys_table.go
@@ -177,6 +177,7 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context,
 	if errors.Is(err, sql.ErrNoRows) {
 		return nil, nil
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows")
 
 	resultMap := make(map[types.RoomNID]ed25519.PublicKey)
 
@@ -188,5 +189,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context,
 		}
 		resultMap[roomNID] = pubkey
 	}
-	return resultMap, err
+	return resultMap, rows.Err()
 }
diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go
index ade2a1616..696d0b0da 100644
--- a/setup/mscs/msc2836/storage.go
+++ b/setup/mscs/msc2836/storage.go
@@ -301,7 +301,7 @@ func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, rec
 		}
 		children = append(children, evInfo)
 	}
-	return children, nil
+	return children, rows.Err()
 }
 
 func (p *DB) ParentForChild(ctx context.Context, eventID, relType string) (*eventInfo, error) {
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index b0148bef5..ec0b27adc 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -392,7 +392,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er
 		})
 	}
 
-	return events, nil
+	return events, rows.Err()
 }
 
 func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) {
diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go
index 4fe4260da..e5208b891 100644
--- a/syncapi/storage/postgres/memberships_table.go
+++ b/syncapi/storage/postgres/memberships_table.go
@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"fmt"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	rstypes "github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
@@ -160,6 +161,7 @@ func (s *membershipsStatements) SelectMemberships(
 	if err != nil {
 		return
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows")
 	var (
 		eventID string
 	)
diff --git a/syncapi/storage/postgres/peeks_table.go b/syncapi/storage/postgres/peeks_table.go
index 64183073d..1120dce0b 100644
--- a/syncapi/storage/postgres/peeks_table.go
+++ b/syncapi/storage/postgres/peeks_table.go
@@ -164,7 +164,7 @@ func (s *peekStatements) SelectPeekingDevices(
 		devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID})
 		result[roomID] = devices
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *peekStatements) SelectMaxPeekID(
diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go
index f37b5331e..53acecce5 100644
--- a/syncapi/storage/postgres/presence_table.go
+++ b/syncapi/storage/postgres/presence_table.go
@@ -144,7 +144,7 @@ func (p *presenceStatements) GetPresenceForUsers(
 		presence.ClientFields.Presence = presence.Presence.String()
 		result = append(result, presence)
 	}
-	return result, err
+	return result, rows.Err()
 }
 
 func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) {
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 78b2e397c..f430fcc05 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -177,7 +177,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers(
 		users = append(users, userID)
 		result[roomID] = users
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
@@ -236,7 +236,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 		}
 		result = append(result, roomID)
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 // SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user.
@@ -419,7 +419,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er
 		})
 	}
 
-	return events, nil
+	return events, rows.Err()
 }
 
 func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) {
diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go
index ebb469d24..e50b5cbf8 100644
--- a/syncapi/storage/sqlite3/invites_table.go
+++ b/syncapi/storage/sqlite3/invites_table.go
@@ -176,7 +176,7 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange(
 	if lastPos == 0 {
 		lastPos = r.To
 	}
-	return result, retired, lastPos, nil
+	return result, retired, lastPos, rows.Err()
 }
 
 func (s *inviteEventsStatements) SelectMaxInviteID(
diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go
index a1b16306c..9e50422e5 100644
--- a/syncapi/storage/sqlite3/memberships_table.go
+++ b/syncapi/storage/sqlite3/memberships_table.go
@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"fmt"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	rstypes "github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
@@ -163,6 +164,7 @@ func (s *membershipsStatements) SelectMemberships(
 	if err != nil {
 		return
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows")
 	var eventID string
 	for rows.Next() {
 		if err = rows.Scan(&eventID); err != nil {
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 93caee806..c7b11d3ef 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -274,7 +274,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		}
 	}
 
-	return stateNeeded, eventIDToEvent, nil
+	return stateNeeded, eventIDToEvent, rows.Err()
 }
 
 // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied,
@@ -520,7 +520,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 			ExcludeFromSync: excludeFromSync,
 		})
 	}
-	return result, nil
+	return result, rows.Err()
 }
 func (s *outputRoomEventsStatements) SelectContextEvent(
 	ctx context.Context, txn *sql.Tx, roomID, eventID string,
diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go
index 36967d1e7..c00fb7a79 100644
--- a/syncapi/storage/sqlite3/output_room_events_topology_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	rstypes "github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
@@ -137,6 +138,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange(
 	} else if err != nil {
 		return
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectEventIDsInRange: failed to close rows")
 
 	// Return the IDs.
 	var eventID string
@@ -155,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange(
 		start = tokens[0]
 		end = tokens[len(tokens)-1]
 	}
-
+	err = rows.Err()
 	return
 }
 
diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go
index 5d5200abc..d8998e2b8 100644
--- a/syncapi/storage/sqlite3/peeks_table.go
+++ b/syncapi/storage/sqlite3/peeks_table.go
@@ -184,7 +184,7 @@ func (s *peekStatements) SelectPeekingDevices(
 		devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID})
 		result[roomID] = devices
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *peekStatements) SelectMaxPeekID(
diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go
index 573fbad6c..40b57e75d 100644
--- a/syncapi/storage/sqlite3/presence_table.go
+++ b/syncapi/storage/sqlite3/presence_table.go
@@ -169,7 +169,7 @@ func (p *presenceStatements) GetPresenceForUsers(
 		presence.ClientFields.Presence = presence.Presence.String()
 		result = append(result, presence)
 	}
-	return result, err
+	return result, rows.Err()
 }
 
 func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) {
diff --git a/userapi/storage/postgres/cross_signing_keys_table.go b/userapi/storage/postgres/cross_signing_keys_table.go
index 138b629d7..deb355718 100644
--- a/userapi/storage/postgres/cross_signing_keys_table.go
+++ b/userapi/storage/postgres/cross_signing_keys_table.go
@@ -77,7 +77,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser(
 	for rows.Next() {
 		var keyTypeInt int16
 		var keyData spec.Base64Bytes
-		if err := rows.Scan(&keyTypeInt, &keyData); err != nil {
+		if err = rows.Scan(&keyTypeInt, &keyData); err != nil {
 			return nil, err
 		}
 		keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt]
@@ -86,6 +86,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser(
 		}
 		r[keyType] = keyData
 	}
+	err = rows.Err()
 	return
 }
 
diff --git a/userapi/storage/postgres/cross_signing_sigs_table.go b/userapi/storage/postgres/cross_signing_sigs_table.go
index 61a381184..cba015e13 100644
--- a/userapi/storage/postgres/cross_signing_sigs_table.go
+++ b/userapi/storage/postgres/cross_signing_sigs_table.go
@@ -98,7 +98,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
 		var userID string
 		var keyID gomatrixserverlib.KeyID
 		var signature spec.Base64Bytes
-		if err := rows.Scan(&userID, &keyID, &signature); err != nil {
+		if err = rows.Scan(&userID, &keyID, &signature); err != nil {
 			return nil, err
 		}
 		if _, ok := r[userID]; !ok {
@@ -106,6 +106,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
 		}
 		r[userID][keyID] = signature
 	}
+	err = rows.Err()
 	return
 }
 
diff --git a/userapi/storage/postgres/key_backup_table.go b/userapi/storage/postgres/key_backup_table.go
index 91a34c357..59944a125 100644
--- a/userapi/storage/postgres/key_backup_table.go
+++ b/userapi/storage/postgres/key_backup_table.go
@@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api.
 		roomData[key.SessionID] = key.KeyBackupSession
 		result[key.RoomID] = roomData
 	}
-	return result, nil
+	return result, rows.Err()
 }
diff --git a/userapi/storage/postgres/key_changes_table.go b/userapi/storage/postgres/key_changes_table.go
index a00494140..de3a9e9c8 100644
--- a/userapi/storage/postgres/key_changes_table.go
+++ b/userapi/storage/postgres/key_changes_table.go
@@ -115,7 +115,7 @@ func (s *keyChangesStatements) SelectKeyChanges(
 	for rows.Next() {
 		var userID string
 		var offset int64
-		if err := rows.Scan(&userID, &offset); err != nil {
+		if err = rows.Scan(&userID, &offset); err != nil {
 			return nil, 0, err
 		}
 		if offset > latestOffset {
@@ -123,5 +123,6 @@ func (s *keyChangesStatements) SelectKeyChanges(
 		}
 		userIDs = append(userIDs, userID)
 	}
+	err = rows.Err()
 	return
 }
diff --git a/userapi/storage/postgres/one_time_keys_table.go b/userapi/storage/postgres/one_time_keys_table.go
index 972a59147..a00f4d6f6 100644
--- a/userapi/storage/postgres/one_time_keys_table.go
+++ b/userapi/storage/postgres/one_time_keys_table.go
@@ -134,7 +134,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de
 		}
 		counts.KeyCount[algorithm] = count
 	}
-	return counts, nil
+	return counts, rows.Err()
 }
 
 func (s *oneTimeKeysStatements) InsertOneTimeKeys(ctx context.Context, txn *sql.Tx, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) {
diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go
index e404c32f2..e4f55ed94 100644
--- a/userapi/storage/postgres/profile_table.go
+++ b/userapi/storage/postgres/profile_table.go
@@ -165,5 +165,5 @@ func (s *profilesStatements) SelectProfilesBySearch(
 			profiles = append(profiles, profile)
 		}
 	}
-	return profiles, nil
+	return profiles, rows.Err()
 }
diff --git a/userapi/storage/postgres/threepid_table.go b/userapi/storage/postgres/threepid_table.go
index 15b42a0a6..fc46061dc 100644
--- a/userapi/storage/postgres/threepid_table.go
+++ b/userapi/storage/postgres/threepid_table.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -94,6 +95,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart(
 	if err != nil {
 		return
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectThreePIDsForLocalpart: failed to close rows")
 
 	threepids = []authtypes.ThreePID{}
 	for rows.Next() {
@@ -107,7 +109,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart(
 			Medium:  medium,
 		})
 	}
-
+	err = rows.Err()
 	return
 }
 
diff --git a/userapi/storage/sqlite3/account_data_table.go b/userapi/storage/sqlite3/account_data_table.go
index 3a6367c45..240647b32 100644
--- a/userapi/storage/sqlite3/account_data_table.go
+++ b/userapi/storage/sqlite3/account_data_table.go
@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -95,6 +96,7 @@ func (s *accountDataStatements) SelectAccountData(
 	if err != nil {
 		return nil, nil, err
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectAccountData: failed to close rows")
 
 	global := map[string]json.RawMessage{}
 	rooms := map[string]map[string]json.RawMessage{}
@@ -118,7 +120,7 @@ func (s *accountDataStatements) SelectAccountData(
 		}
 	}
 
-	return global, rooms, nil
+	return global, rooms, rows.Err()
 }
 
 func (s *accountDataStatements) SelectAccountDataByType(
diff --git a/userapi/storage/sqlite3/cross_signing_keys_table.go b/userapi/storage/sqlite3/cross_signing_keys_table.go
index 5c2ce7039..65b9ff1af 100644
--- a/userapi/storage/sqlite3/cross_signing_keys_table.go
+++ b/userapi/storage/sqlite3/cross_signing_keys_table.go
@@ -76,7 +76,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser(
 	for rows.Next() {
 		var keyTypeInt int16
 		var keyData spec.Base64Bytes
-		if err := rows.Scan(&keyTypeInt, &keyData); err != nil {
+		if err = rows.Scan(&keyTypeInt, &keyData); err != nil {
 			return nil, err
 		}
 		keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt]
@@ -85,6 +85,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser(
 		}
 		r[keyType] = keyData
 	}
+	err = rows.Err()
 	return
 }
 
diff --git a/userapi/storage/sqlite3/cross_signing_sigs_table.go b/userapi/storage/sqlite3/cross_signing_sigs_table.go
index 657264115..bf400a00e 100644
--- a/userapi/storage/sqlite3/cross_signing_sigs_table.go
+++ b/userapi/storage/sqlite3/cross_signing_sigs_table.go
@@ -96,7 +96,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
 		var userID string
 		var keyID gomatrixserverlib.KeyID
 		var signature spec.Base64Bytes
-		if err := rows.Scan(&userID, &keyID, &signature); err != nil {
+		if err = rows.Scan(&userID, &keyID, &signature); err != nil {
 			return nil, err
 		}
 		if _, ok := r[userID]; !ok {
@@ -104,6 +104,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
 		}
 		r[userID][keyID] = signature
 	}
+	err = rows.Err()
 	return
 }
 
diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go
index 23e823116..5ce285c87 100644
--- a/userapi/storage/sqlite3/devices_table.go
+++ b/userapi/storage/sqlite3/devices_table.go
@@ -296,6 +296,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart(
 	if err != nil {
 		return devices, err
 	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectDevicesByLocalpart: failed to close rows")
 
 	var dev api.Device
 	var lastseents sql.NullInt64
@@ -325,7 +326,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart(
 		devices = append(devices, dev)
 	}
 
-	return devices, nil
+	return devices, rows.Err()
 }
 
 func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error) {
diff --git a/userapi/storage/sqlite3/key_backup_table.go b/userapi/storage/sqlite3/key_backup_table.go
index ed2746310..1cdaca180 100644
--- a/userapi/storage/sqlite3/key_backup_table.go
+++ b/userapi/storage/sqlite3/key_backup_table.go
@@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api.
 		roomData[key.SessionID] = key.KeyBackupSession
 		result[key.RoomID] = roomData
 	}
-	return result, nil
+	return result, rows.Err()
 }
diff --git a/userapi/storage/sqlite3/key_changes_table.go b/userapi/storage/sqlite3/key_changes_table.go
index 923bb57eb..7a4898cfb 100644
--- a/userapi/storage/sqlite3/key_changes_table.go
+++ b/userapi/storage/sqlite3/key_changes_table.go
@@ -113,7 +113,7 @@ func (s *keyChangesStatements) SelectKeyChanges(
 	for rows.Next() {
 		var userID string
 		var offset int64
-		if err := rows.Scan(&userID, &offset); err != nil {
+		if err = rows.Scan(&userID, &offset); err != nil {
 			return nil, 0, err
 		}
 		if offset > latestOffset {
@@ -121,5 +121,6 @@ func (s *keyChangesStatements) SelectKeyChanges(
 		}
 		userIDs = append(userIDs, userID)
 	}
+	err = rows.Err()
 	return
 }
diff --git a/userapi/storage/sqlite3/one_time_keys_table.go b/userapi/storage/sqlite3/one_time_keys_table.go
index a992d399c..2a5b1280b 100644
--- a/userapi/storage/sqlite3/one_time_keys_table.go
+++ b/userapi/storage/sqlite3/one_time_keys_table.go
@@ -140,7 +140,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de
 		}
 		counts.KeyCount[algorithm] = count
 	}
-	return counts, nil
+	return counts, rows.Err()
 }
 
 func (s *oneTimeKeysStatements) InsertOneTimeKeys(
diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go
index a20d7e848..7285110bc 100644
--- a/userapi/storage/sqlite3/profile_table.go
+++ b/userapi/storage/sqlite3/profile_table.go
@@ -173,5 +173,5 @@ func (s *profilesStatements) SelectProfilesBySearch(
 			profiles = append(profiles, profile)
 		}
 	}
-	return profiles, nil
+	return profiles, rows.Err()
 }

From 7863a405a5f41acd2e40b40ec288eebe781eac1a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 9 Nov 2023 08:43:27 +0100
Subject: [PATCH 55/57] Use `IsBlacklistedOrBackingOff` to determine if we
 should try to fetch devices (#3254)

Use `IsBlacklistedOrBackingOff` from the federation API to check if we
should fetch devices.

To reduce back pressure, we now only queue retrying servers if there's
space in the channel.
---
 appservice/appservice_test.go                 | 11 ++-
 build/dendritejs-pinecone/main.go             |  4 +-
 build/gobind-yggdrasil/monolith.go            |  2 +-
 clientapi/admin_test.go                       | 20 ++---
 clientapi/clientapi_test.go                   | 31 ++++---
 clientapi/routing/joinroom_test.go            |  7 +-
 clientapi/routing/login_test.go               |  2 +-
 clientapi/routing/register_test.go            |  6 +-
 .../monolith/monolith.go                      |  2 +-
 cmd/dendrite-demo-yggdrasil/main.go           |  9 ++-
 cmd/dendrite/main.go                          |  2 +-
 federationapi/federationapi.go                |  3 +-
 federationapi/internal/api.go                 |  4 +-
 federationapi/routing/profile_test.go         |  8 +-
 federationapi/routing/query_test.go           |  8 +-
 federationapi/routing/send_test.go            |  8 +-
 roomserver/roomserver_test.go                 | 11 ++-
 userapi/internal/device_list_update.go        | 80 ++++++++++++++-----
 userapi/internal/device_list_update_test.go   | 79 +++++++++++++++++-
 userapi/userapi.go                            |  5 +-
 20 files changed, 212 insertions(+), 90 deletions(-)

diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go
index bbdeb47df..eca63371d 100644
--- a/appservice/appservice_test.go
+++ b/appservice/appservice_test.go
@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/matrix-org/dendrite/appservice"
@@ -32,6 +33,10 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestAppserviceInternalAPI(t *testing.T) {
 
 	// Set expected results
@@ -144,7 +149,7 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 		runCases(t, asAPI)
@@ -239,7 +244,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
 	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
-	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 	t.Run("UserIDExists", func(t *testing.T) {
@@ -378,7 +383,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		// Create required internal APIs
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// start the consumer
 		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
 
diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go
index d3d5f59fe..6acc93c7b 100644
--- a/build/dendritejs-pinecone/main.go
+++ b/build/dendritejs-pinecone/main.go
@@ -191,13 +191,13 @@ func startup() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics)
+	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff)
 
 	asQuery := appservice.NewInternalAPI(
 		processCtx, cfg, &natsInstance, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
 	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
 
 	monolith := setup.Monolith{
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 791ad261b..2b227d373 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() {
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go
index b228dd93c..f0e5f004d 100644
--- a/clientapi/admin_test.go
+++ b/clientapi/admin_test.go
@@ -45,7 +45,7 @@ func TestAdminCreateToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -196,7 +196,7 @@ func TestAdminListRegistrationTokens(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -314,7 +314,7 @@ func TestAdminGetRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -415,7 +415,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -509,7 +509,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -693,7 +693,7 @@ func TestAdminResetPassword(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for changing the password/login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -791,7 +791,7 @@ func TestPurgeRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
 
 		// Create the room
@@ -863,7 +863,7 @@ func TestAdminEvacuateRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
@@ -964,7 +964,7 @@ func TestAdminEvacuateUser(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
@@ -1055,7 +1055,7 @@ func TestAdminMarkAsStale(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index 2bb15fba3..2ff4b6503 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushrules"
@@ -49,6 +50,10 @@ type userDevice struct {
 	password    string
 }
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestGetPutDevices(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -121,7 +126,7 @@ func TestGetPutDevices(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -170,7 +175,7 @@ func TestDeleteDevice(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -275,7 +280,7 @@ func TestDeleteDevices(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -442,7 +447,7 @@ func TestSetDisplayname(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -554,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -632,7 +637,7 @@ func TestTyping(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -716,7 +721,7 @@ func TestMembership(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		rsAPI.SetUserAPI(userAPI)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -955,7 +960,7 @@ func TestCapabilities(t *testing.T) {
 		// Needed to create accounts
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1002,7 +1007,7 @@ func TestTurnserver(t *testing.T) {
 	// Needed to create accounts
 	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	//rsAPI.SetUserAPI(userAPI)
 	// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 	AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1100,7 +1105,7 @@ func Test3PID(t *testing.T) {
 		// Needed to create accounts
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1276,7 +1281,7 @@ func TestPushRules(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1663,7 +1668,7 @@ func TestKeys(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -2125,7 +2130,7 @@ func TestKeyBackup(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go
index be3fb429f..bd854efa8 100644
--- a/clientapi/routing/joinroom_test.go
+++ b/clientapi/routing/joinroom_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -21,6 +22,10 @@ import (
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestJoinRoomByIDOrAlias(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -36,7 +41,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 		// Create the users in the userapi
diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go
index 4c4fc3533..1c628b196 100644
--- a/clientapi/routing/login_test.go
+++ b/clientapi/routing/login_test.go
@@ -49,7 +49,7 @@ func TestLogin(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for /login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go
index 69b29e9c2..98455f80a 100644
--- a/clientapi/routing/register_test.go
+++ b/clientapi/routing/register_test.go
@@ -416,7 +416,7 @@ func Test_register(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -596,7 +596,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
@@ -637,7 +637,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go
index 5222d0b8e..d9f44b5cc 100644
--- a/cmd/dendrite-demo-pinecone/monolith/monolith.go
+++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go
@@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite(
 	)
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index b07687538..1e5186348 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -213,14 +213,15 @@ func main() {
 	natsInstance := jetstream.NATSInstance{}
 	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics)
-
-	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
-	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
+
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
+	rsAPI.SetAppserviceAPI(asAPI)
+
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go
index f38263c6c..5234b7504 100644
--- a/cmd/dendrite/main.go
+++ b/cmd/dendrite/main.go
@@ -162,7 +162,7 @@ func main() {
 	// dependency. Other components also need updating after their dependencies are up.
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 	rsAPI.SetAppserviceAPI(asAPI)
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index e148199fb..e2524f66a 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -24,7 +24,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
 	"github.com/matrix-org/dendrite/federationapi/internal"
@@ -102,7 +101,7 @@ func NewInternalAPI(
 	caches *caching.Caches,
 	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,
-) api.FederationInternalAPI {
+) *internal.FederationInternalAPI {
 	cfg := &dendriteCfg.FederationAPI
 
 	federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName)
diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go
index 3e6f39566..67388a102 100644
--- a/federationapi/internal/api.go
+++ b/federationapi/internal/api.go
@@ -112,7 +112,7 @@ func NewFederationInternalAPI(
 	}
 }
 
-func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) {
+func (a *FederationInternalAPI) IsBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) {
 	stats := a.statistics.ForServer(s)
 	if stats.Blacklisted() {
 		return stats, &api.FederationClientError{
@@ -151,7 +151,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti
 func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
 	s spec.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
-	stats, err := a.isBlacklistedOrBackingOff(s)
+	stats, err := a.IsBlacklistedOrBackingOff(s)
 	if err != nil {
 		return nil, err
 	}
diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go
index a31b206c1..ba13e07fc 100644
--- a/federationapi/routing/profile_test.go
+++ b/federationapi/routing/profile_test.go
@@ -26,7 +26,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -67,11 +66,8 @@ func TestHandleQueryProfile(t *testing.T) {
 		keyRing := serverKeyAPI.KeyRing()
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
-		if !ok {
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
+
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)
diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go
index bb14ab031..fd0894d15 100644
--- a/federationapi/routing/query_test.go
+++ b/federationapi/routing/query_test.go
@@ -25,7 +25,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -65,11 +64,8 @@ func TestHandleQueryDirectory(t *testing.T) {
 		keyRing := serverKeyAPI.KeyRing()
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
-		if !ok {
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
+
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)
diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go
index f629479da..ff4f7bd06 100644
--- a/federationapi/routing/send_test.go
+++ b/federationapi/routing/send_test.go
@@ -23,7 +23,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -62,11 +61,8 @@ func TestHandleSend(t *testing.T) {
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true)
 		serverKeyAPI := &signing.YggdrasilKeys{}
 		keyRing := serverKeyAPI.KeyRing()
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
-		if !ok {
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics)
+
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index 22d27ba05..218a0d8a9 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -34,6 +35,10 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 type FakeQuerier struct {
 	api.QuerySenderIDAPI
 }
@@ -58,7 +63,7 @@ func TestUsers(t *testing.T) {
 		})
 
 		t.Run("kick users", func(t *testing.T) {
-			usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+			usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 			rsAPI.SetUserAPI(usrAPI)
 			testKickUsers(t, rsAPI, usrAPI)
 		})
@@ -258,7 +263,7 @@ func TestPurgeRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
 
 		// Create the room
@@ -1050,7 +1055,7 @@ func TestUpgrade(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		rsAPI.SetUserAPI(userAPI)
 
 		for _, tc := range testCases {
diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go
index a4d28188a..b40635160 100644
--- a/userapi/internal/device_list_update.go
+++ b/userapi/internal/device_list_update.go
@@ -25,6 +25,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -108,6 +109,8 @@ type DeviceListUpdater struct {
 	userIDToChan   map[string]chan bool
 	userIDToChanMu *sync.Mutex
 	rsAPI          rsapi.KeyserverRoomserverAPI
+
+	isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error)
 }
 
 // DeviceListUpdaterDatabase is the subset of functionality from storage.Database required for the updater.
@@ -167,25 +170,28 @@ func NewDeviceListUpdater(
 	process *process.ProcessContext, db DeviceListUpdaterDatabase,
 	api DeviceListUpdaterAPI, producer KeyChangeProducer,
 	fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int,
-	rsAPI rsapi.KeyserverRoomserverAPI, thisServer spec.ServerName,
+	rsAPI rsapi.KeyserverRoomserverAPI,
+	thisServer spec.ServerName,
 	enableMetrics bool,
+	isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error),
 ) *DeviceListUpdater {
 	if enableMetrics {
 		prometheus.MustRegister(deviceListUpdaterBackpressure, deviceListUpdaterServersRetrying)
 	}
 	return &DeviceListUpdater{
-		process:        process,
-		userIDToMutex:  make(map[string]*sync.Mutex),
-		mu:             &sync.Mutex{},
-		db:             db,
-		api:            api,
-		producer:       producer,
-		fedClient:      fedClient,
-		thisServer:     thisServer,
-		workerChans:    make([]chan spec.ServerName, numWorkers),
-		userIDToChan:   make(map[string]chan bool),
-		userIDToChanMu: &sync.Mutex{},
-		rsAPI:          rsAPI,
+		process:                     process,
+		userIDToMutex:               make(map[string]*sync.Mutex),
+		mu:                          &sync.Mutex{},
+		db:                          db,
+		api:                         api,
+		producer:                    producer,
+		fedClient:                   fedClient,
+		thisServer:                  thisServer,
+		workerChans:                 make([]chan spec.ServerName, numWorkers),
+		userIDToChan:                make(map[string]chan bool),
+		userIDToChanMu:              &sync.Mutex{},
+		rsAPI:                       rsAPI,
+		isBlacklistedOrBackingOffFn: isBlacklistedOrBackingOffFn,
 	}
 }
 
@@ -362,13 +368,22 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) {
 	if err != nil {
 		return
 	}
+	_, err = u.isBlacklistedOrBackingOffFn(remoteServer)
+	var federationClientError *fedsenderapi.FederationClientError
+	if errors.As(err, &federationClientError) {
+		if federationClientError.Blacklisted {
+			return
+		}
+	}
+
 	hash := fnv.New32a()
 	_, _ = hash.Write([]byte(remoteServer))
 	index := int(int64(hash.Sum32()) % int64(len(u.workerChans)))
 
 	ch := u.assignChannel(userID)
+	// Since workerChans are buffered, we only increment here and let the worker
+	// decrement it once it is done processing.
 	deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Inc()
-	defer deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Dec()
 	u.workerChans[index] <- remoteServer
 	select {
 	case <-ch:
@@ -405,24 +420,38 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) {
 	go func() {
 		var serversToRetry []spec.ServerName
 		for {
-			serversToRetry = serversToRetry[:0] // reuse memory
-			time.Sleep(time.Second)
+			// nuke serversToRetry by re-slicing it to be "empty".
+			// The capacity of the slice is unchanged, which ensures we can reuse the memory.
+			serversToRetry = serversToRetry[:0]
+
+			deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries)))
+			time.Sleep(time.Second * 2)
+
+			// -2, so we have space for incoming device list updates over federation
+			maxServers := (cap(ch) - len(ch)) - 2
+			if maxServers <= 0 {
+				continue
+			}
+
 			retriesMu.Lock()
 			now := time.Now()
 			for srv, retryAt := range retries {
 				if now.After(retryAt) {
 					serversToRetry = append(serversToRetry, srv)
+					if maxServers == len(serversToRetry) {
+						break
+					}
 				}
 			}
+
 			for _, srv := range serversToRetry {
 				delete(retries, srv)
 			}
-			deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries)))
 			retriesMu.Unlock()
+
 			for _, srv := range serversToRetry {
 				deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Inc()
 				ch <- srv
-				deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec()
 			}
 		}
 	}()
@@ -430,8 +459,18 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) {
 		retriesMu.Lock()
 		_, exists := retries[serverName]
 		retriesMu.Unlock()
-		if exists {
-			// Don't retry a server that we're already waiting for.
+
+		// If the serverName is coming from retries, maybe it was
+		// blacklisted in the meantime.
+		_, err := u.isBlacklistedOrBackingOffFn(serverName)
+		var federationClientError *fedsenderapi.FederationClientError
+		// unwrap errors and check for FederationClientError, if found, federationClientError will be not nil
+		errors.As(err, &federationClientError)
+		isBlacklisted := federationClientError != nil && federationClientError.Blacklisted
+
+		// Don't retry a server that we're already waiting for or is blacklisted by now.
+		if exists || isBlacklisted {
+			deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec()
 			continue
 		}
 		waitTime, shouldRetry := u.processServer(serverName)
@@ -442,6 +481,7 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) {
 			}
 			retriesMu.Unlock()
 		}
+		deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec()
 	}
 }
 
diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go
index 14a49bc54..a2f1869d1 100644
--- a/userapi/internal/device_list_update_test.go
+++ b/userapi/internal/device_list_update_test.go
@@ -27,6 +27,8 @@ import (
 	"testing"
 	"time"
 
+	api2 "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -129,6 +131,10 @@ type mockDeviceListUpdaterAPI struct {
 func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) {
 }
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 type roundTripper struct {
 	fn func(*http.Request) (*http.Response, error)
 }
@@ -162,7 +168,7 @@ func TestUpdateHavePrevID(t *testing.T) {
 	}
 	ap := &mockDeviceListUpdaterAPI{}
 	producer := &mockKeyChangeProducer{}
-	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics)
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	event := gomatrixserverlib.DeviceListUpdateEvent{
 		DeviceDisplayName: "Foo Bar",
 		Deleted:           false,
@@ -234,7 +240,7 @@ func TestUpdateNoPrevID(t *testing.T) {
 			`)),
 		}, nil
 	})
-	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics)
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	if err := updater.Start(); err != nil {
 		t.Fatalf("failed to start updater: %s", err)
 	}
@@ -304,7 +310,7 @@ func TestDebounce(t *testing.T) {
 		close(incomingFedReq)
 		return <-fedCh, nil
 	})
-	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics)
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	if err := updater.Start(); err != nil {
 		t.Fatalf("failed to start updater: %s", err)
 	}
@@ -407,7 +413,7 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) {
 
 		updater := NewDeviceListUpdater(processCtx, db, nil,
 			nil, nil,
-			0, rsAPI, "test", caching.DisableMetrics)
+			0, rsAPI, "test", caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		if err := updater.CleanUp(); err != nil {
 			t.Error(err)
 		}
@@ -475,3 +481,68 @@ func Test_dedupeStateList(t *testing.T) {
 		})
 	}
 }
+
+func TestDeviceListUpdaterIgnoreBlacklisted(t *testing.T) {
+	unreachableServer := spec.ServerName("notlocalhost")
+
+	updater := DeviceListUpdater{
+		workerChans: make([]chan spec.ServerName, 1),
+		isBlacklistedOrBackingOffFn: func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+			switch s {
+			case unreachableServer:
+				return nil, &api2.FederationClientError{Blacklisted: true}
+			}
+			return nil, nil
+		},
+		mu:             &sync.Mutex{},
+		userIDToChanMu: &sync.Mutex{},
+		userIDToChan:   make(map[string]chan bool),
+		userIDToMutex:  make(map[string]*sync.Mutex),
+	}
+	workerCh := make(chan spec.ServerName)
+	defer close(workerCh)
+	updater.workerChans[0] = workerCh
+
+	// happy case
+	alice := "@alice:localhost"
+	aliceCh := updater.assignChannel(alice)
+	defer updater.clearChannel(alice)
+
+	// failing case
+	bob := "@bob:" + unreachableServer
+	bobCh := updater.assignChannel(string(bob))
+	defer updater.clearChannel(string(bob))
+
+	expectedServers := map[spec.ServerName]struct{}{
+		"localhost": {},
+	}
+	unexpectedServers := make(map[spec.ServerName]struct{})
+
+	go func() {
+		for serverName := range workerCh {
+			switch serverName {
+			case "localhost":
+				delete(expectedServers, serverName)
+				aliceCh <- true // unblock notifyWorkers
+			case unreachableServer: // this should not happen as it is "filtered" away by the blacklist
+				unexpectedServers[serverName] = struct{}{}
+				bobCh <- true
+			default:
+				unexpectedServers[serverName] = struct{}{}
+			}
+		}
+	}()
+
+	// alice is not blacklisted
+	updater.notifyWorkers(alice)
+	// bob is blacklisted
+	updater.notifyWorkers(string(bob))
+
+	for server := range expectedServers {
+		t.Errorf("Server still in expectedServers map: %s", server)
+	}
+
+	for server := range unexpectedServers {
+		t.Errorf("unexpected server in result: %s", server)
+	}
+}
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 34bf119a0..a1c9b94a9 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -18,10 +18,12 @@ import (
 	"time"
 
 	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/pushgateway"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
 
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
@@ -47,6 +49,7 @@ func NewInternalAPI(
 	rsAPI rsapi.UserRoomserverAPI,
 	fedClient fedsenderapi.KeyserverFederationAPI,
 	enableMetrics bool,
+	blacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error),
 ) *internal.UserInternalAPI {
 	js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream)
 	appServices := dendriteCfg.Derived.ApplicationServices
@@ -100,7 +103,7 @@ func NewInternalAPI(
 		FedClient:            fedClient,
 	}
 
-	updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics)
+	updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics, blacklistedOrBackingOffFn)
 	userAPI.Updater = updater
 	// Remove users which we don't share a room with anymore
 	if err := updater.CleanUp(); err != nil {

From fde4225469c81d4efafbc8e4424eee0585beb9bd Mon Sep 17 00:00:00 2001
From: Nikolai Patrick <41409702+Smartich0ke@users.noreply.github.com>
Date: Wed, 22 Nov 2023 21:43:41 +1030
Subject: [PATCH 56/57] fix typo (#3266)

Fix a tiny spelling mistake in the Grafana dashboard.
Literally a 1 character commit lol
### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x ] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [ x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `Nikolai Patrick nikolaipatrick@wws.sa.edu.au`
---
 helm/dendrite/grafana_dashboards/dendrite-rev2.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev2.json b/helm/dendrite/grafana_dashboards/dendrite-rev2.json
index 817f950b3..420d8bf1b 100644
--- a/helm/dendrite/grafana_dashboards/dendrite-rev2.json
+++ b/helm/dendrite/grafana_dashboards/dendrite-rev2.json
@@ -119,7 +119,7 @@
           "refId": "A"
         }
       ],
-      "title": "Registerd Users",
+      "title": "Registered Users",
       "type": "stat"
     },
     {

From 06e079abacb9c2719f0b73ab193be1e7818c43f6 Mon Sep 17 00:00:00 2001
From: notassigned <22502000+notassigned@users.noreply.github.com>
Date: Wed, 22 Nov 2023 06:14:49 -0500
Subject: [PATCH 57/57] Fix broken links in FAQ.md (#3259)

The links to CONTRUBITING.md and 4_adminapi.md were broken.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [ X] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [X ] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: <Private>

Co-authored-by: kegsay <kegan@matrix.org>
---
 docs/FAQ.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index 570ba677e..82b1581ea 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo
 
 Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
 specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 
-the development efforts through [contributing](../development/contributing). 
+the development efforts through [contributing](./development/CONTRIBUTING.md). 
 
 ## Is there a migration path from Synapse to Dendrite?
 
@@ -105,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom <ve
 
 ## How do I reset somebody's password on my server?
 
-Use the admin endpoint [resetpassword](./administration/adminapi#post-_dendriteadminresetpassworduserid)
+Use the admin endpoint [resetpassword](./administration/4_adminapi.md#post-_dendriteadminresetpassworduserid)
 
 ## Should I use PostgreSQL or SQLite for my databases?