From d58daf96655d8533c40be674fd11b74c38f40999 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 24 Jan 2024 19:24:04 +0100
Subject: [PATCH 01/46] Update sentry reporting (#3305)

This hopefully reduces the garbage we currently produce.
(Using [GlitchTip](https://glitchtip.com/) on my personal instance, this
seems to look better)
---
 federationapi/routing/routing.go                 |  2 ++
 internal/httputil/httpapi.go                     |  2 ++
 relayapi/routing/routing.go                      |  2 ++
 roomserver/internal/input/input.go               | 16 +++++++++-------
 roomserver/internal/input/input_latest_events.go |  1 +
 5 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index dc7a363e7..6328d165e 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -647,6 +647,8 @@ func MakeFedAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetTag("origin", string(fedReq.Origin()))
 			hub.Scope().SetTag("uri", fedReq.RequestURI())
 		}
diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index 1966e7546..c78aadf89 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -76,6 +76,8 @@ func MakeAuthAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetUser(sentry.User{
 				Username: device.UserID,
 			})
diff --git a/relayapi/routing/routing.go b/relayapi/routing/routing.go
index f11b0a7c5..92476d6c2 100644
--- a/relayapi/routing/routing.go
+++ b/relayapi/routing/routing.go
@@ -108,6 +108,8 @@ func MakeRelayAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetTag("origin", string(fedReq.Origin()))
 			hub.Scope().SetTag("uri", fedReq.RequestURI())
 		}
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 20d2cfc7a..104ce94e5 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -108,12 +108,14 @@ type worker struct {
 	r            *Inputer
 	roomID       string
 	subscription *nats.Subscription
+	sentryHub    *sentry.Hub
 }
 
 func (r *Inputer) startWorkerForRoom(roomID string) {
 	v, loaded := r.workers.LoadOrStore(roomID, &worker{
-		r:      r,
-		roomID: roomID,
+		r:         r,
+		roomID:    roomID,
+		sentryHub: sentry.CurrentHub().Clone(),
 	})
 	w := v.(*worker)
 	w.Lock()
@@ -265,9 +267,9 @@ func (w *worker) _next() {
 	// Look up what the next event is that's waiting to be processed.
 	ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute)
 	defer cancel()
-	if scope := sentry.CurrentHub().Scope(); scope != nil {
+	w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
 		scope.SetTag("room_id", w.roomID)
-	}
+	})
 	msgs, err := w.subscription.Fetch(1, nats.Context(ctx))
 	switch err {
 	case nil:
@@ -323,9 +325,9 @@ func (w *worker) _next() {
 		return
 	}
 
-	if scope := sentry.CurrentHub().Scope(); scope != nil {
+	w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
 		scope.SetTag("event_id", inputRoomEvent.Event.EventID())
-	}
+	})
 
 	// Process the room event. If something goes wrong then we'll tell
 	// NATS to terminate the message. We'll store the error result as
@@ -347,7 +349,7 @@ func (w *worker) _next() {
 			}).Warn("Roomserver rejected event")
 		default:
 			if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
-				sentry.CaptureException(err)
+				w.sentryHub.CaptureException(err)
 			}
 			logrus.WithError(err).WithFields(logrus.Fields{
 				"room_id":  w.roomID,
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index ec03d6f13..e9856cc5d 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -298,6 +298,7 @@ func (u *latestEventsUpdater) latestState() error {
 		}).Warnf("State reset detected (removing %d events)", removed)
 		sentry.WithScope(func(scope *sentry.Scope) {
 			scope.SetLevel("warning")
+			scope.SetTag("room_id", u.event.RoomID().String())
 			scope.SetContext("State reset", map[string]interface{}{
 				"Event ID":      u.event.EventID(),
 				"Old state NID": fmt.Sprintf("%d", u.oldStateNID),

From 00217a69d12aa65865966601504060f3b422dc93 Mon Sep 17 00:00:00 2001
From: Joakim Recht <recht@braindump.dk>
Date: Thu, 25 Jan 2024 20:10:46 +0100
Subject: [PATCH 02/46] Only fetch events once for all rooms (#3311)

This refactors `PDUStreamProvider` a bit so that it doesn't trigger a
database query per room, but instead utilizes the fact that it's
possible to bulk query. This improves sync performance significantly
when you have 1000s of rooms.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `Joakim Recht <joakim@beyondwork.ai>`
---
 syncapi/streams/stream_pdu.go | 94 ++++++++++++++++++++++++++---------
 1 file changed, 70 insertions(+), 24 deletions(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 3abb0b3c6..790f5bd1b 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -203,6 +203,12 @@ func (p *PDUStreamProvider) IncrementalSync(
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
 	}
 
+	dbEvents, err := p.getRecentEvents(ctx, stateDeltas, r, eventFilter, snapshot)
+	if err != nil {
+		req.Log.WithError(err).Error("unable to get recent events")
+		return r.From
+	}
+
 	newPos = from
 	for _, delta := range stateDeltas {
 		newRange := r
@@ -218,7 +224,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 			}
 		}
 		var pos types.StreamPosition
-		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req); err != nil {
+		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req, dbEvents); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
 			if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone {
 				return newPos
@@ -240,6 +246,66 @@ func (p *PDUStreamProvider) IncrementalSync(
 	return newPos
 }
 
+func (p *PDUStreamProvider) getRecentEvents(ctx context.Context, stateDeltas []types.StateDelta, r types.Range, eventFilter synctypes.RoomEventFilter, snapshot storage.DatabaseTransaction) (map[string]types.RecentEvents, error) {
+	var roomIDs []string
+	var newlyJoinedRoomIDs []string
+	for _, delta := range stateDeltas {
+		if delta.NewlyJoined {
+			newlyJoinedRoomIDs = append(newlyJoinedRoomIDs, delta.RoomID)
+		} else {
+			roomIDs = append(roomIDs, delta.RoomID)
+		}
+	}
+	dbEvents := make(map[string]types.RecentEvents)
+	if len(roomIDs) > 0 {
+		events, err := snapshot.RecentEvents(
+			ctx, roomIDs, r,
+			&eventFilter, true, true,
+		)
+		if err != nil {
+			if err != sql.ErrNoRows {
+				return nil, err
+			}
+		}
+		for k, v := range events {
+			dbEvents[k] = v
+		}
+	}
+	if len(newlyJoinedRoomIDs) > 0 {
+		// For rooms that were joined in this sync, try to fetch
+		// as much timeline events as allowed by the filter.
+
+		filter := eventFilter
+		// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
+		if eventFilter.Limit < recentEventBackwardsLimit {
+			filter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
+			diff := r.From - r.To
+			if diff > 0 && diff < recentEventBackwardsLimit {
+				filter.Limit = int(diff)
+			}
+		}
+
+		events, err := snapshot.RecentEvents(
+			ctx, newlyJoinedRoomIDs, types.Range{
+				From:      r.To,
+				To:        0,
+				Backwards: true,
+			},
+			&filter, true, true,
+		)
+		if err != nil {
+			if err != sql.ErrNoRows {
+				return nil, err
+			}
+		}
+		for k, v := range events {
+			dbEvents[k] = v
+		}
+	}
+
+	return dbEvents, nil
+}
+
 // Limit the recent events to X when going backwards
 const recentEventBackwardsLimit = 100
 
@@ -253,29 +319,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	eventFilter *synctypes.RoomEventFilter,
 	stateFilter *synctypes.StateFilter,
 	req *types.SyncRequest,
+	dbEvents map[string]types.RecentEvents,
 ) (types.StreamPosition, error) {
 	var err error
-	originalLimit := eventFilter.Limit
-	// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
-	if r.Backwards && originalLimit < recentEventBackwardsLimit {
-		eventFilter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
-		diff := r.From - r.To
-		if diff > 0 && diff < recentEventBackwardsLimit {
-			eventFilter.Limit = int(diff)
-		}
-	}
-
-	dbEvents, err := snapshot.RecentEvents(
-		ctx, []string{delta.RoomID}, r,
-		eventFilter, true, true,
-	)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return r.To, nil
-		}
-		return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err)
-	}
-
 	recentStreamEvents := dbEvents[delta.RoomID].Events
 	limited := dbEvents[delta.RoomID].Limited
 
@@ -337,9 +383,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 	}
 
-	if r.Backwards && len(events) > originalLimit {
+	if r.Backwards && len(events) > eventFilter.Limit {
 		// We're going backwards and the events are ordered chronologically, so take the last `limit` events
-		events = events[len(events)-originalLimit:]
+		events = events[len(events)-eventFilter.Limit:]
 		limited = true
 	}
 

From a4817f31c066fe82550f285d8662befac2468030 Mon Sep 17 00:00:00 2001
From: Matthew Strapp <matt@mattstrapp.net>
Date: Thu, 25 Jan 2024 14:17:20 -0600
Subject: [PATCH 03/46] Allow + in MIDs as per MSC4009 (#3313)

This PR adds `+` to the username regex, per MSC4009.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `Matt Strapp <matt@mattstrapp.net>`
---
 internal/validate.go      |  4 ++--
 internal/validate_test.go | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/internal/validate.go b/internal/validate.go
index da8b35cd3..c831565f5 100644
--- a/internal/validate.go
+++ b/internal/validate.go
@@ -38,9 +38,9 @@ var (
 	ErrPasswordTooLong    = fmt.Errorf("password too long: max %d characters", maxPasswordLength)
 	ErrPasswordWeak       = fmt.Errorf("password too weak: min %d characters", minPasswordLength)
 	ErrUsernameTooLong    = fmt.Errorf("username exceeds the maximum length of %d characters", maxUsernameLength)
-	ErrUsernameInvalid    = errors.New("username can only contain characters a-z, 0-9, or '_-./='")
+	ErrUsernameInvalid    = errors.New("username can only contain characters a-z, 0-9, or '_+-./='")
 	ErrUsernameUnderscore = errors.New("username cannot start with a '_'")
-	validUsernameRegex    = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
+	validUsernameRegex    = regexp.MustCompile(`^[0-9a-z_\-+=./]+$`)
 )
 
 // ValidatePassword returns an error if the password is invalid
diff --git a/internal/validate_test.go b/internal/validate_test.go
index cd2626133..1019102df 100644
--- a/internal/validate_test.go
+++ b/internal/validate_test.go
@@ -129,6 +129,11 @@ func Test_validateUsername(t *testing.T) {
 			localpart: "i_am_allowed=1",
 			domain:    "localhost",
 		},
+		{
+			name:      "special characters are allowed 3",
+			localpart: "+55555555555",
+			domain:    "localhost",
+		},
 		{
 			name:      "not all special characters are allowed",
 			localpart: "notallowed#", // contains #
@@ -139,6 +144,16 @@ func Test_validateUsername(t *testing.T) {
 				JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
 			},
 		},
+		{
+			name:      "not all special characters are allowed 2",
+			localpart: "<notallowed", // contains <
+			domain:    "localhost",
+			wantErr:   ErrUsernameInvalid,
+			wantJSON: &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
+			},
+		},
 		{
 			name:      "username containing numbers",
 			localpart: "hello1337",

From 8f68f1ff539b54f82a1fa1026e863a4076d4c9f4 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 25 Jan 2024 21:35:05 +0100
Subject: [PATCH 04/46] Move `/joined_members` back to the clientapi/roomserver
  (#3312)

Partly reverts #2827 by moving `/joined_members` back to the
clientAPI/roomserver
---
 clientapi/clientapi_test.go      | 127 ++++++++++++++++++++++++++++
 clientapi/routing/memberships.go | 139 +++++++++++++++++++++++++++++++
 clientapi/routing/routing.go     |  10 +++
 syncapi/routing/memberships.go   |  62 +-------------
 syncapi/routing/routing.go       |  14 +---
 syncapi/syncapi_test.go          |  33 --------
 6 files changed, 278 insertions(+), 107 deletions(-)
 create mode 100644 clientapi/routing/memberships.go

diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index 2ff4b6503..1b2f1358b 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -2151,3 +2151,130 @@ func TestKeyBackup(t *testing.T) {
 		}
 	})
 }
+
+func TestGetMembership(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	testCases := []struct {
+		name             string
+		roomID           string
+		user             *test.User
+		additionalEvents func(t *testing.T, room *test.Room)
+		request          func(t *testing.T, room *test.Room, accessToken string) *http.Request
+		wantOK           bool
+		wantMemberCount  int
+	}{
+
+		{
+			name: "/joined_members - Bob never joined",
+			user: bob,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Alice joined",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK:          true,
+			wantMemberCount: 1,
+		},
+		{
+			name: "/joined_members - Alice leaves, shouldn't be able to see members ",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+					"membership": "leave",
+				}, test.WithStateKey(alice.ID))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Bob joins, Alice sees two members",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+					"membership": "join",
+				}, test.WithStateKey(bob.ID))
+			},
+			wantOK:          true,
+			wantMemberCount: 2,
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				room := test.NewRoom(t, alice)
+				t.Cleanup(func() {
+					t.Logf("running cleanup for %s", tc.name)
+				})
+				// inject additional events
+				if tc.additionalEvents != nil {
+					tc.additionalEvents(t, room)
+				}
+				if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+					t.Fatalf("failed to send events: %v", err)
+				}
+
+				w := httptest.NewRecorder()
+				routers.Client.ServeHTTP(w, tc.request(t, room, accessTokens[tc.user].accessToken))
+				if w.Code != 200 && tc.wantOK {
+					t.Logf("%s", w.Body.String())
+					t.Fatalf("got HTTP %d want %d", w.Code, 200)
+				}
+				t.Logf("[%s] Resp: %s", tc.name, w.Body.String())
+
+				// check we got the expected events
+				if tc.wantOK {
+					memberCount := len(gjson.GetBytes(w.Body.Bytes(), "joined").Map())
+					if memberCount != tc.wantMemberCount {
+						t.Fatalf("expected %d members, got %d", tc.wantMemberCount, memberCount)
+					}
+				}
+			})
+		}
+	})
+}
diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go
new file mode 100644
index 000000000..84be498d6
--- /dev/null
+++ b/clientapi/routing/memberships.go
@@ -0,0 +1,139 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// The database stores 'displayname' without an underscore.
+// Deserialize into this and then change to the actual API response
+type databaseJoinedMember struct {
+	DisplayName string `json:"displayname"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// GetJoinedMembers implements
+//
+//	GET /rooms/{roomId}/joined_members
+func GetJoinedMembers(
+	req *http.Request, device *userapi.Device, roomID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	// Validate the userID
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("Device UserID is invalid"),
+		}
+	}
+
+	// Validate the roomID
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}
+	}
+
+	// Get the current memberships for the requesting user to determine
+	// if they are allowed to query this endpoint.
+	queryReq := api.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}
+
+	var queryRes api.QueryMembershipForUserResponse
+	if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil {
+		util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	if !queryRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	if !queryRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	// Get the current membership events
+	var membershipsForRoomResp api.QueryMembershipsForRoomResponse
+	if err = rsAPI.QueryMembershipsForRoom(req.Context(), &api.QueryMembershipsForRoomRequest{
+		JoinedOnly: true,
+		RoomID:     validRoomID.String(),
+	}, &membershipsForRoomResp); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	var res getJoinedMembersResponse
+	res.Joined = make(map[string]joinedMember)
+	for _, ev := range membershipsForRoomResp.JoinEvents {
+		var content databaseJoinedMember
+		if err := json.Unmarshal(ev.Content, &content); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(ev.Sender))
+		if err != nil || userID == nil {
+			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		res.Joined[userID.String()] = joinedMember(content)
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index d4aa1d08d..3e23ab405 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -1513,4 +1513,14 @@ func Setup(
 			return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/joined_members",
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetJoinedMembers(req, device, vars["roomID"], rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }
diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go
index e849adf6d..9cc937d88 100644
--- a/syncapi/routing/memberships.go
+++ b/syncapi/routing/memberships.go
@@ -15,7 +15,6 @@
 package routing
 
 import (
-	"encoding/json"
 	"math"
 	"net/http"
 
@@ -33,31 +32,13 @@ type getMembershipResponse struct {
 	Chunk []synctypes.ClientEvent `json:"chunk"`
 }
 
-// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
-type getJoinedMembersResponse struct {
-	Joined map[string]joinedMember `json:"joined"`
-}
-
-type joinedMember struct {
-	DisplayName string `json:"display_name"`
-	AvatarURL   string `json:"avatar_url"`
-}
-
-// The database stores 'displayname' without an underscore.
-// Deserialize into this and then change to the actual API response
-type databaseJoinedMember struct {
-	DisplayName string `json:"displayname"`
-	AvatarURL   string `json:"avatar_url"`
-}
-
 // GetMemberships implements
 //
 //	GET /rooms/{roomId}/members
-//	GET /rooms/{roomId}/joined_members
 func GetMemberships(
 	req *http.Request, device *userapi.Device, roomID string,
 	syncDB storage.Database, rsAPI api.SyncRoomserverAPI,
-	joinedOnly bool, membership, notMembership *string, at string,
+	membership, notMembership *string, at string,
 ) util.JSONResponse {
 	userID, err := spec.NewUserID(device.UserID, true)
 	if err != nil {
@@ -87,13 +68,6 @@ func GetMemberships(
 		}
 	}
 
-	if joinedOnly && !queryRes.IsInRoom {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
-		}
-	}
-
 	db, err := syncDB.NewDatabaseSnapshot(req.Context())
 	if err != nil {
 		return util.JSONResponse{
@@ -139,40 +113,6 @@ func GetMemberships(
 
 	result := qryRes.Events
 
-	if joinedOnly {
-		var res getJoinedMembersResponse
-		res.Joined = make(map[string]joinedMember)
-		for _, ev := range result {
-			var content databaseJoinedMember
-			if err := json.Unmarshal(ev.Content(), &content); err != nil {
-				util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
-				return util.JSONResponse{
-					Code: http.StatusInternalServerError,
-					JSON: spec.InternalServerError{},
-				}
-			}
-
-			userID, err := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID())
-			if err != nil || userID == nil {
-				util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
-				return util.JSONResponse{
-					Code: http.StatusInternalServerError,
-					JSON: spec.InternalServerError{},
-				}
-			}
-			if err != nil {
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"),
-				}
-			}
-			res.Joined[userID.String()] = joinedMember(content)
-		}
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: res,
-		}
-	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: getMembershipResponse{synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(result), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index a837e1696..78188d1b6 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -197,19 +197,7 @@ func Setup(
 			}
 
 			at := req.URL.Query().Get("at")
-			return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, false, membership, notMembership, at)
+			return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, membership, notMembership, at)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodGet, http.MethodOptions)
-
-	v3mux.Handle("/rooms/{roomID}/joined_members",
-		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			at := req.URL.Query().Get("at")
-			membership := spec.Join
-			return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, true, &membership, nil, at)
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
 }
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index ac5268511..0a2c38ab7 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -753,24 +753,6 @@ func TestGetMembership(t *testing.T) {
 			},
 			wantOK: false,
 		},
-		{
-			name: "/joined_members - Bob never joined",
-			request: func(t *testing.T, room *test.Room) *http.Request {
-				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
-					"access_token": bobDev.AccessToken,
-				}))
-			},
-			wantOK: false,
-		},
-		{
-			name: "/joined_members - Alice joined",
-			request: func(t *testing.T, room *test.Room) *http.Request {
-				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
-					"access_token": aliceDev.AccessToken,
-				}))
-			},
-			wantOK: true,
-		},
 		{
 			name: "Alice leaves before Bob joins, should not be able to see Bob",
 			request: func(t *testing.T, room *test.Room) *http.Request {
@@ -809,21 +791,6 @@ func TestGetMembership(t *testing.T) {
 			wantOK:          true,
 			wantMemberCount: 2,
 		},
-		{
-			name: "/joined_members - Alice leaves, shouldn't be able to see members ",
-			request: func(t *testing.T, room *test.Room) *http.Request {
-				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
-					"access_token": aliceDev.AccessToken,
-				}))
-			},
-			additionalEvents: func(t *testing.T, room *test.Room) {
-				room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
-					"membership": "leave",
-				}, test.WithStateKey(alice.ID))
-			},
-			useSleep: true,
-			wantOK:   false,
-		},
 		{
 			name: "'at' specified, returns memberships before Bob joins",
 			request: func(t *testing.T, room *test.Room) *http.Request {

From 87f028db27e62cca71dadc5988dc8d37f5f20af4 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 26 Jan 2024 14:41:34 +0100
Subject: [PATCH 05/46] Version 0.13.6 (#3315)

---
 .github/workflows/k8s.yml |  2 +-
 CHANGES.md                | 21 +++++++++++++++++++++
 helm/dendrite/Chart.yaml  |  4 ++--
 helm/dendrite/README.md   |  2 +-
 internal/version.go       |  2 +-
 5 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml
index af2750356..6e2533d9a 100644
--- a/.github/workflows/k8s.yml
+++ b/.github/workflows/k8s.yml
@@ -66,7 +66,7 @@ jobs:
       - name: Create k3d cluster
         uses: nolar/setup-k3d-k3s@v1
         with:
-          version: v1.21
+          version: v1.28
       - name: Remove node taints
         run: |
           kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
diff --git a/CHANGES.md b/CHANGES.md
index 97ec7bec4..c41025e91 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,26 @@
 # Changelog
 
+## Dendrite 0.13.6 (2024-01-26)
+
+Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
+
+### Fixes
+
+- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS
+- A rare panic when assigning a state key NID has been fixed
+- A rare panic when checking powerlevels has been fixed
+- Notary keys requests for all keys now work correctly
+- Spec compliance:
+  - Return `M_INVALID_PARAM` when querying room aliases
+  - Handle empty `from` parameter when requesting `/messages`
+  - Add CORP headers on media endpoints
+  - Remove `aliases` from `/publicRooms` responses
+  - Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss))
+- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms
+- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht))
+- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again
+- Backfilling from other servers now only uses at max 100 events instead of potentially thousands
+
 ## Dendrite 0.13.5 (2023-12-12)
 
 Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in
diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index f36f457c5..e28261c8e 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.6"
-appVersion: "0.13.5"
+version: "0.13.7"
+appVersion: "0.13.6"
 description: Dendrite Matrix Homeserver
 type: application
 keywords:
diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index f5f824927..feaa543dd 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.6](https://img.shields.io/badge/Version-0.13.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.5](https://img.shields.io/badge/AppVersion-0.13.5-informational?style=flat-square)
+![Version: 0.13.7](https://img.shields.io/badge/Version-0.13.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
diff --git a/internal/version.go b/internal/version.go
index 3fc52e376..1a7575090 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -18,7 +18,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 13
-	VersionPatch = 5
+	VersionPatch = 6
 	VersionTag   = "" // example: "rc1"
 
 	gitRevLen = 7 // 7 matches the displayed characters on github.com

From a3a18fbcce29c84eb96c3db9c723da951e74ba91 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 29 Jan 2024 20:44:43 +0100
Subject: [PATCH 06/46] Fix x86 tests (#3317)

x86 tests broke with #3298
(Not exactly the tests modified here, but
`TestMessageHistoryVisibility`)
---
 syncapi/types/types.go      |  8 ++++----
 syncapi/types/types_test.go | 17 +++++++++++++++++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index bca11855c..26faf7c05 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -286,8 +286,8 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) {
 		if i > len(positions) {
 			break
 		}
-		var pos int
-		pos, err = strconv.Atoi(p)
+		var pos int64
+		pos, err = strconv.ParseInt(p, 10, 64)
 		if err != nil {
 			return
 		}
@@ -318,8 +318,8 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 		if i >= len(positions) {
 			break
 		}
-		var pos int
-		pos, err = strconv.Atoi(p)
+		var pos int64
+		pos, err = strconv.ParseInt(p, 10, 64)
 		if err != nil {
 			err = ErrMalformedSyncToken
 			return
diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go
index 35e1882cb..6c616ab0d 100644
--- a/syncapi/types/types_test.go
+++ b/syncapi/types/types_test.go
@@ -3,6 +3,7 @@ package types
 import (
 	"context"
 	"encoding/json"
+	"math"
 	"reflect"
 	"testing"
 
@@ -33,12 +34,28 @@ func TestSyncTokens(t *testing.T) {
 		"s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(),
 		"s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(),
 		"t3_1":               TopologyToken{3, 1}.String(),
+		"t9223372036854775807_9223372036854775807": TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64}.String(),
+		"s9223372036854775807_1_2_3_5_0_0_0_6":     StreamingToken{math.MaxInt64, 1, 2, 3, 5, 0, 0, 0, 6}.String(),
 	}
 
 	for a, b := range shouldPass {
 		if a != b {
 			t.Errorf("expected %q, got %q", a, b)
 		}
+
+		// parse as topology token
+		if a[0] == 't' {
+			if _, err := NewTopologyTokenFromString(a); err != nil {
+				t.Errorf("expected %q to pass, but got %q", a, err)
+			}
+		}
+
+		// parse as sync token
+		if a[0] == 's' {
+			if _, err := NewStreamTokenFromString(a); err != nil {
+				t.Errorf("expected %q to pass, but got %q", a, err)
+			}
+		}
 	}
 
 	shouldFail := []string{

From 0f6b81f4560855d57550452463cc8d8b91de9ff2 Mon Sep 17 00:00:00 2001
From: Tulir Asokan <tulir@maunium.net>
Date: Sat, 3 Feb 2024 18:56:13 +0100
Subject: [PATCH 07/46] Modernize appservice paths and authentication (#3316)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This brings Dendrite's appservice spec support up to v1.4, from the
previous level of pre-release-spec support only (even r0.1.0 wasn't
supported for pushing transactions 🙃). There are config options to
revert to the old behavior, but the default is v1.4+ only. [Synapse also
does
that](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#use_appservice_legacy_authorization)

mautrix bridges will drop support for legacy paths and authentication
soon (and possibly also require matrix v1.4 to be advertised, but I
might add some workaround to not require that for dendrite)

Signed-off-by: Tulir Asokan <tulir@maunium.net>
---
 appservice/api/query.go            | 14 ++++-
 appservice/consumers/roomserver.go | 12 +++-
 appservice/query/query.go          | 98 +++++++++++++++++++-----------
 dendrite-sample.yaml               |  7 +++
 setup/config/config_appservice.go  |  3 +
 5 files changed, 95 insertions(+), 39 deletions(-)

diff --git a/appservice/api/query.go b/appservice/api/query.go
index 472266d9e..8e159152e 100644
--- a/appservice/api/query.go
+++ b/appservice/api/query.go
@@ -82,9 +82,17 @@ type UserIDExistsResponse struct {
 }
 
 const (
-	ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
-	ASUserPath     = "/_matrix/app/unstable/thirdparty/user"
-	ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
+	ASProtocolLegacyPath        = "/_matrix/app/unstable/thirdparty/protocol/"
+	ASUserLegacyPath            = "/_matrix/app/unstable/thirdparty/user"
+	ASLocationLegacyPath        = "/_matrix/app/unstable/thirdparty/location"
+	ASRoomAliasExistsLegacyPath = "/rooms/"
+	ASUserExistsLegacyPath      = "/users/"
+
+	ASProtocolPath        = "/_matrix/app/v1/thirdparty/protocol/"
+	ASUserPath            = "/_matrix/app/v1/thirdparty/user"
+	ASLocationPath        = "/_matrix/app/v1/thirdparty/location"
+	ASRoomAliasExistsPath = "/_matrix/app/v1/rooms/"
+	ASUserExistsPath      = "/_matrix/app/v1/users/"
 )
 
 type ProtocolRequest struct {
diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index b7fc1f698..b07b24fcc 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -206,13 +206,21 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	}
 
 	// Send the transaction to the appservice.
-	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken))
+	// https://spec.matrix.org/v1.9/application-service-api/#pushing-events
+	path := "_matrix/app/v1/transactions"
+	if s.cfg.LegacyPaths {
+		path = "transactions"
+	}
+	address := fmt.Sprintf("%s/%s/%s", state.RequestUrl(), path, txnID)
+	if s.cfg.LegacyAuth {
+		address += "?access_token=" + url.QueryEscape(state.HSToken)
+	}
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", state.HSToken))
 	resp, err := state.HTTPClient.Do(req)
 	if err != nil {
 		return state.backoffAndPause(err)
diff --git a/appservice/query/query.go b/appservice/query/query.go
index 5c736f379..7f33e17f8 100644
--- a/appservice/query/query.go
+++ b/appservice/query/query.go
@@ -19,10 +19,10 @@ package query
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"strings"
 	"sync"
 
 	log "github.com/sirupsen/logrus"
@@ -32,9 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-const roomAliasExistsPath = "/rooms/"
-const userIDExistsPath = "/users/"
-
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
 	Cfg           *config.AppServiceAPI
@@ -55,14 +52,23 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
+			path := api.ASRoomAliasExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASRoomAliasExistsLegacyPath
+			}
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 
 			URL.Path += request.Alias
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the room, immediately return.
@@ -70,6 +76,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			req = req.WithContext(ctx)
 
 			resp, err := appservice.HTTPClient.Do(req)
@@ -123,12 +130,21 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath)
+			path := api.ASUserExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASUserExistsLegacyPath
+			}
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 			URL.Path += request.UserID
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the user, immediately return.
@@ -136,6 +152,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
@@ -176,25 +193,22 @@ type thirdpartyResponses interface {
 	api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
 }
 
-func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
-	origURL := url
-	// try v1 and unstable appservice endpoints
-	for _, version := range []string{"v1", "unstable"} {
-		var resp *http.Response
-		var body []byte
-		asURL := strings.Replace(origURL, "unstable", version, 1)
-		resp, err = client.Get(asURL)
-		if err != nil {
-			continue
-		}
-		defer resp.Body.Close() // nolint: errcheck
-		body, err = io.ReadAll(resp.Body)
-		if err != nil {
-			continue
-		}
-		return json.Unmarshal(body, &response)
+func requestDo[T thirdpartyResponses](as *config.ApplicationService, url string, response *T) error {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return err
 	}
-	return err
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", as.HSToken))
+	resp, err := as.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close() // nolint: errcheck
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(body, &response)
 }
 
 func (a *AppServiceQueryAPI) Locations(
@@ -207,16 +221,22 @@ func (a *AppServiceQueryAPI) Locations(
 		return err
 	}
 
+	path := api.ASLocationPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASLocationLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asLocations []api.ASLocationResponse
-		params.Set("access_token", as.HSToken)
+		if a.Cfg.LegacyAuth {
+			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASLocationPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+		if err := requestDo[[]api.ASLocationResponse](&as, url+"?"+params.Encode(), &asLocations); err != nil {
 			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
 			continue
 		}
@@ -242,16 +262,22 @@ func (a *AppServiceQueryAPI) User(
 		return err
 	}
 
+	path := api.ASUserPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASUserLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asUsers []api.ASUserResponse
-		params.Set("access_token", as.HSToken)
+		if a.Cfg.LegacyAuth {
+			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASUserPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+		if err := requestDo[[]api.ASUserResponse](&as, url+"?"+params.Encode(), &asUsers); err != nil {
 			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
 			continue
 		}
@@ -272,6 +298,10 @@ func (a *AppServiceQueryAPI) Protocols(
 	req *api.ProtocolRequest,
 	resp *api.ProtocolResponse,
 ) error {
+	protocolPath := api.ASProtocolPath
+	if a.Cfg.LegacyPaths {
+		protocolPath = api.ASProtocolLegacyPath
+	}
 
 	// get a single protocol response
 	if req.Protocol != "" {
@@ -289,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		response := api.ASProtocolResponse{}
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+req.Protocol, &proto); err != nil {
 				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
@@ -319,7 +349,7 @@ func (a *AppServiceQueryAPI) Protocols(
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+p, &proto); err != nil {
 				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml
index e143a7398..8616e120e 100644
--- a/dendrite-sample.yaml
+++ b/dendrite-sample.yaml
@@ -154,6 +154,13 @@ app_service_api:
   # to be sent to an insecure endpoint.
   disable_tls_validation: false
 
+  # Send the access_token query parameter with appservice requests in addition
+  # to the Authorization header. This can cause hs_tokens to be saved to logs,
+  # so it should not be enabled unless absolutely necessary.
+  legacy_auth: false
+  # Use the legacy unprefixed paths for appservice requests.
+  legacy_paths: false
+
   # Appservice configuration files to load into this homeserver.
   config_files:
   #  - /path/to/appservice_registration.yaml
diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go
index ef10649d2..a95cec046 100644
--- a/setup/config/config_appservice.go
+++ b/setup/config/config_appservice.go
@@ -40,6 +40,9 @@ type AppServiceAPI struct {
 	// on appservice endpoints. This is not recommended in production!
 	DisableTLSValidation bool `yaml:"disable_tls_validation"`
 
+	LegacyAuth  bool `yaml:"legacy_auth"`
+	LegacyPaths bool `yaml:"legacy_paths"`
+
 	ConfigFiles []string `yaml:"config_files"`
 }
 

From 436773ab71463dae030e8a3ba21a32f8fd78655c Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 7 Feb 2024 10:28:10 +0100
Subject: [PATCH 08/46] Disable Element Web tests, only run csapi and
 federation tests

---
 .github/workflows/schedules.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml
index 509861860..b51d234c8 100644
--- a/.github/workflows/schedules.yaml
+++ b/.github/workflows/schedules.yaml
@@ -174,7 +174,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:
@@ -220,6 +220,7 @@ jobs:
           fail_ci_if_error: true
 
   element-web:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -259,6 +260,7 @@ jobs:
           TMPDIR: ${{ runner.temp }}
 
   element-web-pinecone:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:

From be0c27e68865f031e59a091fcec5244a7b48da44 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 8 Feb 2024 09:58:59 +0100
Subject: [PATCH 09/46] Update all the CI actions (#3323)

Also adds a job for the scheduled CI run to only run if there has been a
commit in the last 24h
([StackOverflow](https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where))

[skip ci]
---
 .github/workflows/dendrite.yml   | 71 ++++++++++++++++++++------------
 .github/workflows/docker.yml     | 30 +++++++-------
 .github/workflows/gh-pages.yml   |  2 +-
 .github/workflows/helm.yml       |  2 +-
 .github/workflows/k8s.yml        |  4 +-
 .github/workflows/schedules.yaml | 54 +++++++++++++++++-------
 6 files changed, 101 insertions(+), 62 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index ac40f06b0..5edb1924e 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ false }} # disable for now
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
@@ -41,7 +41,7 @@ jobs:
         with:
           node-version: 14
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -66,11 +66,11 @@ jobs:
     name: Linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: golangci-lint
@@ -102,14 +102,14 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
         with:
           path: |
@@ -141,12 +141,12 @@ jobs:
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -174,12 +174,12 @@ jobs:
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -235,11 +235,11 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: Set up gotestfmt
@@ -247,7 +247,7 @@ jobs:
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -262,10 +262,11 @@ jobs:
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: dendrite
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           flags: unittests
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run database upgrade tests
   upgrade_test:
@@ -274,12 +275,20 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-test-
       - name: Docker version
         run: docker version
       - name: Build upgrade-tests
@@ -296,12 +305,20 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-direct-test-
       - name: Docker version
         run: docker version
       - name: Build upgrade-tests
@@ -340,8 +357,8 @@ jobs:
         SYTEST_BRANCH: ${{ github.head_ref }}
         CGO_ENABLED: ${{ matrix.cgo && 1 }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -364,7 +381,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
@@ -404,8 +421,8 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v4 for dendrite
+        uses: actions/checkout@v4
         with:
           path: dendrite
 
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 8d3a8d674..c795cd366 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -27,22 +27,22 @@ jobs:
       security-events: write # To upload Trivy sarif files
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -98,22 +98,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -159,22 +159,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
index 9df3cceae..30f55b7c8 100644
--- a/.github/workflows/gh-pages.yml
+++ b/.github/workflows/gh-pages.yml
@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Build with Jekyll
diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml
index 9a5eb2b62..d4772e106 100644
--- a/.github/workflows/helm.yml
+++ b/.github/workflows/helm.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml
index 6e2533d9a..a49042bf2 100644
--- a/.github/workflows/k8s.yml
+++ b/.github/workflows/k8s.yml
@@ -17,7 +17,7 @@ jobs:
     outputs:
       changed: ${{ steps.list-changed.outputs.changed }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: azure/setup-helm@v3
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ inputs.checkoutCommit }}
diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml
index b51d234c8..4ba7c2e85 100644
--- a/.github/workflows/schedules.yaml
+++ b/.github/workflows/schedules.yaml
@@ -10,8 +10,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
+    runs-on: ubuntu-latest
+    name: Check latest commit
+    outputs:
+      should_run: ${{ steps.should_run.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: print latest_commit
+        run: echo ${{ github.sha }}
+
+      - id: should_run
+        continue-on-error: true
+        name: check latest commit is less than a day
+        if: ${{ github.event_name == 'schedule' }}
+        run: test -z $(git rev-list  --after="24 hours"  ${{ github.sha }}) && echo "::set-output name=should_run::false"
+
   # run Sytest in different variations
   sytest:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     timeout-minutes: 60
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
@@ -38,8 +56,8 @@ jobs:
         RACE_DETECTION: 1
         COVER: 1
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -62,7 +80,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -78,28 +96,31 @@ jobs:
     needs: sytest # only run once Sytest is done
     if: ${{ always() }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
           go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: sytest
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run Complement
   complement:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     name: "Complement (${{ matrix.label }})"
     timeout-minutes: 60
     runs-on: ubuntu-latest
@@ -129,8 +150,8 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v4 for dendrite
+        uses: actions/checkout@v4
         with:
           path: dendrite
 
@@ -185,7 +206,7 @@ jobs:
         working-directory: complement
 
       - name: Upload Complement logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -199,25 +220,26 @@ jobs:
     needs: complement # only run once Complement is done
     if: ${{ always() }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
           go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: complement
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }} # required
 
   element-web:
     if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
@@ -229,7 +251,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3
@@ -269,7 +291,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3

From e9deb5244edd4827234622ea0d3b11678a33b0ac Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 13 Feb 2024 19:28:52 +0100
Subject: [PATCH 10/46] Fix `/createRoom` and `/invite` containing
 displayname/avatarURL of inviter (#3326)

Fixes #3324
---
 .github/workflows/schedules.yaml              |  8 +--
 clientapi/clientapi_test.go                   | 65 +++++++++++++++++++
 clientapi/routing/membership.go               | 31 ++++-----
 clientapi/routing/server_notices.go           |  2 +-
 roomserver/api/perform.go                     | 18 +++--
 .../internal/perform/perform_create_room.go   | 18 +++--
 roomserver/internal/perform/perform_invite.go |  8 +--
 7 files changed, 100 insertions(+), 50 deletions(-)

diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml
index 4ba7c2e85..e339c14d3 100644
--- a/.github/workflows/schedules.yaml
+++ b/.github/workflows/schedules.yaml
@@ -93,8 +93,8 @@ jobs:
     timeout-minutes: 5
     name: "Sytest Coverage"
     runs-on: ubuntu-latest
-    needs: sytest # only run once Sytest is done
-    if: ${{ always() }}
+    needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
       - uses: actions/checkout@v4
       - name: Install Go
@@ -217,8 +217,8 @@ jobs:
     timeout-minutes: 5
     name: "Complement Coverage"
     runs-on: ubuntu-latest
-    needs: complement # only run once Complement is done
-    if: ${{ always() }}
+    needs: [ complement, check_date ] # only run once Complements is done and there was a commit
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
       - uses: actions/checkout@v4
       - name: Install Go
diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index 1b2f1358b..29ff58665 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -2278,3 +2278,68 @@ func TestGetMembership(t *testing.T) {
 		}
 	})
 }
+
+func TestCreateRoomInvite(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"invite": []string{bob.ID},
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/createRoom", strings.NewReader(string(body)))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+		routers.Client.ServeHTTP(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("expected room creation to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+		}
+
+		roomID := gjson.GetBytes(w.Body.Bytes(), "room_id").Str
+		validRoomID, _ := spec.NewRoomID(roomID)
+		// Now ask the roomserver about the membership event of Bob
+		ev, err := rsAPI.CurrentStateEvent(context.Background(), *validRoomID, spec.MRoomMember, bob.ID)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if ev == nil {
+			t.Fatal("Membership event for Bob does not exist")
+		}
+
+		// Validate that there is NO displayname in content
+		if gjson.GetBytes(ev.Content(), "displayname").Exists() {
+			t.Fatal("Found displayname in invite")
+		}
+	})
+}
diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index 06683c47d..9e41a3794 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -324,19 +324,18 @@ func SendInvite(
 	}
 
 	// We already received the return value, so no need to check for an error here.
-	response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
+	response, _ := sendInvite(req.Context(), device, roomID, body.UserID, body.Reason, cfg, rsAPI, evTime)
 	return response
 }
 
 // sendInvite sends an invitation to a user. Returns a JSONResponse and an error
 func sendInvite(
 	ctx context.Context,
-	profileAPI userapi.ClientUserAPI,
 	device *userapi.Device,
 	roomID, userID, reason string,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
+	evTime time.Time,
 ) (util.JSONResponse, error) {
 	validRoomID, err := spec.NewRoomID(roomID)
 	if err != nil {
@@ -359,13 +358,7 @@ func sendInvite(
 			JSON: spec.InvalidParam("UserID is invalid"),
 		}, err
 	}
-	profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}, err
-	}
+
 	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
 	if err != nil {
 		return util.JSONResponse{
@@ -375,16 +368,14 @@ func sendInvite(
 	}
 	err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
 		InviteInput: roomserverAPI.InviteInput{
-			RoomID:      *validRoomID,
-			Inviter:     *inviter,
-			Invitee:     *invitee,
-			DisplayName: profile.DisplayName,
-			AvatarURL:   profile.AvatarURL,
-			Reason:      reason,
-			IsDirect:    false,
-			KeyID:       identity.KeyID,
-			PrivateKey:  identity.PrivateKey,
-			EventTime:   evTime,
+			RoomID:     *validRoomID,
+			Inviter:    *inviter,
+			Invitee:    *invitee,
+			Reason:     reason,
+			IsDirect:   false,
+			KeyID:      identity.KeyID,
+			PrivateKey: identity.PrivateKey,
+			EventTime:  evTime,
 		},
 		InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
 		SendAsServer:    string(device.UserDomain()),
diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go
index 5deb559df..d4644b3e5 100644
--- a/clientapi/routing/server_notices.go
+++ b/clientapi/routing/server_notices.go
@@ -215,7 +215,7 @@ func SendServerNotice(
 		}
 		if !membershipRes.IsInRoom {
 			// re-invite the user
-			res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
+			res, err := sendInvite(ctx, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, time.Now())
 			if err != nil {
 				return res
 			}
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 9e00da2c0..d6caec08c 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -50,16 +50,14 @@ type PerformLeaveResponse struct {
 }
 
 type InviteInput struct {
-	RoomID      spec.RoomID
-	Inviter     spec.UserID
-	Invitee     spec.UserID
-	DisplayName string
-	AvatarURL   string
-	Reason      string
-	IsDirect    bool
-	KeyID       gomatrixserverlib.KeyID
-	PrivateKey  ed25519.PrivateKey
-	EventTime   time.Time
+	RoomID     spec.RoomID
+	Inviter    spec.UserID
+	Invitee    spec.UserID
+	Reason     string
+	IsDirect   bool
+	KeyID      gomatrixserverlib.KeyID
+	PrivateKey ed25519.PrivateKey
+	EventTime  time.Time
 }
 
 type PerformInviteRequest struct {
diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go
index eb8de7811..093082f90 100644
--- a/roomserver/internal/perform/perform_create_room.go
+++ b/roomserver/internal/perform/perform_create_room.go
@@ -503,16 +503,14 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo
 
 			err = c.RSAPI.PerformInvite(ctx, &api.PerformInviteRequest{
 				InviteInput: api.InviteInput{
-					RoomID:      roomID,
-					Inviter:     userID,
-					Invitee:     *inviteeUserID,
-					DisplayName: createRequest.UserDisplayName,
-					AvatarURL:   createRequest.UserAvatarURL,
-					Reason:      "",
-					IsDirect:    createRequest.IsDirect,
-					KeyID:       createRequest.KeyID,
-					PrivateKey:  createRequest.PrivateKey,
-					EventTime:   createRequest.EventTime,
+					RoomID:     roomID,
+					Inviter:    userID,
+					Invitee:    *inviteeUserID,
+					Reason:     "",
+					IsDirect:   createRequest.IsDirect,
+					KeyID:      createRequest.KeyID,
+					PrivateKey: createRequest.PrivateKey,
+					EventTime:  createRequest.EventTime,
 				},
 				InviteRoomState: globalStrippedState,
 				SendAsServer:    string(userID.Domain()),
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index 3abb69cb9..86563e8c3 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -144,11 +144,9 @@ func (r *Inviter) PerformInvite(
 	}
 
 	content := gomatrixserverlib.MemberContent{
-		Membership:  spec.Invite,
-		DisplayName: req.InviteInput.DisplayName,
-		AvatarURL:   req.InviteInput.AvatarURL,
-		Reason:      req.InviteInput.Reason,
-		IsDirect:    req.InviteInput.IsDirect,
+		Membership: spec.Invite,
+		Reason:     req.InviteInput.Reason,
+		IsDirect:   req.InviteInput.IsDirect,
 	}
 
 	if err = proto.SetContent(content); err != nil {

From ecb7b383e99489839b75c0f5ab2cf6e853780bb1 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 19 Feb 2024 20:19:06 +0100
Subject: [PATCH 11/46] Remove unused `token` (#3331)

Part of https://github.com/matrix-org/dendrite/issues/3225
---
 clientapi/routing/keys.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go
index 72785cda8..871b8b08e 100644
--- a/clientapi/routing/keys.go
+++ b/clientapi/routing/keys.go
@@ -93,7 +93,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device)
 
 type queryKeysRequest struct {
 	Timeout    int                 `json:"timeout"`
-	Token      string              `json:"token"`
 	DeviceKeys map[string][]string `json:"device_keys"`
 }
 
@@ -119,7 +118,6 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) u
 		UserID:        device.UserID,
 		UserToDevices: r.DeviceKeys,
 		Timeout:       r.GetTimeout(),
-		// TODO: Token?
 	}, &queryRes)
 	return util.JSONResponse{
 		Code: 200,

From 8f944f64349be6c434d5e47d9453c85db82d0144 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 20 Feb 2024 08:38:51 +0100
Subject: [PATCH 12/46] Limit filter to `limit/2` for before/after events on
 `/context` (#3332)

Part of https://github.com/matrix-org/dendrite/issues/3224
---
 syncapi/routing/context.go | 6 ++++++
 syncapi/syncapi_test.go    | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index b0c91c40b..b136c69a0 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -110,6 +110,7 @@ func Context(
 	}
 
 	stateFilter := synctypes.StateFilter{
+		Limit:                   filter.Limit,
 		NotSenders:              filter.NotSenders,
 		NotTypes:                filter.NotTypes,
 		Senders:                 filter.Senders,
@@ -157,6 +158,11 @@ func Context(
 		}
 	}
 
+	// Limit is split up for before/after events
+	if filter.Limit > 1 {
+		filter.Limit = filter.Limit / 2
+	}
+
 	eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, roomID, filter)
 	if err != nil && err != sql.ErrNoRows {
 		logrus.WithError(err).Error("unable to fetch before events")
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 0a2c38ab7..0392f209a 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -1136,7 +1136,7 @@ func testContext(t *testing.T, dbType test.DBType) {
 		},
 		{
 			name:             "events are not limited",
-			wantBeforeLength: 7,
+			wantBeforeLength: 5,
 		},
 		{
 			name: "all events are limited",

From f4e77453cb082f594a36acdeecf36a738cfa8a40 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 21 Feb 2024 14:10:22 +0100
Subject: [PATCH 13/46] Speed up start up time by batch querying ACL events
 (#3334)

This should significantly speed up start up times on servers with many
rooms.
---
 clientapi/clientapi_test.go               |  9 ++++--
 roomserver/acls/acls.go                   | 35 +++++++++++------------
 roomserver/internal/input/input_events.go |  9 +++++-
 roomserver/producers/roomevent.go         |  9 +++++-
 roomserver/storage/tables/interface.go    |  4 +++
 5 files changed, 43 insertions(+), 23 deletions(-)

diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index 29ff58665..fffe4b6b8 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -958,7 +958,8 @@ func TestCapabilities(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
@@ -1005,7 +1006,8 @@ func TestTurnserver(t *testing.T) {
 	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 	// Needed to create accounts
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
 	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	//rsAPI.SetUserAPI(userAPI)
@@ -1103,7 +1105,8 @@ func Test3PID(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go
index e247c7553..660f4f3bb 100644
--- a/roomserver/acls/acls.go
+++ b/roomserver/acls/acls.go
@@ -23,7 +23,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
@@ -34,10 +34,10 @@ const MRoomServerACL = "m.room.server_acl"
 type ServerACLDatabase interface {
 	// GetKnownRooms returns a list of all rooms we know about.
 	GetKnownRooms(ctx context.Context) ([]string, error)
-	// GetStateEvent returns the state event of a given type for a given room with a given state key
-	// If no event could be found, returns nil
-	// If there was an issue during the retrieval, returns an error
-	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*types.HeaderedEvent, error)
+
+	// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
+	// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
+	GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
 }
 
 type ServerACLs struct {
@@ -58,15 +58,14 @@ func NewServerACLs(db ServerACLDatabase) *ServerACLs {
 	// For each room, let's see if we have a server ACL state event. If we
 	// do then we'll process it into memory so that we have the regexes to
 	// hand.
-	for _, room := range rooms {
-		state, err := db.GetStateEvent(ctx, room, MRoomServerACL, "")
-		if err != nil {
-			logrus.WithError(err).Errorf("Failed to get server ACLs for room %q", room)
-			continue
-		}
-		if state != nil {
-			acls.OnServerACLUpdate(state.PDU)
-		}
+
+	events, err := db.GetBulkStateContent(ctx, rooms, []gomatrixserverlib.StateKeyTuple{{EventType: MRoomServerACL, StateKey: ""}}, false)
+	if err != nil {
+		logrus.WithError(err).Errorf("Failed to get server ACLs for all rooms: %q", err)
+	}
+
+	for _, event := range events {
+		acls.OnServerACLUpdate(event)
 	}
 	return acls
 }
@@ -90,9 +89,9 @@ func compileACLRegex(orig string) (*regexp.Regexp, error) {
 	return regexp.Compile(escaped)
 }
 
-func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
+func (s *ServerACLs) OnServerACLUpdate(strippedEvent tables.StrippedEvent) {
 	acls := &serverACL{}
-	if err := json.Unmarshal(state.Content(), &acls.ServerACL); err != nil {
+	if err := json.Unmarshal([]byte(strippedEvent.ContentValue), &acls.ServerACL); err != nil {
 		logrus.WithError(err).Errorf("Failed to unmarshal state content for server ACLs")
 		return
 	}
@@ -118,10 +117,10 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
 		"allow_ip_literals": acls.AllowIPLiterals,
 		"num_allowed":       len(acls.allowedRegexes),
 		"num_denied":        len(acls.deniedRegexes),
-	}).Debugf("Updating server ACLs for %q", state.RoomID())
+	}).Debugf("Updating server ACLs for %q", strippedEvent.RoomID)
 	s.aclsMutex.Lock()
 	defer s.aclsMutex.Unlock()
-	s.acls[state.RoomID().String()] = acls
+	s.acls[strippedEvent.RoomID] = acls
 }
 
 func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID string) bool {
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 1d9208434..657ca8719 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -509,7 +510,13 @@ func (r *Inputer) processRoomEvent(
 					logrus.WithError(err).Error("failed to get server ACLs")
 				}
 				if aclEvent != nil {
-					r.ACLs.OnServerACLUpdate(aclEvent)
+					strippedEvent := tables.StrippedEvent{
+						RoomID:       aclEvent.RoomID().String(),
+						EventType:    aclEvent.Type(),
+						StateKey:     *aclEvent.StateKey(),
+						ContentValue: string(aclEvent.Content()),
+					}
+					r.ACLs.OnServerACLUpdate(strippedEvent)
 				}
 			}
 		}
diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go
index af7e10580..894e6d81b 100644
--- a/roomserver/producers/roomevent.go
+++ b/roomserver/producers/roomevent.go
@@ -17,6 +17,7 @@ package producers
 import (
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
@@ -75,7 +76,13 @@ func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.Outpu
 
 			if eventType == acls.MRoomServerACL && update.NewRoomEvent.Event.StateKeyEquals("") {
 				ev := update.NewRoomEvent.Event.PDU
-				defer r.ACLs.OnServerACLUpdate(ev)
+				strippedEvent := tables.StrippedEvent{
+					RoomID:       ev.RoomID().String(),
+					EventType:    ev.Type(),
+					StateKey:     *ev.StateKey(),
+					ContentValue: string(ev.Content()),
+				}
+				defer r.ACLs.OnServerACLUpdate(strippedEvent)
 			}
 		}
 		logger.Tracef("Producing to topic '%s'", r.Topic)
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 0ae064e6b..b3cb31880 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -235,6 +235,10 @@ func ExtractContentValue(ev *types.HeaderedEvent) string {
 		key = "topic"
 	case "m.room.guest_access":
 		key = "guest_access"
+	case "m.room.server_acl":
+		// We need the entire content and not only one key, so we can use it
+		// on startup to generate the ACLs. This is merely a workaround.
+		return string(content)
 	}
 	result := gjson.GetBytes(content, key)
 	if !result.Exists() {

From 4ccf6d6f67e7029a0c4f074aca93b0bcf267e5b6 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 28 Feb 2024 20:58:56 +0100
Subject: [PATCH 14/46] Cache ACLs regexes (#3336)

Since #3334 didn't change much on d.m.org, this is another attempt to
speed up startup.

Given moderation bots like Mjolnir/Draupnir are in many rooms with quite
often the same or similar ACLs, caching the compiled regexes _should_
reduce the startup time.

Using a pointer to the `*regexp.Regex` ensures we only store _one_
instance of a regex in memory, instead of potentially storing it hundred
of times. This should reduce memory consumption on servers with many
rooms with ACLs drastically. (5.1MB vs 1.7MB with this change on my
server with 8 ACL'd rooms [3 using the same ACLs])

[skip ci]
---
 roomserver/acls/acls.go      | 47 ++++++++++++++++++++++++------
 roomserver/acls/acls_test.go | 56 +++++++++++++++++++++++++++++++++---
 2 files changed, 91 insertions(+), 12 deletions(-)

diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go
index 660f4f3bb..017682e05 100644
--- a/roomserver/acls/acls.go
+++ b/roomserver/acls/acls.go
@@ -41,15 +41,21 @@ type ServerACLDatabase interface {
 }
 
 type ServerACLs struct {
-	acls      map[string]*serverACL // room ID -> ACL
-	aclsMutex sync.RWMutex          // protects the above
+	acls               map[string]*serverACL      // room ID -> ACL
+	aclsMutex          sync.RWMutex               // protects the above
+	aclRegexCache      map[string]**regexp.Regexp // Cache from "serverName" -> pointer to a regex
+	aclRegexCacheMutex sync.RWMutex               // protects the above
 }
 
 func NewServerACLs(db ServerACLDatabase) *ServerACLs {
 	ctx := context.TODO()
 	acls := &ServerACLs{
 		acls: make(map[string]*serverACL),
+		// Be generous when creating the cache, as in reality
+		// there are hundreds of servers in an ACL.
+		aclRegexCache: make(map[string]**regexp.Regexp, 100),
 	}
+
 	// Look up all of the rooms that the current state server knows about.
 	rooms, err := db.GetKnownRooms(ctx)
 	if err != nil {
@@ -67,6 +73,7 @@ func NewServerACLs(db ServerACLDatabase) *ServerACLs {
 	for _, event := range events {
 		acls.OnServerACLUpdate(event)
 	}
+
 	return acls
 }
 
@@ -78,8 +85,8 @@ type ServerACL struct {
 
 type serverACL struct {
 	ServerACL
-	allowedRegexes []*regexp.Regexp
-	deniedRegexes  []*regexp.Regexp
+	allowedRegexes []**regexp.Regexp
+	deniedRegexes  []**regexp.Regexp
 }
 
 func compileACLRegex(orig string) (*regexp.Regexp, error) {
@@ -89,6 +96,25 @@ func compileACLRegex(orig string) (*regexp.Regexp, error) {
 	return regexp.Compile(escaped)
 }
 
+// cachedCompileACLRegex is a wrapper around compileACLRegex with added caching
+func (s *ServerACLs) cachedCompileACLRegex(orig string) (**regexp.Regexp, error) {
+	s.aclRegexCacheMutex.RLock()
+	re, ok := s.aclRegexCache[orig]
+	if ok {
+		s.aclRegexCacheMutex.RUnlock()
+		return re, nil
+	}
+	s.aclRegexCacheMutex.RUnlock()
+	compiled, err := compileACLRegex(orig)
+	if err != nil {
+		return nil, err
+	}
+	s.aclRegexCacheMutex.Lock()
+	defer s.aclRegexCacheMutex.Unlock()
+	s.aclRegexCache[orig] = &compiled
+	return &compiled, nil
+}
+
 func (s *ServerACLs) OnServerACLUpdate(strippedEvent tables.StrippedEvent) {
 	acls := &serverACL{}
 	if err := json.Unmarshal([]byte(strippedEvent.ContentValue), &acls.ServerACL); err != nil {
@@ -100,14 +126,14 @@ func (s *ServerACLs) OnServerACLUpdate(strippedEvent tables.StrippedEvent) {
 	// special characters and then replace * and ? with their regex counterparts.
 	// https://matrix.org/docs/spec/client_server/r0.6.1#m-room-server-acl
 	for _, orig := range acls.Allowed {
-		if expr, err := compileACLRegex(orig); err != nil {
+		if expr, err := s.cachedCompileACLRegex(orig); err != nil {
 			logrus.WithError(err).Errorf("Failed to compile allowed regex")
 		} else {
 			acls.allowedRegexes = append(acls.allowedRegexes, expr)
 		}
 	}
 	for _, orig := range acls.Denied {
-		if expr, err := compileACLRegex(orig); err != nil {
+		if expr, err := s.cachedCompileACLRegex(orig); err != nil {
 			logrus.WithError(err).Errorf("Failed to compile denied regex")
 		} else {
 			acls.deniedRegexes = append(acls.deniedRegexes, expr)
@@ -118,6 +144,11 @@ func (s *ServerACLs) OnServerACLUpdate(strippedEvent tables.StrippedEvent) {
 		"num_allowed":       len(acls.allowedRegexes),
 		"num_denied":        len(acls.deniedRegexes),
 	}).Debugf("Updating server ACLs for %q", strippedEvent.RoomID)
+
+	// Clear out Denied and Allowed, now that we have the compiled regexes.
+	// They are not needed anymore from this point on.
+	acls.Denied = nil
+	acls.Allowed = nil
 	s.aclsMutex.Lock()
 	defer s.aclsMutex.Unlock()
 	s.acls[strippedEvent.RoomID] = acls
@@ -150,14 +181,14 @@ func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID s
 	// Check if the hostname matches one of the denied regexes. If it does then
 	// the server is banned from the room.
 	for _, expr := range acls.deniedRegexes {
-		if expr.MatchString(string(serverName)) {
+		if (*expr).MatchString(string(serverName)) {
 			return true
 		}
 	}
 	// Check if the hostname matches one of the allowed regexes. If it does then
 	// the server is NOT banned from the room.
 	for _, expr := range acls.allowedRegexes {
-		if expr.MatchString(string(serverName)) {
+		if (*expr).MatchString(string(serverName)) {
 			return false
 		}
 	}
diff --git a/roomserver/acls/acls_test.go b/roomserver/acls/acls_test.go
index 9fb6a5581..efe1d2093 100644
--- a/roomserver/acls/acls_test.go
+++ b/roomserver/acls/acls_test.go
@@ -15,8 +15,14 @@
 package acls
 
 import (
+	"context"
 	"regexp"
 	"testing"
+
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestOpenACLsWithBlacklist(t *testing.T) {
@@ -38,8 +44,8 @@ func TestOpenACLsWithBlacklist(t *testing.T) {
 		ServerACL: ServerACL{
 			AllowIPLiterals: true,
 		},
-		allowedRegexes: []*regexp.Regexp{allowRegex},
-		deniedRegexes:  []*regexp.Regexp{denyRegex},
+		allowedRegexes: []**regexp.Regexp{&allowRegex},
+		deniedRegexes:  []**regexp.Regexp{&denyRegex},
 	}
 
 	if acls.IsServerBannedFromRoom("1.2.3.4", roomID) {
@@ -77,8 +83,8 @@ func TestDefaultACLsWithWhitelist(t *testing.T) {
 		ServerACL: ServerACL{
 			AllowIPLiterals: false,
 		},
-		allowedRegexes: []*regexp.Regexp{allowRegex},
-		deniedRegexes:  []*regexp.Regexp{},
+		allowedRegexes: []**regexp.Regexp{&allowRegex},
+		deniedRegexes:  []**regexp.Regexp{},
 	}
 
 	if !acls.IsServerBannedFromRoom("1.2.3.4", roomID) {
@@ -103,3 +109,45 @@ func TestDefaultACLsWithWhitelist(t *testing.T) {
 		t.Fatal("Expected qux.com:4567 to be allowed but wasn't")
 	}
 }
+
+var (
+	content1 = `{"allow":["*"],"allow_ip_literals":false,"deny":["hello.world", "*.hello.world"]}`
+)
+
+type dummyACLDB struct{}
+
+func (d dummyACLDB) GetKnownRooms(ctx context.Context) ([]string, error) {
+	return []string{"1", "2"}, nil
+}
+
+func (d dummyACLDB) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) {
+	return []tables.StrippedEvent{
+		{
+			RoomID:       "1",
+			ContentValue: content1,
+		},
+		{
+			RoomID:       "2",
+			ContentValue: content1,
+		},
+	}, nil
+}
+
+func TestCachedRegex(t *testing.T) {
+	db := dummyACLDB{}
+	wantBannedServer := spec.ServerName("hello.world")
+
+	acls := NewServerACLs(db)
+
+	// Check that hello.world is banned in room 1
+	banned := acls.IsServerBannedFromRoom(wantBannedServer, "1")
+	assert.True(t, banned)
+
+	// Check that hello.world is banned in room 2
+	banned = acls.IsServerBannedFromRoom(wantBannedServer, "2")
+	assert.True(t, banned)
+
+	// Check that matrix.hello.world is banned in room 2
+	banned = acls.IsServerBannedFromRoom("matrix."+wantBannedServer, "2")
+	assert.True(t, banned)
+}

From 865fff5f03d06f649736c7fcca134086dfccf7ab Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 28 Feb 2024 20:59:34 +0100
Subject: [PATCH 15/46] Make usage of relays optional, avoid DB roundtrips
 (#3337)

This should avoid 2 additional DB roundtrips if we don't want to use
relays.

So instead of possibly doing roughly 20k trips to the DB, we are now
"only" doing ~6600.

---------

Co-authored-by: devonh <devon.dmytro@gmail.com>
---
 federationapi/federationapi.go                  |  5 +----
 federationapi/internal/federationclient_test.go | 10 +++++-----
 federationapi/internal/perform_test.go          | 10 +++++-----
 federationapi/queue/queue_test.go               |  2 +-
 federationapi/statistics/statistics.go          | 11 +++++++++++
 federationapi/statistics/statistics_test.go     |  4 ++--
 setup/config/config_federationapi.go            |  7 +++++++
 7 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index efbfa3315..d3730c7ca 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -113,10 +113,7 @@ func NewInternalAPI(
 		_ = federationDB.RemoveAllServersFromBlacklist()
 	}
 
-	stats := statistics.NewStatistics(
-		federationDB,
-		cfg.FederationMaxRetries+1,
-		cfg.P2PFederationRetriesUntilAssumedOffline+1)
+	stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1, cfg.P2PFederationRetriesUntilAssumedOffline+1, cfg.EnableRelays)
 
 	js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)
 
diff --git a/federationapi/internal/federationclient_test.go b/federationapi/internal/federationclient_test.go
index fe8d84ffb..47efb11da 100644
--- a/federationapi/internal/federationclient_test.go
+++ b/federationapi/internal/federationclient_test.go
@@ -61,7 +61,7 @@ func TestFederationClientQueryKeys(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -92,7 +92,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -122,7 +122,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{shouldFail: true}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -152,7 +152,7 @@ func TestFederationClientClaimKeys(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -183,7 +183,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
diff --git a/federationapi/internal/perform_test.go b/federationapi/internal/perform_test.go
index 2795a018a..82f9b9db1 100644
--- a/federationapi/internal/perform_test.go
+++ b/federationapi/internal/perform_test.go
@@ -66,7 +66,7 @@ func TestPerformWakeupServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -112,7 +112,7 @@ func TestQueryRelayServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -153,7 +153,7 @@ func TestRemoveRelayServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -193,7 +193,7 @@ func TestPerformDirectoryLookup(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
@@ -232,7 +232,7 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go
index 73d3b0598..6da863427 100644
--- a/federationapi/queue/queue_test.go
+++ b/federationapi/queue/queue_test.go
@@ -117,7 +117,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 		txRelayCount:         *atomic.NewUint32(0),
 	}
 
-	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline, false)
 	signingInfo := []*fclient.SigningIdentity{
 		{
 			KeyID:      "ed21019:auto",
diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go
index e5fc4b940..e133fc9c9 100644
--- a/federationapi/statistics/statistics.go
+++ b/federationapi/statistics/statistics.go
@@ -34,12 +34,15 @@ type Statistics struct {
 	// mark the destination as offline. At this point we should attempt
 	// to send messages to the user's async relay servers if we know them.
 	FailuresUntilAssumedOffline uint32
+
+	enableRelays bool
 }
 
 func NewStatistics(
 	db storage.Database,
 	failuresUntilBlacklist uint32,
 	failuresUntilAssumedOffline uint32,
+	enableRelays bool,
 ) Statistics {
 	return Statistics{
 		DB:                          db,
@@ -47,6 +50,7 @@ func NewStatistics(
 		FailuresUntilAssumedOffline: failuresUntilAssumedOffline,
 		backoffTimers:               make(map[spec.ServerName]*time.Timer),
 		servers:                     make(map[spec.ServerName]*ServerStatistics),
+		enableRelays:                enableRelays,
 	}
 }
 
@@ -73,6 +77,13 @@ func (s *Statistics) ForServer(serverName spec.ServerName) *ServerStatistics {
 		} else {
 			server.blacklisted.Store(blacklisted)
 		}
+
+		// Don't bother hitting the database 2 additional times
+		// if we don't want to use relays.
+		if !s.enableRelays {
+			return server
+		}
+
 		assumedOffline, err := s.DB.IsServerAssumedOffline(context.Background(), serverName)
 		if err != nil {
 			logrus.WithError(err).Errorf("Failed to get assumed offline entry %q", serverName)
diff --git a/federationapi/statistics/statistics_test.go b/federationapi/statistics/statistics_test.go
index a930bc3b0..4376a9050 100644
--- a/federationapi/statistics/statistics_test.go
+++ b/federationapi/statistics/statistics_test.go
@@ -16,7 +16,7 @@ const (
 )
 
 func TestBackoff(t *testing.T) {
-	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	server := ServerStatistics{
 		statistics: &stats,
 		serverName: "test.com",
@@ -106,7 +106,7 @@ func TestBackoff(t *testing.T) {
 }
 
 func TestRelayServersListing(t *testing.T) {
-	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	server := ServerStatistics{statistics: &stats}
 	server.AddRelayServers([]spec.ServerName{"relay1", "relay1", "relay2"})
 	relayServers := server.KnownRelayServers()
diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go
index a72eee369..073c46e03 100644
--- a/setup/config/config_federationapi.go
+++ b/setup/config/config_federationapi.go
@@ -18,6 +18,13 @@ type FederationAPI struct {
 	// The default value is 16 if not specified, which is circa 18 hours.
 	FederationMaxRetries uint32 `yaml:"send_max_retries"`
 
+	// P2P Feature: Whether relaying to specific nodes should be enabled.
+	// Defaults to false.
+	// Note: Enabling relays introduces a huge startup delay, if you are not using
+	// relays and have many servers to re-hydrate on start. Only enable this
+	// if you are using relays!
+	EnableRelays bool `yaml:"enable_relays"`
+
 	// P2P Feature: How many consecutive failures that we should tolerate when
 	// sending federation requests to a specific server until we should assume they
 	// are offline. If we assume they are offline then we will attempt to send

From e4a579f10f708953143375f6df52346aae694e07 Mon Sep 17 00:00:00 2001
From: Anton Molyboha <anton.molyboha@gmail.com>
Date: Wed, 28 Feb 2024 21:02:25 +0100
Subject: [PATCH 16/46] FAQ.md: jetstream needs to be backed up too. (#3327)

In the section "What data needs to be kept if transferring/backing up
Dendrite?" of the FAQ, add jetstream directory to the list.

It seems to be a common mistake when moving dendrite to a different
computer, that the jetstream directory is not copied.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [ ] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `Anton Molyboha <anton.molyboha@gmail.com>`
---
 docs/FAQ.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index 82b1581ea..2ef9e6c2b 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -117,6 +117,7 @@ The list of files that need to be stored is:
 - matrix-key.pem
 - dendrite.yaml
 - the postgres or sqlite DB
+- the jetstream directory
 - the media store
 - the search index (although this can be regenerated)
 

From 58bc289a3731d9403e5660cefa7a222edd1a8c57 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 28 Feb 2024 21:03:35 +0100
Subject: [PATCH 17/46] Bump nokogiri from 1.14.3 to 1.16.2 in /docs (#3319)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.14.3
to 1.16.2.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/sparklemotion/nokogiri/releases">nokogiri's
releases</a>.</em></p>
<blockquote>
<h2>v1.16.2 / 2024-02-04</h2>
<h3>Security</h3>
<ul>
<li>[CRuby] Vendored libxml2 is updated to address CVE-2024-25062. See
<a
href="https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-xc9x-jj77-9p9j">GHSA-xc9x-jj77-9p9j</a>
for more information.</li>
</ul>
<h3>Dependencies</h3>
<ul>
<li>[CRuby] Vendored libxml2 is updated to <a
href="https://gitlab.gnome.org/GNOME/libxml2/-/releases/v2.12.5">v2.12.5</a>
from v2.12.4. (<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
</ul>
<hr />
<p>sha256 checksums:</p>

<pre><code>69ba15d2a2498324489ed63850997f0b8f684260114ea81116d3082f16551d2d
nokogiri-1.16.2-aarch64-linux.gem
6a05ce42e3587a40cf8936ece0beaa5d32922254215d2e8cf9ad40588bb42e57
nokogiri-1.16.2-arm-linux.gem
c957226c8e36b31be6a3afb8602e2128282bf8b40ea51016c4cd21aa2608d3f8
nokogiri-1.16.2-arm64-darwin.gem
122652bfc338cd8a54a692ac035e245e41fd3b8283299202ca26e7a7d50db310
nokogiri-1.16.2-java.gem
7344b5072ca69fc5bedb61cb01a3b765b93a27aae5a2a845c2ba7200e4345074
nokogiri-1.16.2-x64-mingw-ucrt.gem
a2a5e184a424111a0d5b77947986484920ad708009c667f061e8d02035c562dd
nokogiri-1.16.2-x64-mingw32.gem
833efddeb51a6c2c9f6356295623c2b2e0d50050d468695c59bd929162953323
nokogiri-1.16.2-x86-linux.gem
e67fc0418dffaff9dc8b1dc65f0605282c3fee9488832d0223b620b4319e0b53
nokogiri-1.16.2-x86-mingw32.gem
5def799e5f139f21a79d7cf71172313a7b6fb0e4b2a31ab9bd5d4ad305994539
nokogiri-1.16.2-x86_64-darwin.gem
5b146240ac6ec6c40fd4367623e74442bca45a542bd3282b1d4d18b07b8e5dfe
nokogiri-1.16.2-x86_64-linux.gem
68922ee5cde27497d995c46f2821957bae961947644eed2822d173daf7567f9c
nokogiri-1.16.2.gem
</code></pre>
<h2>v1.16.1 / 2024-02-03</h2>
<h3>Dependencies</h3>
<ul>
<li>[CRuby] Vendored libxml2 is updated to <a
href="https://gitlab.gnome.org/GNOME/libxml2/-/releases/v2.12.4">v2.12.4</a>
from v2.12.3. (<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
</ul>
<h3>Fixed</h3>
<ul>
<li>[CRuby] <code>XML::Reader</code> defaults the encoding to UTF-8 if
it's not specified in either the document or as a method parameter.
Previously non-ASCII characters were serialized as NCRs in this case. <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/2891">#2891</a>
(<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
<li>[CRuby] Restored support for compilation by GCC versions earlier
than 4.6, which was broken in v1.15.0 (540e9aee). <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3090">#3090</a>
(<a
href="https://github.com/adfoster-r7"><code>@​adfoster-r7</code></a>)</li>
<li>[CRuby] Patched upstream libxml2 to allow parsing HTML5 in the
context of a namespaced node (e.g., foreign content like MathML).
[#3112, <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3116">#3116</a>]
(<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
<li>[CRuby] Fixed a small memory leak in libgumbo (HTML5 parser) when
the maximum tree depth limit is hit. [#3098, <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3100">#3100</a>]
(<a
href="https://github.com/stevecheckoway"><code>@​stevecheckoway</code></a>)</li>
</ul>
<hr />
<p>sha256 checksums:</p>

<pre><code>a541f35e5b9798a0c97300f9ee18f4217da2a2945a6d5499e4123b9018f9cafc
nokogiri-1.16.1-aarch64-linux.gem
6b82affd195000ab2f9c36cc08744ec2d2fcf6d8da88d59a2db67e83211f7c69
nokogiri-1.16.1-arm-linux.gem
&lt;/tr&gt;&lt;/table&gt;
</code></pre>
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md">nokogiri's
changelog</a>.</em></p>
<blockquote>
<h2>v1.16.2 / 2024-02-04</h2>
<h3>Security</h3>
<ul>
<li>[CRuby] Vendored libxml2 is updated to address CVE-2024-25062. See
<a
href="https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-xc9x-jj77-9p9j">GHSA-xc9x-jj77-9p9j</a>
for more information.</li>
</ul>
<h3>Dependencies</h3>
<ul>
<li>[CRuby] Vendored libxml2 is updated to <a
href="https://gitlab.gnome.org/GNOME/libxml2/-/releases/v2.12.5">v2.12.5</a>
from v2.12.4. (<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
</ul>
<h2>v1.16.1 / 2024-02-03</h2>
<h3>Dependencies</h3>
<ul>
<li>[CRuby] Vendored libxml2 is updated to <a
href="https://gitlab.gnome.org/GNOME/libxml2/-/releases/v2.12.4">v2.12.4</a>
from v2.12.3. (<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
</ul>
<h3>Fixed</h3>
<ul>
<li>[CRuby] <code>XML::Reader</code> defaults the encoding to UTF-8 if
it's not specified in either the document or as a method parameter.
Previously non-ASCII characters were serialized as NCRs in this case. <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/2891">#2891</a>
(<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
<li>[CRuby] Restored support for compilation by GCC versions earlier
than 4.6, which was broken in v1.15.0 (540e9aee). <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3090">#3090</a>
(<a
href="https://github.com/adfoster-r7"><code>@​adfoster-r7</code></a>)</li>
<li>[CRuby] Patched upstream libxml2 to allow parsing HTML5 in the
context of a namespaced node (e.g., foreign content like MathML).
[#3112, <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3116">#3116</a>]
(<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</li>
<li>[CRuby] Fixed a small memory leak in libgumbo (HTML5 parser) when
the maximum tree depth limit is hit. [#3098, <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3100">#3100</a>]
(<a
href="https://github.com/stevecheckoway"><code>@​stevecheckoway</code></a>)</li>
</ul>
<h2>v1.16.0 / 2023-12-27</h2>
<h3>Notable Changes</h3>
<h4>Ruby</h4>
<p>This release introduces native gem support for Ruby 3.3.</p>
<p>This release ends support for Ruby 2.7, for which <a
href="https://www.ruby-lang.org/en/downloads/branches/">upstream support
ended 2023-03-31</a>.</p>
<h4>Pattern matching</h4>
<p>This version marks <em>official support</em> for the pattern matching
API in <code>XML::Attr</code>, <code>XML::Document</code>,
<code>XML::DocumentFragment</code>, <code>XML::Namespace</code>,
<code>XML::Node</code>, and <code>XML::NodeSet</code> (and their
subclasses), originally introduced as an experimental feature in
v1.14.0. (<a
href="https://github.com/flavorjones"><code>@​flavorjones</code></a>)</p>
<p>Documentation on what can be matched:</p>
<ul>
<li><a
href="https://nokogiri.org/rdoc/Nokogiri/XML/Attr.html?h=deconstruct#method-i-deconstruct_keys"><code>XML::Attr#deconstruct_keys</code></a></li>
<li><a
href="https://nokogiri.org/rdoc/Nokogiri/XML/Document.html?h=deconstruct#method-i-deconstruct_keys"><code>XML::Document#deconstruct_keys</code></a></li>
<li><a
href="https://nokogiri.org/rdoc/Nokogiri/XML/Namespace.html?h=deconstruct+namespace#method-i-deconstruct_keys"><code>XML::Namespace#deconstruct_keys</code></a></li>
<li><a
href="https://nokogiri.org/rdoc/Nokogiri/XML/Node.html?h=deconstruct#method-i-deconstruct_keys"><code>XML::Node#deconstruct_keys</code></a></li>
<li><a
href="https://nokogiri.org/rdoc/Nokogiri/XML/DocumentFragment.html?h=deconstruct#method-i-deconstruct"><code>XML::DocumentFragment#deconstruct</code></a></li>
<li><a
href="https://nokogiri.org/rdoc/Nokogiri/XML/NodeSet.html?h=deconstruct#method-i-deconstruct"><code>XML::NodeSet#deconstruct</code></a></li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/673756fdd69d1036874b7d7250cc38a51fd4d7b8"><code>673756f</code></a>
version bump to v1.16.2</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/74ffd67a8efb9972657e5c4625fd8419bbccbe06"><code>74ffd67</code></a>
dep: update libxml to 2.12.5 (branch v1.16.x) (<a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3122">#3122</a>)</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/0d4018dc7009580659c101fc41efb3babcfec229"><code>0d4018d</code></a>
dep: update libxml2 to v2.12.5</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/f33a25f4378df33912ebc6b4ebc0f9e8e80ddfa8"><code>f33a25f</code></a>
dep: remove patch from <a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3112">#3112</a>
which has been released upstream</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/e99416896a182bc520a7940bbe286ec33597ab2b"><code>e994168</code></a>
version bump to v1.16.1</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/77ea2f228c20e79c848ca2906813ea5b5010281b"><code>77ea2f2</code></a>
dev: add files to manifest ignore list</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/756f27c6b7a23294d84bdcca5e03a639d0dd7421"><code>756f27c</code></a>
build(deps): bump actions/{download,upload}-artifact from 3 to 4</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/464f8d41eb73ca9c6dae0b366afcf5f4e8bff342"><code>464f8d4</code></a>
.gitignore: clangd-related files</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/2beeb960691df28dd5ebf828192c65b60250670f"><code>2beeb96</code></a>
doc: update CHANGELOG</li>
<li><a
href="https://github.com/sparklemotion/nokogiri/commit/a26536d7a41fd40c52940e165bb5a4f6b4c39662"><code>a26536d</code></a>
fix: apply upstream patch for in-context parsing (<a
href="https://redirect.github.com/sparklemotion/nokogiri/issues/3116">#3116</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/sparklemotion/nokogiri/compare/v1.14.3...v1.16.2">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=nokogiri&package-manager=bundler&previous-version=1.14.3&new-version=1.16.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

[skip ci]
---
 docs/Gemfile.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
index 195f60c6f..791d0f857 100644
--- a/docs/Gemfile.lock
+++ b/docs/Gemfile.lock
@@ -231,9 +231,9 @@ GEM
       jekyll-seo-tag (~> 2.1)
     minitest (5.17.0)
     multipart-post (2.1.1)
-    nokogiri (1.14.3-arm64-darwin)
+    nokogiri (1.16.2-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.3-x86_64-linux)
+    nokogiri (1.16.2-x86_64-linux)
       racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
@@ -241,7 +241,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
-    racc (1.6.2)
+    racc (1.7.3)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)

From 4892b08dd53d845bce19938d1b6a93e80c9e6db3 Mon Sep 17 00:00:00 2001
From: WrenIX <133280015+wrenix@users.noreply.github.com>
Date: Wed, 28 Feb 2024 21:20:41 +0100
Subject: [PATCH 18/46] fix(helm): change strategy to Recreate (#3325)

Current dendrite needs an PVC and replica of 1 is forced, so best way of
update and change of configuration is to stop and start (instatt of
start multiple dendrite pod with deadlock of binding pvc)

see: #3258

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

 Signed-off-by: `Your Name <your@email.example.org>`
 - [x] version bump of helm Chart

Signed-off-by: WrenIX <dev.github@wrenix.eu>

[skip ci]
---
 helm/dendrite/Chart.yaml  | 2 +-
 helm/dendrite/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index e28261c8e..82425bb28 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.7"
+version: "0.13.8"
 appVersion: "0.13.6"
 description: Dendrite Matrix Homeserver
 type: application
diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml
index afce1d930..eaf65f810 100644
--- a/helm/dendrite/values.yaml
+++ b/helm/dendrite/values.yaml
@@ -92,7 +92,7 @@ extraVolumeMounts: []
 strategy:
   # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
   # If you are using ReadWriteOnce volumes, you should probably use Recreate
-  type: RollingUpdate
+  type: Recreate
   rollingUpdate:
     # -- Maximum number of pods that can be unavailable during the update process
     maxUnavailable: 25%

From 4452833099e9084af26c141d8b9dfcfd9c1fac62 Mon Sep 17 00:00:00 2001
From: WrenIX <133280015+wrenix@users.noreply.github.com>
Date: Thu, 29 Feb 2024 08:04:40 +0100
Subject: [PATCH 19/46] chore(helm): use empty/nil storageClass for helm-docs
 (#3245)

i believe that `nil` would be false in the if :
```yaml
storageClass:
```
is still handled correct.

---
In past ( #3191 ), will have the problem with an empty string `""`:
```yaml
storageClass: ""
```

---
do you take another look @S7evinK ?

Signed-off-by: WrenIX <dev.github@wrenix.eu>
---
 helm/dendrite/README.md   |  7 ++++--
 helm/dendrite/values.yaml | 53 ++++++++++++++++++---------------------
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index feaa543dd..c830bb283 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.7](https://img.shields.io/badge/Version-0.13.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
+![Version: 0.13.8](https://img.shields.io/badge/Version-0.13.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -48,13 +48,16 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | signing_key.create | bool | `true` | Create a new signing key, if not exists |
 | signing_key.existingSecret | string | `""` | Use an existing secret |
 | resources | object | sets some sane default values | Default resource requests/limits. |
-| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: <storageClass> # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is #   set, choosing the default provisioner.  (gp2 on AWS, standard on #   GKE, AWS & OpenStack) # storageClass: "" |
+| persistence.storageClass | string | `nil` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack)  |
 | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream |
 | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume |
+| persistence.jetstream.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files |
 | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume |
+| persistence.media.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index |
 | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume |
+| persistence.search.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod |
 | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod |
 | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml
index eaf65f810..bd8c50d53 100644
--- a/helm/dendrite/values.yaml
+++ b/helm/dendrite/values.yaml
@@ -26,13 +26,13 @@ persistence:
   # -- The storage class to use for volume claims.
   # Used unless specified at the specific component.
   # Defaults to the cluster default storage class.
-  ## If defined, storageClassName: <storageClass>
-  ## If set to "-", storageClassName: "", which disables dynamic provisioning
-  ## If undefined (the default) or set to null, no storageClassName spec is
-  ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
-  ##   GKE, AWS & OpenStack)
-  ##
-  # storageClass: ""
+  # If defined, storageClassName: <storageClass>
+  # If set to "-", storageClassName: "", which disables dynamic provisioning
+  # If undefined (the default) or set to null, no storageClassName spec is
+  #   set, choosing the default provisioner.  (gp2 on AWS, standard on
+  #   GKE, AWS & OpenStack)
+  #
+  storageClass:
   jetstream:
     # -- Use an existing volume claim for jetstream
     existingClaim: ""
@@ -40,13 +40,12 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    ## If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
-    ##
-    # storageClass: ""
+    # If defined, storageClassName: <storageClass>
+    # If set to "-", storageClassName: "", which disables dynamic provisioning
+    # If undefined (the default) or set to null, no storageClassName spec is
+    #   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    #   GKE, AWS & OpenStack)
+    storageClass:
   media:
     # -- Use an existing volume claim for media files
     existingClaim: ""
@@ -54,13 +53,12 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    ## If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
-    ##
-    # storageClass: ""
+    # If defined, storageClassName: <storageClass>
+    # If set to "-", storageClassName: "", which disables dynamic provisioning
+    # If undefined (the default) or set to null, no storageClassName spec is
+    #   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    #   GKE, AWS & OpenStack)
+    storageClass:
   search:
     # -- Use an existing volume claim for the fulltext search index
     existingClaim: ""
@@ -68,13 +66,12 @@ persistence:
     capacity: "1Gi"
     # -- The storage class to use for volume claims.
     # Defaults to persistence.storageClass
-    ## If defined, storageClassName: <storageClass>
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
-    ## If undefined (the default) or set to null, no storageClassName spec is
-    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
-    ##   GKE, AWS & OpenStack)
-    ##
-    # storageClass: ""
+    # If defined, storageClassName: <storageClass>
+    # If set to "-", storageClassName: "", which disables dynamic provisioning
+    # If undefined (the default) or set to null, no storageClassName spec is
+    #   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    #   GKE, AWS & OpenStack)
+    storageClass:
 
 # -- Add additional volumes to the Dendrite Pod
 extraVolumes: []

From 66865597e2785c5b8c8a040b8e64a2ac45df6a67 Mon Sep 17 00:00:00 2001
From: Alexandre Oliveira <1251067+Xinayder@users.noreply.github.com>
Date: Thu, 29 Feb 2024 08:13:59 +0100
Subject: [PATCH 20/46] Use port number instead of name for k8s service port
 (#3256)

I've found an issue when deploying Dendrite's Helm chart on my local
cluster. The template for generating an Ingress resource tries to find
the service port using a name (`http`), but the template that generates
the Service resource, instead, identifies the resource with a port
number.

According to the [Kubernetes
ServiceSpec](https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/),
`ports.targetPort` can be either a number or a string; if it's the
latter, it will be looked up as a named port in the pod's container
ports.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

[skip ci]
---
 helm/dendrite/templates/ingress.yaml | 9 +++++----
 helm/dendrite/templates/service.yaml | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml
index 4bcaee12d..eee762511 100644
--- a/helm/dendrite/templates/ingress.yaml
+++ b/helm/dendrite/templates/ingress.yaml
@@ -4,6 +4,7 @@
 {{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}}
 {{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}}
 {{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}}
+
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
@@ -56,7 +57,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -72,7 +73,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -88,7 +89,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -105,7 +106,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
diff --git a/helm/dendrite/templates/service.yaml b/helm/dendrite/templates/service.yaml
index 3b571df1f..1b709c79c 100644
--- a/helm/dendrite/templates/service.yaml
+++ b/helm/dendrite/templates/service.yaml
@@ -14,4 +14,4 @@ spec:
     - name: http
       protocol: TCP
       port: {{ .Values.service.port }}
-      targetPort: 8008
\ No newline at end of file
+      targetPort: http
\ No newline at end of file

From ad3a3e7bedda6551a952c0ec40511b04ea11e925 Mon Sep 17 00:00:00 2001
From: Varac <varac@users.noreply.github.com>
Date: Thu, 29 Feb 2024 08:46:40 +0100
Subject: [PATCH 21/46] [helm] Update postgresql chart to 14.2.3 (#3292)

This change introduces a major Postgresql upgrade
(from 15.1.0 to 16.1.0).

From

https://artifacthub.io/packages/helm/bitnami/postgresql/13.2.24#to-13-0-0:

This major version changes the default PostgreSQL image from 15.x to
16.x. Follow the [official
instructions](https://www.postgresql.org/docs/15/upgrading.html) to
upgrade to 16.x.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

This PR doesn't need a Go unit tests since it doesn't touch any code,
only the helm chart is affected.

Signed-off-by: Varac Anero <varac@varac.net>

---------

Signed-off-by: Varac Anero <varac@varac.net>
Co-authored-by: Till Faelligen <2353100+S7evinK@users.noreply.github.com>

[skip ci]
---
 .gitignore               |  5 ++++-
 helm/dendrite/Chart.yaml | 11 ++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 043956ee4..ce1c9461d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -77,4 +77,7 @@ media_store/
 build
 
 # golang workspaces
-go.work*
\ No newline at end of file
+go.work*
+
+# helm chart
+helm/dendrite/charts/
diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index 82425bb28..2ede0cfe9 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,9 +1,10 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.8"
+version: "0.14.0"
 appVersion: "0.13.6"
 description: Dendrite Matrix Homeserver
 type: application
+icon: https://avatars.githubusercontent.com/u/8418310?s=48&v=4
 keywords:
   - matrix
   - chat
@@ -13,7 +14,7 @@ home: https://github.com/matrix-org/dendrite
 sources:
   - https://github.com/matrix-org/dendrite
 dependencies:
-- name: postgresql
-  version: 12.1.7
-  repository: https://charts.bitnami.com/bitnami
-  condition: postgresql.enabled
+  - name: postgresql
+    version: 14.2.3
+    repository: https://charts.bitnami.com/bitnami
+    condition: postgresql.enabled

From 09f15a3d3fa15a8d6c48a8051f0f2a524ea73344 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 29 Feb 2024 08:58:25 +0100
Subject: [PATCH 22/46] [Helm] Update Postgres image to 16.2.0, update readme

---
 helm/dendrite/README.md   | 10 ++++++----
 helm/dendrite/values.yaml |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index c830bb283..091f575f0 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.8](https://img.shields.io/badge/Version-0.13.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
+![Version: 0.14.0](https://img.shields.io/badge/Version-0.14.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -37,7 +37,7 @@ Create a folder `appservices` and place your configurations in there.  The confi
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 |
+| https://charts.bitnami.com/bitnami | postgresql | 14.2.3 |
 ## Values
 
 | Key | Type | Default | Description |
@@ -60,7 +60,7 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | persistence.search.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod |
 | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod |
-| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
+| strategy.type | string | `"Recreate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
 | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process |
 | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods |
 | dendrite_config.version | int | `2` |  |
@@ -142,7 +142,7 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration |
 | postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. |
 | postgresql.image.repository | string | `"bitnami/postgresql"` |  |
-| postgresql.image.tag | string | `"15.1.0"` |  |
+| postgresql.image.tag | string | `"16.2.0"` |  |
 | postgresql.auth.username | string | `"dendrite"` |  |
 | postgresql.auth.password | string | `"changeme"` |  |
 | postgresql.auth.database | string | `"dendrite"` |  |
@@ -189,3 +189,5 @@ grafana:
 ```
 PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`)
 
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.13.0](https://github.com/norwoodj/helm-docs/releases/v1.13.0)
\ No newline at end of file
diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml
index bd8c50d53..6c5abc90e 100644
--- a/helm/dendrite/values.yaml
+++ b/helm/dendrite/values.yaml
@@ -375,7 +375,7 @@ postgresql:
   enabled: false
   image:
     repository: bitnami/postgresql
-    tag: "15.1.0"
+    tag: "16.2.0"
   auth:
     username: dendrite
     password: changeme

From 928c8c8c4af0e933e52dbf42d9237234d1a3bc9d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 5 Mar 2024 20:41:35 +0100
Subject: [PATCH 23/46] Query rooms with ACLs instead of all rooms (#3338)

This now should actually speed up startup times.
This is because _many_ rooms (like DMs) don't have room ACLs, this means
that we had around 95% pointless DB queries. (as queried on d.m.org)
---
 roomserver/acls/acls.go                       |  6 ++--
 roomserver/acls/acls_test.go                  |  2 +-
 roomserver/api/api.go                         |  3 ++
 roomserver/internal/query/query.go            |  5 +++
 roomserver/roomserver_test.go                 | 35 ++++++++++++++++++
 roomserver/storage/interface.go               |  5 +--
 roomserver/storage/postgres/events_table.go   | 30 ++++++++++++++++
 roomserver/storage/postgres/rooms_table.go    | 22 ------------
 roomserver/storage/shared/storage.go          | 21 +++++++++--
 roomserver/storage/sqlite3/events_table.go    | 34 ++++++++++++++++++
 roomserver/storage/sqlite3/rooms_table.go     | 23 ------------
 .../storage/tables/events_table_test.go       | 36 +++++++++++++++++++
 roomserver/storage/tables/interface.go        |  3 +-
 roomserver/storage/tables/rooms_table_test.go |  7 +---
 14 files changed, 171 insertions(+), 61 deletions(-)

diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go
index 017682e05..4950e6231 100644
--- a/roomserver/acls/acls.go
+++ b/roomserver/acls/acls.go
@@ -32,8 +32,8 @@ import (
 const MRoomServerACL = "m.room.server_acl"
 
 type ServerACLDatabase interface {
-	// GetKnownRooms returns a list of all rooms we know about.
-	GetKnownRooms(ctx context.Context) ([]string, error)
+	// RoomsWithACLs returns all room IDs for rooms with ACLs
+	RoomsWithACLs(ctx context.Context) ([]string, error)
 
 	// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
 	// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
@@ -57,7 +57,7 @@ func NewServerACLs(db ServerACLDatabase) *ServerACLs {
 	}
 
 	// Look up all of the rooms that the current state server knows about.
-	rooms, err := db.GetKnownRooms(ctx)
+	rooms, err := db.RoomsWithACLs(ctx)
 	if err != nil {
 		logrus.WithError(err).Fatalf("Failed to get known rooms")
 	}
diff --git a/roomserver/acls/acls_test.go b/roomserver/acls/acls_test.go
index efe1d2093..09920308c 100644
--- a/roomserver/acls/acls_test.go
+++ b/roomserver/acls/acls_test.go
@@ -116,7 +116,7 @@ var (
 
 type dummyACLDB struct{}
 
-func (d dummyACLDB) GetKnownRooms(ctx context.Context) ([]string, error) {
+func (d dummyACLDB) RoomsWithACLs(ctx context.Context) ([]string, error) {
 	return []string{"1", "2"}, nil
 }
 
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index ef5bc3d17..a4300d0c0 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -86,6 +86,9 @@ type RoomserverInternalAPI interface {
 		req *QueryAuthChainRequest,
 		res *QueryAuthChainResponse,
 	) error
+
+	// RoomsWithACLs returns all room IDs for rooms with ACLs
+	RoomsWithACLs(ctx context.Context) ([]string, error)
 }
 
 type UserRoomPrivateKeyCreator interface {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 74b010281..c7f8d100c 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -1099,3 +1099,8 @@ func (r *Queryer) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID,
 
 	return nil, nil
 }
+
+// RoomsWithACLs returns all room IDs for rooms with ACLs
+func (r *Queryer) RoomsWithACLs(ctx context.Context) ([]string, error) {
+	return r.DB.RoomsWithACLs(ctx)
+}
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
index 88e335711..85312efd9 100644
--- a/roomserver/roomserver_test.go
+++ b/roomserver/roomserver_test.go
@@ -1284,3 +1284,38 @@ func TestRoomConsumerRecreation(t *testing.T) {
 	wantAckWait := input.MaximumMissingProcessingTime + (time.Second * 10)
 	assert.Equal(t, wantAckWait, info.Config.AckWait)
 }
+
+func TestRoomsWithACLs(t *testing.T) {
+	ctx := context.Background()
+	alice := test.NewUser(t)
+	noACLRoom := test.NewRoom(t, alice)
+	aclRoom := test.NewRoom(t, alice)
+
+	aclRoom.CreateAndInsert(t, alice, "m.room.server_acl", map[string]any{
+		"deny":  []string{"evilhost.test"},
+		"allow": []string{"*"},
+	}, test.WithStateKey(""))
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := &jetstream.NATSInstance{}
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		// start JetStream listeners
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		for _, room := range []*test.Room{noACLRoom, aclRoom} {
+			// Create the rooms
+			err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false)
+			assert.NoError(t, err)
+		}
+
+		// Validate that we only have one ACLd room.
+		roomsWithACLs, err := rsAPI.RoomsWithACLs(ctx)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{aclRoom.ID}, roomsWithACLs)
+	})
+}
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 0638252b2..a1a722c5e 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -170,8 +170,6 @@ type Database interface {
 	GetServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName spec.ServerName) (bool, error)
 	// GetKnownUsers searches all users that userID knows about.
 	GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error)
-	// GetKnownRooms returns a list of all rooms we know about.
-	GetKnownRooms(ctx context.Context) ([]string, error)
 	// ForgetRoom sets a flag in the membership table, that the user wishes to forget a specific room
 	ForgetRoom(ctx context.Context, userID, roomID string, forget bool) error
 
@@ -193,6 +191,9 @@ type Database interface {
 	MaybeRedactEvent(
 		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, querier api.QuerySenderIDAPI,
 	) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error)
+
+	// RoomsWithACLs returns all room IDs for rooms with ACLs
+	RoomsWithACLs(ctx context.Context) ([]string, error)
 }
 
 type UserRoomKeys interface {
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index 1c9cd1599..180a03cd6 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -68,6 +68,10 @@ CREATE TABLE IF NOT EXISTS roomserver_events (
 
 -- Create an index which helps in resolving membership events (event_type_nid = 5) - (used for history visibility)
 CREATE INDEX IF NOT EXISTS roomserver_events_memberships_idx ON roomserver_events (room_nid, event_state_key_nid) WHERE (event_type_nid = 5);
+
+-- The following indexes are used by bulkSelectStateEventByNIDSQL 
+CREATE INDEX IF NOT EXISTS roomserver_event_event_type_nid_idx ON roomserver_events (event_type_nid);
+CREATE INDEX IF NOT EXISTS roomserver_event_state_key_nid_idx ON roomserver_events (event_state_key_nid);
 `
 
 const insertEventSQL = "" +
@@ -147,6 +151,8 @@ const selectRoomNIDsForEventNIDsSQL = "" +
 const selectEventRejectedSQL = "" +
 	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
 
+const selectRoomsWithEventTypeNIDSQL = `SELECT DISTINCT room_nid FROM roomserver_events WHERE event_type_nid = $1`
+
 type eventStatements struct {
 	insertEventStmt                               *sql.Stmt
 	selectEventStmt                               *sql.Stmt
@@ -166,6 +172,7 @@ type eventStatements struct {
 	selectMaxEventDepthStmt                       *sql.Stmt
 	selectRoomNIDsForEventNIDsStmt                *sql.Stmt
 	selectEventRejectedStmt                       *sql.Stmt
+	selectRoomsWithEventTypeNIDStmt               *sql.Stmt
 }
 
 func CreateEventsTable(db *sql.DB) error {
@@ -206,6 +213,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		{&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL},
 		{&s.selectRoomNIDsForEventNIDsStmt, selectRoomNIDsForEventNIDsSQL},
 		{&s.selectEventRejectedStmt, selectEventRejectedSQL},
+		{&s.selectRoomsWithEventTypeNIDStmt, selectRoomsWithEventTypeNIDSQL},
 	}.Prepare(db)
 }
 
@@ -582,3 +590,25 @@ func (s *eventStatements) SelectEventRejected(
 	err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected)
 	return
 }
+
+func (s *eventStatements) SelectRoomsWithEventTypeNID(
+	ctx context.Context, txn *sql.Tx, eventTypeNID types.EventTypeNID,
+) ([]types.RoomNID, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectRoomsWithEventTypeNIDStmt)
+	rows, err := stmt.QueryContext(ctx, eventTypeNID)
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithEventTypeNID: rows.close() failed")
+	if err != nil {
+		return nil, err
+	}
+
+	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
+	for rows.Next() {
+		if err := rows.Scan(&roomNID); err != nil {
+			return nil, err
+		}
+		roomNIDs = append(roomNIDs, roomNID)
+	}
+
+	return roomNIDs, rows.Err()
+}
diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go
index bc3820b2c..4de6dee46 100644
--- a/roomserver/storage/postgres/rooms_table.go
+++ b/roomserver/storage/postgres/rooms_table.go
@@ -76,9 +76,6 @@ const selectRoomVersionsForRoomNIDsSQL = "" +
 const selectRoomInfoSQL = "" +
 	"SELECT room_version, room_nid, state_snapshot_nid, latest_event_nids FROM roomserver_rooms WHERE room_id = $1"
 
-const selectRoomIDsSQL = "" +
-	"SELECT room_id FROM roomserver_rooms WHERE array_length(latest_event_nids, 1) > 0"
-
 const bulkSelectRoomIDsSQL = "" +
 	"SELECT room_id FROM roomserver_rooms WHERE room_nid = ANY($1)"
 
@@ -94,7 +91,6 @@ type roomStatements struct {
 	updateLatestEventNIDsStmt          *sql.Stmt
 	selectRoomVersionsForRoomNIDsStmt  *sql.Stmt
 	selectRoomInfoStmt                 *sql.Stmt
-	selectRoomIDsStmt                  *sql.Stmt
 	bulkSelectRoomIDsStmt              *sql.Stmt
 	bulkSelectRoomNIDsStmt             *sql.Stmt
 }
@@ -116,29 +112,11 @@ func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 		{&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL},
 		{&s.selectRoomVersionsForRoomNIDsStmt, selectRoomVersionsForRoomNIDsSQL},
 		{&s.selectRoomInfoStmt, selectRoomInfoSQL},
-		{&s.selectRoomIDsStmt, selectRoomIDsSQL},
 		{&s.bulkSelectRoomIDsStmt, bulkSelectRoomIDsSQL},
 		{&s.bulkSelectRoomNIDsStmt, bulkSelectRoomNIDsSQL},
 	}.Prepare(db)
 }
 
-func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
-	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
-	rows, err := stmt.QueryContext(ctx)
-	if err != nil {
-		return nil, err
-	}
-	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
-	var roomIDs []string
-	var roomID string
-	for rows.Next() {
-		if err = rows.Scan(&roomID); err != nil {
-			return nil, err
-		}
-		roomIDs = append(roomIDs, roomID)
-	}
-	return roomIDs, rows.Err()
-}
 func (s *roomStatements) InsertRoomNID(
 	ctx context.Context, txn *sql.Tx,
 	roomID string, roomVersion gomatrixserverlib.RoomVersion,
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 682cead6c..cde2e6569 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1625,9 +1625,24 @@ func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString strin
 	return d.MembershipTable.SelectKnownUsers(ctx, nil, stateKeyNID, searchString, limit)
 }
 
-// GetKnownRooms returns a list of all rooms we know about.
-func (d *Database) GetKnownRooms(ctx context.Context) ([]string, error) {
-	return d.RoomsTable.SelectRoomIDsWithEvents(ctx, nil)
+func (d *Database) RoomsWithACLs(ctx context.Context) ([]string, error) {
+
+	eventTypeNID, err := d.GetOrCreateEventTypeNID(ctx, "m.room.server_acl")
+	if err != nil {
+		return nil, err
+	}
+
+	roomNIDs, err := d.EventsTable.SelectRoomsWithEventTypeNID(ctx, nil, eventTypeNID)
+	if err != nil {
+		return nil, err
+	}
+
+	roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, roomNIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	return roomIDs, nil
 }
 
 // ForgetRoom sets a users room to forgotten
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index 2c269bced..26401e45d 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -44,6 +44,14 @@ const eventsSchema = `
 	auth_event_nids TEXT NOT NULL DEFAULT '[]',
 	is_rejected BOOLEAN NOT NULL DEFAULT FALSE
   );
+
+-- Create an index which helps in resolving membership events (event_type_nid = 5) - (used for history visibility)
+CREATE INDEX IF NOT EXISTS roomserver_events_memberships_idx ON roomserver_events (room_nid, event_state_key_nid) WHERE (event_type_nid = 5);
+
+-- The following indexes are used by bulkSelectStateEventByNIDSQL 
+CREATE INDEX IF NOT EXISTS roomserver_event_event_type_nid_idx ON roomserver_events (event_type_nid);
+CREATE INDEX IF NOT EXISTS roomserver_event_state_key_nid_idx ON roomserver_events (event_state_key_nid);
+
 `
 
 const insertEventSQL = `
@@ -120,6 +128,8 @@ const selectRoomNIDsForEventNIDsSQL = "" +
 const selectEventRejectedSQL = "" +
 	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
 
+const selectRoomsWithEventTypeNIDSQL = `SELECT DISTINCT room_nid FROM roomserver_events WHERE event_type_nid = $1`
+
 type eventStatements struct {
 	db                                            *sql.DB
 	insertEventStmt                               *sql.Stmt
@@ -135,6 +145,7 @@ type eventStatements struct {
 	bulkSelectStateAtEventAndReferenceStmt        *sql.Stmt
 	bulkSelectEventIDStmt                         *sql.Stmt
 	selectEventRejectedStmt                       *sql.Stmt
+	selectRoomsWithEventTypeNIDStmt               *sql.Stmt
 	//bulkSelectEventNIDStmt               *sql.Stmt
 	//bulkSelectUnsentEventNIDStmt         *sql.Stmt
 	//selectRoomNIDsForEventNIDsStmt       *sql.Stmt
@@ -192,6 +203,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		//{&s.bulkSelectUnsentEventNIDStmt, bulkSelectUnsentEventNIDSQL},
 		//{&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL},
 		{&s.selectEventRejectedStmt, selectEventRejectedSQL},
+		{&s.selectRoomsWithEventTypeNIDStmt, selectRoomsWithEventTypeNIDSQL},
 	}.Prepare(db)
 }
 
@@ -682,3 +694,25 @@ func (s *eventStatements) SelectEventRejected(
 	err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected)
 	return
 }
+
+func (s *eventStatements) SelectRoomsWithEventTypeNID(
+	ctx context.Context, txn *sql.Tx, eventTypeNID types.EventTypeNID,
+) ([]types.RoomNID, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectRoomsWithEventTypeNIDStmt)
+	rows, err := stmt.QueryContext(ctx, eventTypeNID)
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithEventTypeNID: rows.close() failed")
+	if err != nil {
+		return nil, err
+	}
+
+	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
+	for rows.Next() {
+		if err := rows.Scan(&roomNID); err != nil {
+			return nil, err
+		}
+		roomNIDs = append(roomNIDs, roomNID)
+	}
+
+	return roomNIDs, rows.Err()
+}
diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go
index 22700a710..5034b2425 100644
--- a/roomserver/storage/sqlite3/rooms_table.go
+++ b/roomserver/storage/sqlite3/rooms_table.go
@@ -65,9 +65,6 @@ const selectRoomVersionsForRoomNIDsSQL = "" +
 const selectRoomInfoSQL = "" +
 	"SELECT room_version, room_nid, state_snapshot_nid, latest_event_nids FROM roomserver_rooms WHERE room_id = $1"
 
-const selectRoomIDsSQL = "" +
-	"SELECT room_id FROM roomserver_rooms WHERE latest_event_nids != '[]'"
-
 const bulkSelectRoomIDsSQL = "" +
 	"SELECT room_id FROM roomserver_rooms WHERE room_nid IN ($1)"
 
@@ -87,7 +84,6 @@ type roomStatements struct {
 	updateLatestEventNIDsStmt          *sql.Stmt
 	//selectRoomVersionForRoomNIDStmt    *sql.Stmt
 	selectRoomInfoStmt *sql.Stmt
-	selectRoomIDsStmt  *sql.Stmt
 }
 
 func CreateRoomsTable(db *sql.DB) error {
@@ -108,29 +104,10 @@ func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 		{&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL},
 		//{&s.selectRoomVersionForRoomNIDsStmt, selectRoomVersionForRoomNIDsSQL},
 		{&s.selectRoomInfoStmt, selectRoomInfoSQL},
-		{&s.selectRoomIDsStmt, selectRoomIDsSQL},
 		{&s.selectRoomNIDForUpdateStmt, selectRoomNIDForUpdateSQL},
 	}.Prepare(db)
 }
 
-func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
-	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
-	rows, err := stmt.QueryContext(ctx)
-	if err != nil {
-		return nil, err
-	}
-	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
-	var roomIDs []string
-	var roomID string
-	for rows.Next() {
-		if err = rows.Scan(&roomID); err != nil {
-			return nil, err
-		}
-		roomIDs = append(roomIDs, roomID)
-	}
-	return roomIDs, rows.Err()
-}
-
 func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
 	var info types.RoomInfo
 	var latestNIDsJSON string
diff --git a/roomserver/storage/tables/events_table_test.go b/roomserver/storage/tables/events_table_test.go
index 5ed805648..52aeacc2f 100644
--- a/roomserver/storage/tables/events_table_test.go
+++ b/roomserver/storage/tables/events_table_test.go
@@ -2,6 +2,7 @@ package tables_test
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -147,3 +148,38 @@ func Test_EventsTable(t *testing.T) {
 		assert.Equal(t, int64(len(room.Events())+1), maxDepth)
 	})
 }
+
+func TestRoomsWithACL(t *testing.T) {
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		eventStateKeys, closeEventStateKeys := mustCreateEventTypesTable(t, dbType)
+		defer closeEventStateKeys()
+
+		eventsTable, closeEventsTable := mustCreateEventsTable(t, dbType)
+		defer closeEventsTable()
+
+		ctx := context.Background()
+
+		// insert the m.room.server_acl event type
+		eventTypeNID, err := eventStateKeys.InsertEventTypeNID(ctx, nil, "m.room.server_acl")
+		assert.Nil(t, err)
+
+		// Create ACL'd rooms
+		var wantRoomNIDs []types.RoomNID
+		for i := 0; i < 10; i++ {
+			_, _, err = eventsTable.InsertEvent(ctx, nil, types.RoomNID(i), eventTypeNID, types.EmptyStateKeyNID, fmt.Sprintf("$1337+%d", i), nil, 0, false)
+			assert.Nil(t, err)
+			wantRoomNIDs = append(wantRoomNIDs, types.RoomNID(i))
+		}
+
+		// Create non-ACL'd rooms (eventTypeNID+1)
+		for i := 10; i < 20; i++ {
+			_, _, err = eventsTable.InsertEvent(ctx, nil, types.RoomNID(i), eventTypeNID+1, types.EmptyStateKeyNID, fmt.Sprintf("$1337+%d", i), nil, 0, false)
+			assert.Nil(t, err)
+		}
+
+		gotRoomNIDs, err := eventsTable.SelectRoomsWithEventTypeNID(ctx, nil, eventTypeNID)
+		assert.Nil(t, err)
+		assert.Equal(t, wantRoomNIDs, gotRoomNIDs)
+	})
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index b3cb31880..ff810a2bd 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -69,6 +69,8 @@ type Events interface {
 	SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error)
 	SelectRoomNIDsForEventNIDs(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (roomNIDs map[types.EventNID]types.RoomNID, err error)
 	SelectEventRejected(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string) (rejected bool, err error)
+
+	SelectRoomsWithEventTypeNID(ctx context.Context, txn *sql.Tx, eventTypeNID types.EventTypeNID) ([]types.RoomNID, error)
 }
 
 type Rooms interface {
@@ -80,7 +82,6 @@ type Rooms interface {
 	UpdateLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventNIDs []types.EventNID, lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID) error
 	SelectRoomVersionsForRoomNIDs(ctx context.Context, txn *sql.Tx, roomNID []types.RoomNID) (map[types.RoomNID]gomatrixserverlib.RoomVersion, error)
 	SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error)
-	SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error)
 	BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error)
 	BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error)
 }
diff --git a/roomserver/storage/tables/rooms_table_test.go b/roomserver/storage/tables/rooms_table_test.go
index eddd012c8..e97e3e339 100644
--- a/roomserver/storage/tables/rooms_table_test.go
+++ b/roomserver/storage/tables/rooms_table_test.go
@@ -74,11 +74,6 @@ func TestRoomsTable(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Nil(t, roomInfo)
 
-		// There are no rooms with latestEventNIDs yet
-		roomIDs, err := tab.SelectRoomIDsWithEvents(ctx, nil)
-		assert.NoError(t, err)
-		assert.Equal(t, 0, len(roomIDs))
-
 		roomVersions, err := tab.SelectRoomVersionsForRoomNIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
 		assert.NoError(t, err)
 		assert.Equal(t, roomVersions[wantRoomNID], room.Version)
@@ -86,7 +81,7 @@ func TestRoomsTable(t *testing.T) {
 		_, ok := roomVersions[1337]
 		assert.False(t, ok)
 
-		roomIDs, err = tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
+		roomIDs, err := tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
 		assert.NoError(t, err)
 		assert.Equal(t, []string{room.ID}, roomIDs)
 

From de954991787053b97936b8476d9a68fc29c289ae Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 21 Mar 2024 10:24:53 +0100
Subject: [PATCH 24/46] Update golangci config (#3343)

`deadline` is now deprecated in favor of `timeout` and currently breaks
CI.
The other changes remove some warnings produced.
---
 .golangci.yml | 46 +++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index 5bee0a885..6f3fd3627 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -6,7 +6,7 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 30m
+  timeout: 5m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
@@ -18,24 +18,6 @@ run:
   #build-tags:
   #  - mytag
 
-  # which dirs to skip: they won't be analyzed;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but next dirs are always skipped independently
-  # from this option's value:
-  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
-    - bin
-    - docs
-
-  # which files to skip: they will be analyzed, but issues from them
-  # won't be reported. Default value is empty list, but there is
-  # no need to include all autogenerated files, we confidently recognize
-  # autogenerated files. If it's not please let us know.
-  skip-files:
-    - ".*\\.md$"
-    - ".*\\.sh$"
-    - "^cmd/syncserver-integration-tests/testdata.go$"
-
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit
   # automatic updating of go.mod described above. Instead, it fails when any changes
@@ -50,7 +32,8 @@ run:
 # output configuration options
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
-  format: colored-line-number
+  formats:
+    - format: colored-line-number
 
   # print lines of code with issue, default is true
   print-issued-lines: true
@@ -79,9 +62,8 @@ linters-settings:
     # see https://github.com/kisielk/errcheck#excluding-functions for details
     #exclude: /path/to/file.txt
   govet:
-    # report about shadowed variables
-    check-shadowing: true
-
+    enable:
+      - shadow
     # settings per analyzer
     settings:
       printf: # analyzer name, run `go tool vet help` to see all analyzers
@@ -217,6 +199,24 @@ linters:
 
 
 issues:
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  exclude-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  exclude-dirs:
+    - bin
+    - docs
+
   # List of regexps of issue texts to exclude, empty list by default.
   # But independently from this option we use default exclude patterns,
   # it can be disabled by `exclude-use-default: false`. To list all

From b9abbf7b20b4faaffe754c4a1ea4d5f0e7bd72b9 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 21 Mar 2024 19:27:34 +0100
Subject: [PATCH 25/46] Add event reporting (#3340)

Part of #3216 and #3226

There will be a follow up PR which is going to add the same admin
endpoints Synapse has, so existing tools also work for Dendrite.
---
 clientapi/clientapi_test.go                   | 89 ++++++++++++++++++
 clientapi/routing/report_event.go             | 93 +++++++++++++++++++
 clientapi/routing/routing.go                  | 10 ++
 roomserver/api/api.go                         |  7 ++
 roomserver/internal/api.go                    |  8 ++
 roomserver/storage/interface.go               |  9 ++
 .../storage/postgres/reported_events_table.go | 88 ++++++++++++++++++
 roomserver/storage/postgres/storage.go        |  8 ++
 roomserver/storage/shared/storage.go          | 54 +++++++++++
 .../storage/sqlite3/reported_events_table.go  | 87 +++++++++++++++++
 roomserver/storage/sqlite3/storage.go         |  9 +-
 roomserver/storage/tables/interface.go        | 13 +++
 12 files changed, 474 insertions(+), 1 deletion(-)
 create mode 100644 clientapi/routing/report_event.go
 create mode 100644 roomserver/storage/postgres/reported_events_table.go
 create mode 100644 roomserver/storage/sqlite3/reported_events_table.go

diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go
index fffe4b6b8..c550b2083 100644
--- a/clientapi/clientapi_test.go
+++ b/clientapi/clientapi_test.go
@@ -2346,3 +2346,92 @@ func TestCreateRoomInvite(t *testing.T) {
 		}
 	})
 }
+
+func TestReportEvent(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(charlie.ID))
+	eventToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice:   {},
+			bob:     {},
+			charlie: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		t.Run("Bob is not joined and should not be able to report the event", func(t *testing.T) {
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined but the event does not exist", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/$doesNotExist", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined and allowed to report the event", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected report to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+	})
+}
diff --git a/clientapi/routing/report_event.go b/clientapi/routing/report_event.go
new file mode 100644
index 000000000..4dc6498d8
--- /dev/null
+++ b/clientapi/routing/report_event.go
@@ -0,0 +1,93 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+type reportEventRequest struct {
+	Reason string `json:"reason"`
+	Score  int64  `json:"score"`
+}
+
+func ReportEvent(
+	req *http.Request,
+	device *userAPI.Device,
+	roomID, eventID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	defer req.Body.Close() // nolint: errcheck
+
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.NotFound("You don't have permission to report this event, bad userID"),
+		}
+	}
+	// The requesting user must be a member of the room
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
+	if errRes != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound, // Spec demands this...
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	// Parse the request
+	report := reportEventRequest{}
+	if resErr := httputil.UnmarshalJSONRequest(req, &report); resErr != nil {
+		return *resErr
+	}
+
+	queryRes := &api.QueryEventsByIDResponse{}
+	if err = rsAPI.QueryEventsByID(req.Context(), &api.QueryEventsByIDRequest{
+		RoomID:   roomID,
+		EventIDs: []string{eventID},
+	}, queryRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	// No event was found or it was already redacted
+	if len(queryRes.Events) == 0 || queryRes.Events[0].Redacted() {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	_, err = rsAPI.InsertReportedEvent(req.Context(), roomID, eventID, device.UserID, report.Reason, report.Score)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 3e23ab405..40e598229 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -1523,4 +1523,14 @@ func Setup(
 			return GetJoinedMembers(req, device, vars["roomID"], rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/report/{eventID}",
+		httputil.MakeAuthAPI("report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return ReportEvent(req, device, vars["roomID"], vars["eventID"], rsAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
 }
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index a4300d0c0..62aac1441 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -223,6 +223,7 @@ type ClientRoomserverAPI interface {
 	UserRoomPrivateKeyCreator
 	QueryRoomHierarchyAPI
 	DefaultRoomVersionAPI
+
 	QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error
 	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
 	QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error)
@@ -264,6 +265,12 @@ type ClientRoomserverAPI interface {
 	RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error)
 
 	SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error)
+
+	InsertReportedEvent(
+		ctx context.Context,
+		roomID, eventID, reportingUserID, reason string,
+		score int64,
+	) (int64, error)
 }
 
 type UserRoomserverAPI interface {
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index 1e08f6a3a..a71fd2d15 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -340,3 +340,11 @@ func (r *RoomserverInternalAPI) SigningIdentityFor(ctx context.Context, roomID s
 func (r *RoomserverInternalAPI) AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) {
 	return r.DB.AssignRoomNID(ctx, roomID, roomVersion)
 }
+
+func (r *RoomserverInternalAPI) InsertReportedEvent(
+	ctx context.Context,
+	roomID, eventID, reportingUserID, reason string,
+	score int64,
+) (int64, error) {
+	return r.DB.InsertReportedEvent(ctx, roomID, eventID, reportingUserID, reason, score)
+}
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index a1a722c5e..5f9b5b2b4 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -30,6 +30,7 @@ import (
 
 type Database interface {
 	UserRoomKeys
+	ReportedEvents
 	// Do we support processing input events for more than one room at a time?
 	SupportsConcurrentRoomInputs() bool
 	AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error)
@@ -257,3 +258,11 @@ type EventDatabase interface {
 	) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error)
 	StoreEvent(ctx context.Context, event gomatrixserverlib.PDU, roomInfo *types.RoomInfo, eventTypeNID types.EventTypeNID, eventStateKeyNID types.EventStateKeyNID, authEventNIDs []types.EventNID, isRejected bool) (types.EventNID, types.StateAtEvent, error)
 }
+
+type ReportedEvents interface {
+	InsertReportedEvent(
+		ctx context.Context,
+		roomID, eventID, reportingUserID, reason string,
+		score int64,
+	) (int64, error)
+}
diff --git a/roomserver/storage/postgres/reported_events_table.go b/roomserver/storage/postgres/reported_events_table.go
new file mode 100644
index 000000000..01debcf9a
--- /dev/null
+++ b/roomserver/storage/postgres/reported_events_table.go
@@ -0,0 +1,88 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+)
+
+const reportedEventsScheme = `
+CREATE SEQUENCE IF NOT EXISTS roomserver_reported_events_id_seq;
+CREATE TABLE IF NOT EXISTS roomserver_reported_events
+(
+	id 					BIGINT PRIMARY KEY DEFAULT nextval('roomserver_reported_events_id_seq'),
+    room_nid 			BIGINT NOT NULL,
+	event_nid 			BIGINT NOT NULL,
+    reporting_user_nid	INTEGER NOT NULL, -- the user reporting the event
+    event_sender_nid	INTEGER NOT NULL, -- the user who sent the reported event
+    reason      		TEXT,
+    score       		INTEGER,
+    received_ts 		BIGINT NOT NULL
+);`
+
+const insertReportedEventSQL = `
+	INSERT INTO roomserver_reported_events (room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts) 
+	VALUES ($1, $2, $3, $4, $5, $6, $7)
+	RETURNING id
+`
+
+type reportedEventsStatements struct {
+	insertReportedEventsStmt *sql.Stmt
+}
+
+func CreateReportedEventsTable(db *sql.DB) error {
+	_, err := db.Exec(reportedEventsScheme)
+	return err
+}
+
+func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) {
+	s := &reportedEventsStatements{}
+
+	return s, sqlutil.StatementList{
+		{&s.insertReportedEventsStmt, insertReportedEventSQL},
+	}.Prepare(db)
+}
+
+func (r *reportedEventsStatements) InsertReportedEvent(
+	ctx context.Context,
+	txn *sql.Tx,
+	roomNID types.RoomNID,
+	eventNID types.EventNID,
+	reportingUserID types.EventStateKeyNID,
+	eventSenderID types.EventStateKeyNID,
+	reason string,
+	score int64,
+) (int64, error) {
+	stmt := sqlutil.TxStmt(txn, r.insertReportedEventsStmt)
+
+	var reportID int64
+	err := stmt.QueryRowContext(ctx,
+		roomNID,
+		eventNID,
+		reportingUserID,
+		eventSenderID,
+		reason,
+		score,
+		spec.AsTimestamp(time.Now()),
+	).Scan(&reportID)
+	return reportID, err
+}
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index c5c206cfb..1068230f7 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -134,6 +134,9 @@ func (d *Database) create(db *sql.DB) error {
 	if err := CreateUserRoomKeysTable(db); err != nil {
 		return err
 	}
+	if err := CreateReportedEventsTable(db); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -199,6 +202,10 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
+	reportedEvents, err := PrepareReportedEventsTable(db)
+	if err != nil {
+		return err
+	}
 
 	d.Database = shared.Database{
 		DB: db,
@@ -212,6 +219,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 			EventStateKeysTable: eventStateKeys,
 			PrevEventsTable:     prevEvents,
 			RedactionsTable:     redactions,
+			ReportedEventsTable: reportedEvents,
 		},
 		Cache:              cache,
 		Writer:             writer,
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index cde2e6569..f1fb3cf99 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -61,6 +61,7 @@ type EventDatabase struct {
 	EventStateKeysTable tables.EventStateKeys
 	PrevEventsTable     tables.PreviousEvents
 	RedactionsTable     tables.Redactions
+	ReportedEventsTable tables.ReportedEvents
 }
 
 func (d *Database) SupportsConcurrentRoomInputs() bool {
@@ -1882,6 +1883,59 @@ func (d *Database) SelectUserIDsForPublicKeys(ctx context.Context, publicKeys ma
 	return result, err
 }
 
+// InsertReportedEvent stores a reported event.
+func (d *Database) InsertReportedEvent(
+	ctx context.Context,
+	roomID, eventID, reportingUserID, reason string,
+	score int64,
+) (int64, error) {
+	roomInfo, err := d.roomInfo(ctx, nil, roomID)
+	if err != nil {
+		return 0, err
+	}
+	if roomInfo == nil {
+		return 0, fmt.Errorf("room does not exist")
+	}
+
+	events, err := d.eventsFromIDs(ctx, nil, roomInfo, []string{eventID}, NoFilter)
+	if err != nil {
+		return 0, err
+	}
+	if len(events) == 0 {
+		return 0, fmt.Errorf("unable to find requested event")
+	}
+
+	stateKeyNIDs, err := d.EventStateKeyNIDs(ctx, []string{reportingUserID, events[0].SenderID().ToUserID().String()})
+	if err != nil {
+		return 0, fmt.Errorf("failed to query eventStateKeyNIDs: %w", err)
+	}
+
+	// We expect exactly 2 stateKeyNIDs
+	if len(stateKeyNIDs) != 2 {
+		return 0, fmt.Errorf("expected 2 stateKeyNIDs, received %d", len(stateKeyNIDs))
+	}
+
+	var reportID int64
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		reportID, err = d.ReportedEventsTable.InsertReportedEvent(
+			ctx,
+			txn,
+			roomInfo.RoomNID,
+			events[0].EventNID,
+			stateKeyNIDs[reportingUserID],
+			stateKeyNIDs[events[0].SenderID().ToUserID().String()],
+			reason,
+			score,
+		)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	return reportID, err
+}
+
 // FIXME TODO: Remove all this - horrible dupe with roomserver/state. Can't use the original impl because of circular loops
 // it should live in this package!
 
diff --git a/roomserver/storage/sqlite3/reported_events_table.go b/roomserver/storage/sqlite3/reported_events_table.go
new file mode 100644
index 000000000..4a8582fc4
--- /dev/null
+++ b/roomserver/storage/sqlite3/reported_events_table.go
@@ -0,0 +1,87 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+)
+
+const reportedEventsScheme = `
+CREATE TABLE IF NOT EXISTS roomserver_reported_events
+(
+    id					INTEGER PRIMARY KEY AUTOINCREMENT,
+    room_nid 			INTEGER NOT NULL,
+	event_nid 			INTEGER NOT NULL,
+    reporting_user_nid	INTEGER NOT NULL, -- the user reporting the event
+    event_sender_nid	INTEGER NOT NULL, -- the user who sent the reported event
+    reason      		TEXT,
+    score       		INTEGER,
+    received_ts 		INTEGER NOT NULL
+);`
+
+const insertReportedEventSQL = `
+	INSERT INTO roomserver_reported_events (room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts) 
+	VALUES ($1, $2, $3, $4, $5, $6, $7)
+	RETURNING id
+`
+
+type reportedEventsStatements struct {
+	insertReportedEventsStmt *sql.Stmt
+}
+
+func CreateReportedEventsTable(db *sql.DB) error {
+	_, err := db.Exec(reportedEventsScheme)
+	return err
+}
+
+func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) {
+	s := &reportedEventsStatements{}
+
+	return s, sqlutil.StatementList{
+		{&s.insertReportedEventsStmt, insertReportedEventSQL},
+	}.Prepare(db)
+}
+
+func (r *reportedEventsStatements) InsertReportedEvent(
+	ctx context.Context,
+	txn *sql.Tx,
+	roomNID types.RoomNID,
+	eventNID types.EventNID,
+	reportingUserID types.EventStateKeyNID,
+	eventSenderID types.EventStateKeyNID,
+	reason string,
+	score int64,
+) (int64, error) {
+	stmt := sqlutil.TxStmt(txn, r.insertReportedEventsStmt)
+
+	var reportID int64
+	err := stmt.QueryRowContext(ctx,
+		roomNID,
+		eventNID,
+		reportingUserID,
+		eventSenderID,
+		reason,
+		score,
+		spec.AsTimestamp(time.Now()),
+	).Scan(&reportID)
+	return reportID, err
+}
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 98d88f923..191c07223 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -141,7 +141,9 @@ func (d *Database) create(db *sql.DB) error {
 	if err := CreateUserRoomKeysTable(db); err != nil {
 		return err
 	}
-
+	if err := CreateReportedEventsTable(db); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -206,6 +208,10 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
+	reportedEvents, err := PrepareReportedEventsTable(db)
+	if err != nil {
+		return err
+	}
 
 	d.Database = shared.Database{
 		DB: db,
@@ -219,6 +225,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 			EventJSONTable:      eventJSON,
 			PrevEventsTable:     prevEvents,
 			RedactionsTable:     redactions,
+			ReportedEventsTable: reportedEvents,
 		},
 		Cache:              cache,
 		Writer:             writer,
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index ff810a2bd..cc0114372 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -127,6 +127,19 @@ type Invites interface {
 	SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, []byte, error)
 }
 
+type ReportedEvents interface {
+	InsertReportedEvent(
+		ctx context.Context,
+		txn *sql.Tx,
+		roomNID types.RoomNID,
+		eventNID types.EventNID,
+		reportingUserID types.EventStateKeyNID,
+		eventSenderID types.EventStateKeyNID,
+		reason string,
+		score int64,
+	) (int64, error)
+}
+
 type MembershipState int64
 
 const (

From a00b976a00994fc3ff6a862d7365b2c202e84ccf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 22 Mar 2024 22:29:53 +0100
Subject: [PATCH 26/46] Bump google.golang.org/protobuf from 1.30.0 to 1.33.0
 (#3339)

Bumps google.golang.org/protobuf from 1.30.0 to 1.33.0.


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/protobuf&package-manager=go_modules&previous-version=1.30.0&new-version=1.33.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 234381a4f..f3745dbb0 100644
--- a/go.mod
+++ b/go.mod
@@ -128,7 +128,7 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.12.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
diff --git a/go.sum b/go.sum
index 3129f40e7..69f172b1e 100644
--- a/go.sum
+++ b/go.sum
@@ -464,8 +464,8 @@ gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6d
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

From 1bdf0cc5414a47fbdab9ae76fa7d549349d648ec Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 22 Mar 2024 22:30:28 +0100
Subject: [PATCH 27/46] Bump github.com/docker/docker from 24.0.7+incompatible
 to 24.0.9+incompatible (#3341)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [github.com/docker/docker](https://github.com/docker/docker) from
24.0.7+incompatible to 24.0.9+incompatible.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/docker/docker/releases">github.com/docker/docker's
releases</a>.</em></p>
<blockquote>
<h2>v24.0.9</h2>
<h2>24.0.9</h2>
<p>For a full list of pull requests and changes in this release, refer
to the relevant GitHub milestones:</p>
<ul>
<li><a
href="https://github.com/docker/cli/issues?q=is%3Aclosed+milestone%3A24.0.9">docker/cli,
24.0.9 milestone</a></li>
<li><a
href="https://github.com/moby/moby/issues?q=is%3Aclosed+milestone%3A24.0.9">moby/moby,
24.0.9 milestone</a></li>
</ul>
<h2>Security</h2>
<p>This release contains security fixes for the following CVEs affecting
Docker Engine and its components.</p>
<table>
<thead>
<tr>
<th>CVE</th>
<th>Component</th>
<th>Fix version</th>
<th>Severity</th>
</tr>
</thead>
<tbody>
<tr>
<td><a
href="https://scout.docker.com/v/CVE-2024-21626">CVE-2024-21626</a></td>
<td>runc</td>
<td>1.1.12</td>
<td>High, CVSS 8.6</td>
</tr>
<tr>
<td><a
href="https://scout.docker.com/v/CVE-2024-24557">CVE-2024-24557</a></td>
<td>Docker Engine</td>
<td>24.0.9</td>
<td>Medium, CVSS 6.9</td>
</tr>
</tbody>
</table>
<blockquote>
<p><strong>Important</strong> ⚠️</p>
<p>Note that this release of Docker Engine doesn't include fixes for the
following known vulnerabilities in BuildKit:</p>
<ul>
<li><a
href="https://scout.docker.com/v/CVE-2024-23651">CVE-2024-23651</a></li>
<li><a
href="https://scout.docker.com/v/CVE-2024-23652">CVE-2024-23652</a></li>
<li><a
href="https://scout.docker.com/v/CVE-2024-23653">CVE-2024-23653</a></li>
<li><a
href="https://scout.docker.com/v/CVE-2024-23650">CVE-2024-23650</a></li>
</ul>
<p>To address these vulnerabilities, upgrade to <a
href="https://github.com/docker/docker/blob/HEAD/25.0.md#2502">Docker
Engine v25.0.2</a>.</p>
</blockquote>
<p>For more information about the security issues addressed in this
release, and the unaddressed vulnerabilities in BuildKit, refer to the
<a
href="https://www.docker.com/blog/docker-security-advisory-multiple-vulnerabilities-in-runc-buildkit-and-moby/">blog
post</a>. For details about each vulnerability, see the relevant
security advisory:</p>
<ul>
<li><a
href="https://github.com/opencontainers/runc/security/advisories/GHSA-xr7r-f8xq-vfvv">CVE-2024-21626</a></li>
<li><a
href="https://github.com/moby/moby/security/advisories/GHSA-xw73-rw38-6vjc">CVE-2024-24557</a></li>
</ul>
<h3>Packaging updates</h3>
<ul>
<li>Upgrade runc to <a
href="https://github.com/opencontainers/runc/releases/tag/v1.1.12">v1.1.12</a>.
<a
href="https://redirect.github.com/moby/moby/pull/47269">moby/moby#47269</a></li>
<li>Upgrade containerd to <a
href="https://github.com/containerd/containerd/releases/tag/v1.7.13">v1.7.13</a>
(static binaries only). <a
href="https://redirect.github.com/moby/moby/pull/47280">moby/moby#47280</a></li>
</ul>
<h2>v24.0.8</h2>
<h2>24.0.8</h2>
<p>For a full list of pull requests and changes in this release, refer
to the relevant GitHub milestones:</p>
<ul>
<li><a
href="https://github.com/docker/cli/issues?q=is%3Aclosed+milestone%3A24.0.8">docker/cli,
24.0.8 milestone</a></li>
<li><a
href="https://github.com/moby/moby/issues?q=is%3Aclosed+milestone%3A24.0.8">moby/moby,
24.0.8 milestone</a></li>
</ul>
<h3>Bug fixes and enhancements</h3>
<ul>
<li>Live restore: Containers with auto remove (<code>docker run
--rm</code>) are no longer forcibly removed on engine restart. <a
href="https://redirect.github.com/moby/moby/pull/46869">moby/moby#46857</a></li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/moby/moby/commit/fca702de7f71362c8d103073c7e4a1d0a467fadd"><code>fca702d</code></a>
Merge pull request from GHSA-xw73-rw38-6vjc</li>
<li><a
href="https://github.com/moby/moby/commit/f78a7726d747847e443a5a5a4b4ad8ab31d87d78"><code>f78a772</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/47281">#47281</a>
from thaJeztah/24.0_backport_bump_containerd_binary...</li>
<li><a
href="https://github.com/moby/moby/commit/61afffeeb3d4264db7a697ca8bd3d25824bee182"><code>61afffe</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/47270">#47270</a>
from thaJeztah/24.0_backport_bump_runc_binary_1.1.12</li>
<li><a
href="https://github.com/moby/moby/commit/b38e74c4e095d584e21576e9cc43a355446e5b71"><code>b38e74c</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/47276">#47276</a>
from thaJeztah/24.0_backport_bump_runc_1.1.12</li>
<li><a
href="https://github.com/moby/moby/commit/dac56638adccd215bae6cc23146f29e4697e1e98"><code>dac5663</code></a>
update containerd binary to v1.7.13</li>
<li><a
href="https://github.com/moby/moby/commit/20e1af361628a31afd1af58d25cd6ea4e495669f"><code>20e1af3</code></a>
vendor: github.com/opencontainers/runc v1.1.12</li>
<li><a
href="https://github.com/moby/moby/commit/858919d39968c687de3afb0a0a3a212d60ef2a99"><code>858919d</code></a>
update runc binary to v1.1.12</li>
<li><a
href="https://github.com/moby/moby/commit/141ad39e38a9a44b7487933d74815863c2c588e6"><code>141ad39</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/47266">#47266</a>
from vvoland/ci-fix-makeps1-templatefail-24</li>
<li><a
href="https://github.com/moby/moby/commit/db968c672bcd6eeed09a0ad35cac843a5ffe7e48"><code>db968c6</code></a>
hack/make.ps1: Fix go list pattern</li>
<li><a
href="https://github.com/moby/moby/commit/61c51fbb5aeb648eb5f97704b8c75be3ccf1c9a0"><code>61c51fb</code></a>
Merge pull request <a
href="https://redirect.github.com/docker/docker/issues/47221">#47221</a>
from vvoland/pkg-pools-close-noop-24</li>
<li>Additional commits viewable in <a
href="https://github.com/docker/docker/compare/v24.0.7...v24.0.9">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/docker/docker&package-manager=go_modules&previous-version=24.0.7+incompatible&new-version=24.0.9+incompatible)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index f3745dbb0..0dde02252 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/blevesearch/bleve/v2 v2.3.8
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v24.0.9+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
 	github.com/gologme/log v1.3.0
diff --git a/go.sum b/go.sum
index 69f172b1e..f526d6586 100644
--- a/go.sum
+++ b/go.sum
@@ -89,8 +89,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
+github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=

From 79072c3dcdc88b77dd5a49c013a0c62624dd3224 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 22 Mar 2024 22:32:30 +0100
Subject: [PATCH 28/46] Add `/_synapse/admin/v1/event_reports` endpoint (#3342)

Based on #3340

This adds a `/_synapse/admin/v1/event_reports` endpoint, the same
Synapse has. This way existing tools also work with Dendrite.
Given this is already getting huge (even though many test lines),
splitting this into two PRs. (The next adds "getting one report" and
"deleting reports")

[skip ci]
---
 clientapi/admin_test.go                       | 244 ++++++++++++++++++
 clientapi/routing/admin.go                    |  42 +++
 clientapi/routing/routing.go                  |  14 +
 roomserver/api/api.go                         |   1 +
 roomserver/api/query.go                       |  17 ++
 roomserver/internal/query/query.go            |   5 +
 roomserver/storage/interface.go               |   1 +
 .../storage/postgres/reported_events_table.go |  98 ++++++-
 roomserver/storage/shared/storage.go          | 125 +++++++++
 .../storage/sqlite3/reported_events_table.go  |  95 ++++++-
 roomserver/storage/tables/interface.go        |   9 +
 11 files changed, 647 insertions(+), 4 deletions(-)

diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go
index f0e5f004d..2444f0be1 100644
--- a/clientapi/admin_test.go
+++ b/clientapi/admin_test.go
@@ -2,10 +2,12 @@ package clientapi
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -1092,3 +1094,245 @@ func TestAdminMarkAsStale(t *testing.T) {
 		}
 	})
 }
+
+func TestAdminQueryEventReports(t *testing.T) {
+	alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+	room2 := test.NewRoom(t, alice)
+
+	// room2 has a name and canonical alias
+	room2.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": "Testing"}, test.WithStateKey(""))
+	room2.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": "#testing"}, test.WithStateKey(""))
+
+	// Join the rooms with Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+	room2.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	// Create a few events to report
+	eventsToReportPerRoom := make(map[string][]string)
+	for i := 0; i < 10; i++ {
+		ev1 := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+		ev2 := room2.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+		eventsToReportPerRoom[room.ID] = append(eventsToReportPerRoom[room.ID], ev1.EventID())
+		eventsToReportPerRoom[room2.ID] = append(eventsToReportPerRoom[room2.ID], ev2.EventID())
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		/*if dbType == test.DBTypeSQLite {
+			t.Skip()
+		}*/
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room2.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		// Report all events
+		for roomID, eventIDs := range eventsToReportPerRoom {
+			for _, eventID := range eventIDs {
+				req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", roomID, eventID), strings.NewReader(string(body)))
+				req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+				routers.Client.ServeHTTP(w, req)
+
+				if w.Code != http.StatusOK {
+					t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+				}
+			}
+		}
+
+		type response struct {
+			EventReports []api.QueryAdminEventReportsResponse `json:"event_reports"`
+			Total        int64                                `json:"total"`
+			NextToken    *int64                               `json:"next_token,omitempty"`
+		}
+
+		t.Run("Can query all reports", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 20
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can filter on room", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 10
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can filter on user_id", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?user_id=%s", "@doesnotexist:test"), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+
+			// The user does not exist, so we expect no results
+			wantCount := 0
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can set direction=f", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&dir=f", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 10
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+			// we now should have the first reported event
+			wantEventID := eventsToReportPerRoom[room.ID][0]
+			gotEventID := resp.EventReports[0].EventID
+			if gotEventID != wantEventID {
+				t.Fatalf("expected eventID to be %v, got %v", wantEventID, gotEventID)
+			}
+		})
+
+		t.Run("Can limit and paginate", func(t *testing.T) {
+			var from int64 = 0
+			var limit int64 = 5
+			var wantTotal int64 = 10 // We expect there to be 10 events in total
+			var resp response
+			for from+limit <= wantTotal {
+				resp = response{}
+				t.Logf("Getting reports starting from %d", from)
+				w = httptest.NewRecorder()
+				req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&limit=%d&from=%d", room2.ID, limit, from), strings.NewReader(string(body)))
+				req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+				routers.SynapseAdmin.ServeHTTP(w, req)
+
+				if w.Code != http.StatusOK {
+					t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+				}
+
+				if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+					t.Fatal(err)
+				}
+
+				wantCount := 5 // we are limited to 5
+				if len(resp.EventReports) != wantCount {
+					t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+				}
+				if resp.Total != int64(wantTotal) {
+					t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+				}
+
+				// We've reached the end
+				if (from + int64(len(resp.EventReports))) == wantTotal {
+					return
+				}
+
+				// The next_token should be set
+				if resp.NextToken == nil {
+					t.Fatal("expected nextToken to be set")
+				}
+				from = *resp.NextToken
+			}
+		})
+	})
+}
diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index 519666076..e91e098a4 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -495,3 +495,45 @@ func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverA
 		JSON: struct{}{},
 	}
 }
+
+// GetEventReports returns reported events for a given user/room.
+func GetEventReports(
+	req *http.Request,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+	from, limit uint64,
+	backwards bool,
+	userID, roomID string,
+) util.JSONResponse {
+
+	eventReports, count, err := rsAPI.QueryAdminEventReports(req.Context(), from, limit, backwards, userID, roomID)
+	if err != nil {
+		logrus.WithError(err).Error("failed to query event reports")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	resp := map[string]any{
+		"event_reports": eventReports,
+		"total":         count,
+	}
+
+	// Add a next_token if there are still reports
+	if int64(from+limit) < count {
+		resp["next_token"] = int(from) + len(eventReports)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp,
+	}
+}
+
+func parseUint64OrDefault(input string, defaultValue uint64) uint64 {
+	v, err := strconv.ParseUint(input, 10, 64)
+	if err != nil {
+		return defaultValue
+	}
+	return v
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 40e598229..dc63a2c24 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -1533,4 +1533,18 @@ func Setup(
 			return ReportEvent(req, device, vars["roomID"], vars["eventID"], rsAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports",
+		httputil.MakeAdminAPI("admin_report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			from := parseUint64OrDefault(req.URL.Query().Get("from"), 0)
+			limit := parseUint64OrDefault(req.URL.Query().Get("limit"), 100)
+			dir := req.URL.Query().Get("dir")
+			userID := req.URL.Query().Get("user_id")
+			roomID := req.URL.Query().Get("room_id")
+
+			// Go backwards if direction is empty or "b"
+			backwards := dir == "" || dir == "b"
+			return GetEventReports(req, rsAPI, from, limit, backwards, userID, roomID)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 }
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 62aac1441..ac4ea5bab 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -271,6 +271,7 @@ type ClientRoomserverAPI interface {
 		roomID, eventID, reportingUserID, reason string,
 		score int64,
 	) (int64, error)
+	QueryAdminEventReports(ctx context.Context, from, limit uint64, backwards bool, userID, roomID string) ([]QueryAdminEventReportsResponse, int64, error)
 }
 
 type UserRoomserverAPI interface {
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index 893d5dccf..9a7acab9a 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -346,6 +346,23 @@ type QueryServerBannedFromRoomResponse struct {
 	Banned bool `json:"banned"`
 }
 
+type QueryAdminEventReportsResponse struct {
+	ID               int64                  `json:"id"`
+	Score            int64                  `json:"score"`
+	EventNID         types.EventNID         `json:"-"` // only used to query the state
+	RoomNID          types.RoomNID          `json:"-"` // only used to query the state
+	ReportingUserNID types.EventStateKeyNID `json:"-"` // only used in the DB
+	SenderNID        types.EventStateKeyNID `json:"-"` // only used in the DB
+	RoomID           string                 `json:"room_id"`
+	EventID          string                 `json:"event_id"`
+	UserID           string                 `json:"user_id"` // the user reporting the event
+	Reason           string                 `json:"reason"`
+	Sender           string                 `json:"sender"` // the user sending the reported event
+	CanonicalAlias   string                 `json:"canonical_alias"`
+	RoomName         string                 `json:"name"`
+	ReceivedTS       spec.Timestamp         `json:"received_ts"`
+}
+
 // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) {
 	se := make(map[string]string)
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index c7f8d100c..b1b11df31 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -1104,3 +1104,8 @@ func (r *Queryer) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID,
 func (r *Queryer) RoomsWithACLs(ctx context.Context) ([]string, error) {
 	return r.DB.RoomsWithACLs(ctx)
 }
+
+// QueryAdminEventReports returns event reports given a filter.
+func (r *Queryer) QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error) {
+	return r.DB.QueryAdminEventReports(ctx, from, limit, backwards, userID, roomID)
+}
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 5f9b5b2b4..eb169cbbb 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -195,6 +195,7 @@ type Database interface {
 
 	// RoomsWithACLs returns all room IDs for rooms with ACLs
 	RoomsWithACLs(ctx context.Context) ([]string, error)
+	QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID string, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error)
 }
 
 type UserRoomKeys interface {
diff --git a/roomserver/storage/postgres/reported_events_table.go b/roomserver/storage/postgres/reported_events_table.go
index 01debcf9a..70393833c 100644
--- a/roomserver/storage/postgres/reported_events_table.go
+++ b/roomserver/storage/postgres/reported_events_table.go
@@ -19,7 +19,9 @@ import (
 	"database/sql"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -32,8 +34,8 @@ CREATE TABLE IF NOT EXISTS roomserver_reported_events
 	id 					BIGINT PRIMARY KEY DEFAULT nextval('roomserver_reported_events_id_seq'),
     room_nid 			BIGINT NOT NULL,
 	event_nid 			BIGINT NOT NULL,
-    reporting_user_nid	INTEGER NOT NULL, -- the user reporting the event
-    event_sender_nid	INTEGER NOT NULL, -- the user who sent the reported event
+    reporting_user_nid	BIGINT NOT NULL, -- the user reporting the event
+    event_sender_nid	BIGINT NOT NULL, -- the user who sent the reported event
     reason      		TEXT,
     score       		INTEGER,
     received_ts 		BIGINT NOT NULL
@@ -45,8 +47,38 @@ const insertReportedEventSQL = `
 	RETURNING id
 `
 
+const selectReportedEventsDescSQL = `
+WITH countReports AS (
+    SELECT count(*) as report_count
+    FROM roomserver_reported_events
+    WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT)
+)
+SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts
+FROM roomserver_reported_events, countReports
+WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT)
+ORDER BY received_ts DESC
+OFFSET $3
+LIMIT $4
+`
+
+const selectReportedEventsAscSQL = `
+WITH countReports AS (
+    SELECT count(*) as report_count
+    FROM roomserver_reported_events
+    WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT)
+)
+SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts
+FROM roomserver_reported_events, countReports
+WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT)
+ORDER BY received_ts ASC
+OFFSET $3
+LIMIT $4
+`
+
 type reportedEventsStatements struct {
-	insertReportedEventsStmt *sql.Stmt
+	insertReportedEventsStmt     *sql.Stmt
+	selectReportedEventsDescStmt *sql.Stmt
+	selectReportedEventsAscStmt  *sql.Stmt
 }
 
 func CreateReportedEventsTable(db *sql.DB) error {
@@ -59,6 +91,8 @@ func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) {
 
 	return s, sqlutil.StatementList{
 		{&s.insertReportedEventsStmt, insertReportedEventSQL},
+		{&s.selectReportedEventsDescStmt, selectReportedEventsDescSQL},
+		{&s.selectReportedEventsAscStmt, selectReportedEventsAscSQL},
 	}.Prepare(db)
 }
 
@@ -86,3 +120,61 @@ func (r *reportedEventsStatements) InsertReportedEvent(
 	).Scan(&reportID)
 	return reportID, err
 }
+
+func (r *reportedEventsStatements) SelectReportedEvents(
+	ctx context.Context,
+	txn *sql.Tx,
+	from, limit uint64,
+	backwards bool,
+	reportingUserID types.EventStateKeyNID,
+	roomNID types.RoomNID,
+) ([]api.QueryAdminEventReportsResponse, int64, error) {
+	var stmt *sql.Stmt
+	if backwards {
+		stmt = sqlutil.TxStmt(txn, r.selectReportedEventsDescStmt)
+	} else {
+		stmt = sqlutil.TxStmt(txn, r.selectReportedEventsAscStmt)
+	}
+
+	var qryRoomNID *types.RoomNID
+	if roomNID > 0 {
+		qryRoomNID = &roomNID
+	}
+	var qryReportingUser *types.EventStateKeyNID
+	if reportingUserID > 0 {
+		qryReportingUser = &reportingUserID
+	}
+
+	rows, err := stmt.QueryContext(ctx,
+		qryRoomNID,
+		qryReportingUser,
+		from,
+		limit,
+	)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectReportedEvents: failed to close rows")
+
+	var result []api.QueryAdminEventReportsResponse
+	var row api.QueryAdminEventReportsResponse
+	var count int64
+	for rows.Next() {
+		if err = rows.Scan(
+			&count,
+			&row.ID,
+			&row.RoomNID,
+			&row.EventNID,
+			&row.ReportingUserNID,
+			&row.SenderNID,
+			&row.Reason,
+			&row.Score,
+			&row.ReceivedTS,
+		); err != nil {
+			return nil, 0, err
+		}
+		result = append(result, row)
+	}
+
+	return result, count, rows.Err()
+}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index f1fb3cf99..c8c34907b 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1936,6 +1936,131 @@ func (d *Database) InsertReportedEvent(
 	return reportID, err
 }
 
+// QueryAdminEventReports returns event reports given a filter.
+func (d *Database) QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID string, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error) {
+	// Filter on roomID, if requested
+	var roomNID types.RoomNID
+	if roomID != "" {
+		roomInfo, err := d.RoomInfo(ctx, roomID)
+		if err != nil {
+			return nil, 0, err
+		}
+		roomNID = roomInfo.RoomNID
+	}
+
+	// Same as above, but for userID
+	var userNID types.EventStateKeyNID
+	if userID != "" {
+		stateKeysMap, err := d.EventStateKeyNIDs(ctx, []string{userID})
+		if err != nil {
+			return nil, 0, err
+		}
+		if len(stateKeysMap) != 1 {
+			return nil, 0, fmt.Errorf("failed to get eventStateKeyNID for %s", userID)
+		}
+		userNID = stateKeysMap[userID]
+	}
+
+	// Query all reported events matching the filters
+	reports, count, err := d.ReportedEventsTable.SelectReportedEvents(ctx, nil, from, limit, backwards, userNID, roomNID)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to SelectReportedEvents: %w", err)
+	}
+
+	// TODO: The below code may be inefficient due to many DB round trips and needs to be revisited.
+	// 	For the time being, this is "good enough".
+	qryRoomNIDs := make([]types.RoomNID, 0, len(reports))
+	qryEventNIDs := make([]types.EventNID, 0, len(reports))
+	qryStateKeyNIDs := make([]types.EventStateKeyNID, 0, len(reports))
+	for _, report := range reports {
+		qryRoomNIDs = append(qryRoomNIDs, report.RoomNID)
+		qryEventNIDs = append(qryEventNIDs, report.EventNID)
+		qryStateKeyNIDs = append(qryStateKeyNIDs, report.ReportingUserNID, report.SenderNID)
+	}
+
+	// This also de-dupes the roomIDs, otherwise we would query the same
+	// roomIDs in GetBulkStateContent multiple times
+	roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, qryRoomNIDs)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// TODO: replace this with something more efficient, as it loads the entire state snapshot.
+	stateContent, err := d.GetBulkStateContent(ctx, roomIDs, []gomatrixserverlib.StateKeyTuple{
+		{EventType: spec.MRoomName, StateKey: ""},
+		{EventType: spec.MRoomCanonicalAlias, StateKey: ""},
+	}, false)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	eventIDMap, err := d.EventIDs(ctx, qryEventNIDs)
+	if err != nil {
+		logrus.WithError(err).Error("unable to map eventNIDs to eventIDs")
+		return nil, 0, err
+	}
+	if len(eventIDMap) != len(qryEventNIDs) {
+		return nil, 0, fmt.Errorf("expected %d eventIDs, got %d", len(qryEventNIDs), len(eventIDMap))
+	}
+
+	// Get a map from EventStateKeyNID to userID
+	userNIDMap, err := d.EventStateKeys(ctx, qryStateKeyNIDs)
+	if err != nil {
+		logrus.WithError(err).Error("unable to map userNIDs to userIDs")
+		return nil, 0, err
+	}
+
+	// Create a cache from roomNID to roomID to avoid hitting the DB again
+	roomNIDIDCache := make(map[types.RoomNID]string, len(roomIDs))
+	for i := 0; i < len(reports); i++ {
+		cachedRoomID := roomNIDIDCache[reports[i].RoomNID]
+		if cachedRoomID == "" {
+			// We need to query this again, as we otherwise don't have a way to match roomNID -> roomID
+			roomIDs, err = d.RoomsTable.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{reports[i].RoomNID})
+			if err != nil {
+				return nil, 0, err
+			}
+			if len(roomIDs) == 0 || len(roomIDs) > 1 {
+				logrus.Warnf("unable to map roomNID %d to a roomID, was this room deleted?", roomNID)
+				continue
+			}
+			roomNIDIDCache[reports[i].RoomNID] = roomIDs[0]
+			cachedRoomID = roomIDs[0]
+		}
+
+		reports[i].EventID = eventIDMap[reports[i].EventNID]
+		reports[i].RoomID = cachedRoomID
+		roomName, canonicalAlias := findRoomNameAndCanonicalAlias(stateContent, cachedRoomID)
+		reports[i].RoomName = roomName
+		reports[i].CanonicalAlias = canonicalAlias
+		reports[i].Sender = userNIDMap[reports[i].SenderNID]
+		reports[i].UserID = userNIDMap[reports[i].ReportingUserNID]
+	}
+
+	return reports, count, nil
+}
+
+// findRoomNameAndCanonicalAlias loops over events to find the corresponding room name and canonicalAlias
+// for a given roomID.
+func findRoomNameAndCanonicalAlias(events []tables.StrippedEvent, roomID string) (name, canonicalAlias string) {
+	for _, ev := range events {
+		if ev.RoomID != roomID {
+			continue
+		}
+		if ev.EventType == spec.MRoomName {
+			name = ev.ContentValue
+		}
+		if ev.EventType == spec.MRoomCanonicalAlias {
+			canonicalAlias = ev.ContentValue
+		}
+		// We found both wanted values, break the loop
+		if name != "" && canonicalAlias != "" {
+			break
+		}
+	}
+	return name, canonicalAlias
+}
+
 // FIXME TODO: Remove all this - horrible dupe with roomserver/state. Can't use the original impl because of circular loops
 // it should live in this package!
 
diff --git a/roomserver/storage/sqlite3/reported_events_table.go b/roomserver/storage/sqlite3/reported_events_table.go
index 4a8582fc4..65584f4cf 100644
--- a/roomserver/storage/sqlite3/reported_events_table.go
+++ b/roomserver/storage/sqlite3/reported_events_table.go
@@ -19,7 +19,9 @@ import (
 	"database/sql"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -44,8 +46,38 @@ const insertReportedEventSQL = `
 	RETURNING id
 `
 
+const selectReportedEventsDescSQL = `
+WITH countReports AS (
+    SELECT count(*) as report_count
+    FROM roomserver_reported_events
+    WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2)
+)
+SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts
+FROM roomserver_reported_events, countReports
+WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2)
+ORDER BY received_ts DESC
+LIMIT $3
+OFFSET $4
+`
+
+const selectReportedEventsAscSQL = `
+WITH countReports AS (
+    SELECT count(*) as report_count
+    FROM roomserver_reported_events
+    WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2)
+)
+SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts
+FROM roomserver_reported_events, countReports
+WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2)
+ORDER BY received_ts ASC
+LIMIT $3
+OFFSET $4
+`
+
 type reportedEventsStatements struct {
-	insertReportedEventsStmt *sql.Stmt
+	insertReportedEventsStmt     *sql.Stmt
+	selectReportedEventsDescStmt *sql.Stmt
+	selectReportedEventsAscStmt  *sql.Stmt
 }
 
 func CreateReportedEventsTable(db *sql.DB) error {
@@ -58,6 +90,8 @@ func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) {
 
 	return s, sqlutil.StatementList{
 		{&s.insertReportedEventsStmt, insertReportedEventSQL},
+		{&s.selectReportedEventsDescStmt, selectReportedEventsDescSQL},
+		{&s.selectReportedEventsAscStmt, selectReportedEventsAscSQL},
 	}.Prepare(db)
 }
 
@@ -85,3 +119,62 @@ func (r *reportedEventsStatements) InsertReportedEvent(
 	).Scan(&reportID)
 	return reportID, err
 }
+
+func (r *reportedEventsStatements) SelectReportedEvents(
+	ctx context.Context,
+	txn *sql.Tx,
+	from, limit uint64,
+	backwards bool,
+	reportingUserID types.EventStateKeyNID,
+	roomNID types.RoomNID,
+) ([]api.QueryAdminEventReportsResponse, int64, error) {
+
+	var stmt *sql.Stmt
+	if backwards {
+		stmt = sqlutil.TxStmt(txn, r.selectReportedEventsDescStmt)
+	} else {
+		stmt = sqlutil.TxStmt(txn, r.selectReportedEventsAscStmt)
+	}
+
+	var qryRoomNID *types.RoomNID
+	if roomNID > 0 {
+		qryRoomNID = &roomNID
+	}
+	var qryReportingUser *types.EventStateKeyNID
+	if reportingUserID > 0 {
+		qryReportingUser = &reportingUserID
+	}
+
+	rows, err := stmt.QueryContext(ctx,
+		qryRoomNID,
+		qryReportingUser,
+		limit,
+		from,
+	)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectReportedEvents: failed to close rows")
+
+	var result []api.QueryAdminEventReportsResponse
+	var row api.QueryAdminEventReportsResponse
+	var count int64
+	for rows.Next() {
+		if err = rows.Scan(
+			&count,
+			&row.ID,
+			&row.RoomNID,
+			&row.EventNID,
+			&row.ReportingUserNID,
+			&row.SenderNID,
+			&row.Reason,
+			&row.Score,
+			&row.ReceivedTS,
+		); err != nil {
+			return nil, 0, err
+		}
+		result = append(result, row)
+	}
+
+	return result, count, rows.Err()
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index cc0114372..d41b50329 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -6,6 +6,7 @@ import (
 	"database/sql"
 	"errors"
 
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/tidwall/gjson"
@@ -138,6 +139,14 @@ type ReportedEvents interface {
 		reason string,
 		score int64,
 	) (int64, error)
+	SelectReportedEvents(
+		ctx context.Context,
+		txn *sql.Tx,
+		from, limit uint64,
+		backwards bool,
+		reportingUserID types.EventStateKeyNID,
+		roomNID types.RoomNID,
+	) ([]api.QueryAdminEventReportsResponse, int64, error)
 }
 
 type MembershipState int64

From 81f73c9f8df6dd3078a93b6ca978ecbb9c95df16 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 22 Mar 2024 22:33:23 +0100
Subject: [PATCH 29/46] Reuse existing NATS connection (#3345)

If using external NATS, we opened unnecessary connections. This now
re-uses existing connections.

[skip ci]
---
 setup/jetstream/nats.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 8820e86b2..8630a1411 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -38,7 +38,12 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
 	defer natsLock.Unlock()
 	// check if we need an in-process NATS Server
 	if len(cfg.Addresses) != 0 {
-		return setupNATS(process, cfg, nil)
+		// reuse existing connections
+		if s.nc != nil {
+			return s.js, s.nc
+		}
+		s.js, s.nc = setupNATS(process, cfg, nil)
+		return s.js, s.nc
 	}
 	if s.Server == nil {
 		var err error

From ad0a7d09e89fe18c9e2b08f23f5817a5231c6074 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 22 Mar 2024 22:54:29 +0100
Subject: [PATCH 30/46] Add getting/deleting single event report (#3344)

Based on https://github.com/matrix-org/dendrite/pull/3342

Adds `GET /_synapse/admin/v1/event_reports/{reportID}` and `DELETE
/_synapse/admin/v1/event_reports/{reportID}`
---
 clientapi/admin_test.go                       | 137 ++++++++++++++++++
 clientapi/routing/admin.go                    |  48 ++++++
 clientapi/routing/routing.go                  |  22 ++-
 roomserver/api/api.go                         |   2 +
 roomserver/api/query.go                       |   5 +
 roomserver/internal/perform/perform_admin.go  |   4 +
 roomserver/internal/query/query.go            |   5 +
 roomserver/storage/interface.go               |   2 +
 .../storage/postgres/reported_events_table.go |  41 ++++++
 roomserver/storage/shared/storage.go          |  68 +++++++++
 .../storage/sqlite3/reported_events_table.go  |  41 ++++++
 roomserver/storage/tables/interface.go        |   6 +
 12 files changed, 380 insertions(+), 1 deletion(-)

diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go
index 2444f0be1..b2adeb757 100644
--- a/clientapi/admin_test.go
+++ b/clientapi/admin_test.go
@@ -1336,3 +1336,140 @@ func TestAdminQueryEventReports(t *testing.T) {
 		})
 	})
 }
+
+func TestEventReportsGetDelete(t *testing.T) {
+	alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	// Add a name and alias
+	roomName := "Testing"
+	alias := "#testing"
+	room.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": roomName}, test.WithStateKey(""))
+	room.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": alias}, test.WithStateKey(""))
+
+	// Join the rooms with Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	// Create a few events to report
+
+	eventIDToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		// Report the event
+		req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventIDToReport.EventID()), strings.NewReader(string(body)))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+		routers.Client.ServeHTTP(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+		}
+
+		t.Run("Can not query with invalid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/abc", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusBadRequest {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Can query with valid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			resp := api.QueryAdminEventReportResponse{}
+			if err = json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			// test a few things
+			if resp.EventID != eventIDToReport.EventID() {
+				t.Fatalf("expected eventID to be %s, got %s instead", eventIDToReport.EventID(), resp.EventID)
+			}
+			if resp.RoomName != roomName {
+				t.Fatalf("expected roomName to be %s, got %s instead", roomName, resp.RoomName)
+			}
+			if resp.CanonicalAlias != alias {
+				t.Fatalf("expected alias to be %s, got %s instead", alias, resp.CanonicalAlias)
+			}
+			if reflect.DeepEqual(resp.EventJSON, eventIDToReport.JSON()) {
+				t.Fatal("mismatching eventJSON")
+			}
+		})
+
+		t.Run("Can delete with a valid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodDelete, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Can not query deleted report", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code == http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+	})
+}
diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index e91e098a4..68e62b08f 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -530,6 +530,54 @@ func GetEventReports(
 	}
 }
 
+func GetEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	report, err := rsAPI.QueryAdminEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: report,
+	}
+}
+
+func DeleteEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	err = rsAPI.PerformAdminDeleteEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
 func parseUint64OrDefault(input string, defaultValue uint64) uint64 {
 	v, err := strconv.ParseUint(input, 10, 64)
 	if err != nil {
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index dc63a2c24..c96c6538c 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -1535,7 +1535,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	synapseAdminRouter.Handle("/admin/v1/event_reports",
-		httputil.MakeAdminAPI("admin_report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeAdminAPI("admin_report_events", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			from := parseUint64OrDefault(req.URL.Query().Get("from"), 0)
 			limit := parseUint64OrDefault(req.URL.Query().Get("limit"), 100)
 			dir := req.URL.Query().Get("dir")
@@ -1547,4 +1547,24 @@ func Setup(
 			return GetEventReports(req, rsAPI, from, limit, backwards, userID, roomID)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event_delete", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return DeleteEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodDelete, http.MethodOptions)
 }
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index ac4ea5bab..dffb6d479 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -272,6 +272,8 @@ type ClientRoomserverAPI interface {
 		score int64,
 	) (int64, error)
 	QueryAdminEventReports(ctx context.Context, from, limit uint64, backwards bool, userID, roomID string) ([]QueryAdminEventReportsResponse, int64, error)
+	QueryAdminEventReport(ctx context.Context, reportID uint64) (QueryAdminEventReportResponse, error)
+	PerformAdminDeleteEventReport(ctx context.Context, reportID uint64) error
 }
 
 type UserRoomserverAPI interface {
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index 9a7acab9a..c4c019f99 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -363,6 +363,11 @@ type QueryAdminEventReportsResponse struct {
 	ReceivedTS       spec.Timestamp         `json:"received_ts"`
 }
 
+type QueryAdminEventReportResponse struct {
+	QueryAdminEventReportsResponse
+	EventJSON json.RawMessage `json:"event_json"`
+}
+
 // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) {
 	se := make(map[string]string)
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index ae203854b..1b8817234 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -354,3 +354,7 @@ func (r *Admin) PerformAdminDownloadState(
 
 	return nil
 }
+
+func (r *Admin) PerformAdminDeleteEventReport(ctx context.Context, reportID uint64) error {
+	return r.DB.AdminDeleteEventReport(ctx, reportID)
+}
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index b1b11df31..886d00492 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -1109,3 +1109,8 @@ func (r *Queryer) RoomsWithACLs(ctx context.Context) ([]string, error) {
 func (r *Queryer) QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error) {
 	return r.DB.QueryAdminEventReports(ctx, from, limit, backwards, userID, roomID)
 }
+
+// QueryAdminEventReport returns a single event report.
+func (r *Queryer) QueryAdminEventReport(ctx context.Context, reportID uint64) (api.QueryAdminEventReportResponse, error) {
+	return r.DB.QueryAdminEventReport(ctx, reportID)
+}
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index eb169cbbb..ab105e6f9 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -196,6 +196,8 @@ type Database interface {
 	// RoomsWithACLs returns all room IDs for rooms with ACLs
 	RoomsWithACLs(ctx context.Context) ([]string, error)
 	QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID string, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error)
+	QueryAdminEventReport(ctx context.Context, reportID uint64) (api.QueryAdminEventReportResponse, error)
+	AdminDeleteEventReport(ctx context.Context, reportID uint64) error
 }
 
 type UserRoomKeys interface {
diff --git a/roomserver/storage/postgres/reported_events_table.go b/roomserver/storage/postgres/reported_events_table.go
index 70393833c..c46f47b34 100644
--- a/roomserver/storage/postgres/reported_events_table.go
+++ b/roomserver/storage/postgres/reported_events_table.go
@@ -75,10 +75,20 @@ OFFSET $3
 LIMIT $4
 `
 
+const selectReportedEventSQL = `
+SELECT id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts
+FROM roomserver_reported_events
+WHERE id = $1
+`
+
+const deleteReportedEventSQL = `DELETE FROM roomserver_reported_events WHERE id = $1`
+
 type reportedEventsStatements struct {
 	insertReportedEventsStmt     *sql.Stmt
 	selectReportedEventsDescStmt *sql.Stmt
 	selectReportedEventsAscStmt  *sql.Stmt
+	selectReportedEventStmt      *sql.Stmt
+	deleteReportedEventStmt      *sql.Stmt
 }
 
 func CreateReportedEventsTable(db *sql.DB) error {
@@ -93,6 +103,8 @@ func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) {
 		{&s.insertReportedEventsStmt, insertReportedEventSQL},
 		{&s.selectReportedEventsDescStmt, selectReportedEventsDescSQL},
 		{&s.selectReportedEventsAscStmt, selectReportedEventsAscSQL},
+		{&s.selectReportedEventStmt, selectReportedEventSQL},
+		{&s.deleteReportedEventStmt, deleteReportedEventSQL},
 	}.Prepare(db)
 }
 
@@ -178,3 +190,32 @@ func (r *reportedEventsStatements) SelectReportedEvents(
 
 	return result, count, rows.Err()
 }
+
+func (r *reportedEventsStatements) SelectReportedEvent(
+	ctx context.Context,
+	txn *sql.Tx,
+	reportID uint64,
+) (api.QueryAdminEventReportResponse, error) {
+	stmt := sqlutil.TxStmt(txn, r.selectReportedEventStmt)
+
+	var row api.QueryAdminEventReportResponse
+	if err := stmt.QueryRowContext(ctx, reportID).Scan(
+		&row.ID,
+		&row.RoomNID,
+		&row.EventNID,
+		&row.ReportingUserNID,
+		&row.SenderNID,
+		&row.Reason,
+		&row.Score,
+		&row.ReceivedTS,
+	); err != nil {
+		return api.QueryAdminEventReportResponse{}, err
+	}
+	return row, nil
+}
+
+func (r *reportedEventsStatements) DeleteReportedEvent(ctx context.Context, txn *sql.Tx, reportID uint64) error {
+	stmt := sqlutil.TxStmt(txn, r.deleteReportedEventStmt)
+	_, err := stmt.ExecContext(ctx, reportID)
+	return err
+}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index c8c34907b..7b04641bf 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -2040,6 +2040,74 @@ func (d *Database) QueryAdminEventReports(ctx context.Context, from uint64, limi
 	return reports, count, nil
 }
 
+func (d *Database) QueryAdminEventReport(ctx context.Context, reportID uint64) (api.QueryAdminEventReportResponse, error) {
+
+	report, err := d.ReportedEventsTable.SelectReportedEvent(ctx, nil, reportID)
+	if err != nil {
+		return api.QueryAdminEventReportResponse{}, err
+	}
+
+	// Get a map from EventStateKeyNID to userID
+	userNIDMap, err := d.EventStateKeys(ctx, []types.EventStateKeyNID{report.ReportingUserNID, report.SenderNID})
+	if err != nil {
+		logrus.WithError(err).Error("unable to map userNIDs to userIDs")
+		return report, err
+	}
+
+	roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{report.RoomNID})
+	if err != nil {
+		return report, err
+	}
+
+	if len(roomIDs) != 1 {
+		return report, fmt.Errorf("expected one roomID, got %d", len(roomIDs))
+	}
+
+	// TODO: replace this with something more efficient, as it loads the entire state snapshot.
+	stateContent, err := d.GetBulkStateContent(ctx, roomIDs, []gomatrixserverlib.StateKeyTuple{
+		{EventType: spec.MRoomName, StateKey: ""},
+		{EventType: spec.MRoomCanonicalAlias, StateKey: ""},
+	}, false)
+	if err != nil {
+		return report, err
+	}
+
+	eventIDMap, err := d.EventIDs(ctx, []types.EventNID{report.EventNID})
+	if err != nil {
+		logrus.WithError(err).Error("unable to map eventNIDs to eventIDs")
+		return report, err
+	}
+	if len(eventIDMap) != 1 {
+		return report, fmt.Errorf("expected %d eventIDs, got %d", 1, len(eventIDMap))
+	}
+
+	eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, nil, []types.EventNID{report.EventNID})
+	if err != nil {
+		return report, err
+	}
+	if len(eventJSONs) != 1 {
+		return report, fmt.Errorf("expected %d eventJSONs, got %d", 1, len(eventJSONs))
+	}
+
+	roomName, canonicalAlias := findRoomNameAndCanonicalAlias(stateContent, roomIDs[0])
+
+	report.Sender = userNIDMap[report.SenderNID]
+	report.UserID = userNIDMap[report.ReportingUserNID]
+	report.RoomID = roomIDs[0]
+	report.RoomName = roomName
+	report.CanonicalAlias = canonicalAlias
+	report.EventID = eventIDMap[report.EventNID]
+	report.EventJSON = eventJSONs[0].EventJSON
+
+	return report, nil
+}
+
+func (d *Database) AdminDeleteEventReport(ctx context.Context, reportID uint64) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.ReportedEventsTable.DeleteReportedEvent(ctx, txn, reportID)
+	})
+}
+
 // findRoomNameAndCanonicalAlias loops over events to find the corresponding room name and canonicalAlias
 // for a given roomID.
 func findRoomNameAndCanonicalAlias(events []tables.StrippedEvent, roomID string) (name, canonicalAlias string) {
diff --git a/roomserver/storage/sqlite3/reported_events_table.go b/roomserver/storage/sqlite3/reported_events_table.go
index 65584f4cf..b72cb0685 100644
--- a/roomserver/storage/sqlite3/reported_events_table.go
+++ b/roomserver/storage/sqlite3/reported_events_table.go
@@ -74,10 +74,20 @@ LIMIT $3
 OFFSET $4
 `
 
+const selectReportedEventSQL = `
+SELECT id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts
+FROM roomserver_reported_events
+WHERE id = $1
+`
+
+const deleteReportedEventSQL = `DELETE FROM roomserver_reported_events WHERE id = $1`
+
 type reportedEventsStatements struct {
 	insertReportedEventsStmt     *sql.Stmt
 	selectReportedEventsDescStmt *sql.Stmt
 	selectReportedEventsAscStmt  *sql.Stmt
+	selectReportedEventStmt      *sql.Stmt
+	deleteReportedEventStmt      *sql.Stmt
 }
 
 func CreateReportedEventsTable(db *sql.DB) error {
@@ -92,6 +102,8 @@ func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) {
 		{&s.insertReportedEventsStmt, insertReportedEventSQL},
 		{&s.selectReportedEventsDescStmt, selectReportedEventsDescSQL},
 		{&s.selectReportedEventsAscStmt, selectReportedEventsAscSQL},
+		{&s.selectReportedEventStmt, selectReportedEventSQL},
+		{&s.deleteReportedEventStmt, deleteReportedEventSQL},
 	}.Prepare(db)
 }
 
@@ -178,3 +190,32 @@ func (r *reportedEventsStatements) SelectReportedEvents(
 
 	return result, count, rows.Err()
 }
+
+func (r *reportedEventsStatements) SelectReportedEvent(
+	ctx context.Context,
+	txn *sql.Tx,
+	reportID uint64,
+) (api.QueryAdminEventReportResponse, error) {
+	stmt := sqlutil.TxStmt(txn, r.selectReportedEventStmt)
+
+	var row api.QueryAdminEventReportResponse
+	if err := stmt.QueryRowContext(ctx, reportID).Scan(
+		&row.ID,
+		&row.RoomNID,
+		&row.EventNID,
+		&row.ReportingUserNID,
+		&row.SenderNID,
+		&row.Reason,
+		&row.Score,
+		&row.ReceivedTS,
+	); err != nil {
+		return api.QueryAdminEventReportResponse{}, err
+	}
+	return row, nil
+}
+
+func (r *reportedEventsStatements) DeleteReportedEvent(ctx context.Context, txn *sql.Tx, reportID uint64) error {
+	stmt := sqlutil.TxStmt(txn, r.deleteReportedEventStmt)
+	_, err := stmt.ExecContext(ctx, reportID)
+	return err
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index d41b50329..02f6992c4 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -147,6 +147,12 @@ type ReportedEvents interface {
 		reportingUserID types.EventStateKeyNID,
 		roomNID types.RoomNID,
 	) ([]api.QueryAdminEventReportsResponse, int64, error)
+	SelectReportedEvent(
+		ctx context.Context,
+		txn *sql.Tx,
+		reportID uint64,
+	) (api.QueryAdminEventReportResponse, error)
+	DeleteReportedEvent(ctx context.Context, txn *sql.Tx, reportID uint64) error
 }
 
 type MembershipState int64

From b732eede2738dcf88727b403fa1e9494466bf7a8 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Mar 2024 20:40:45 +0100
Subject: [PATCH 31/46] Fix spaces over federation (#3347)

Fixes #2504

 A few issues with the previous iteration:
- We never returned `inaccessible_children`, which (if I read the code
correctly), made Synapse raise an error and thus not returning the
requested rooms
- For restricted rooms, we didn't return the list of allowed rooms
---
 clientapi/routing/room_hierarchy.go           |  2 +-
 federationapi/routing/query.go                |  7 +-
 go.mod                                        |  2 +-
 go.sum                                        |  4 +-
 roomserver/api/api.go                         |  7 +-
 roomserver/api/wrapper.go                     |  6 +-
 .../internal/query/query_room_hierarchy.go    | 77 ++++++++++++-------
 7 files changed, 68 insertions(+), 37 deletions(-)

diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go
index 2884d2c32..cf9d43dd1 100644
--- a/clientapi/routing/room_hierarchy.go
+++ b/clientapi/routing/room_hierarchy.go
@@ -138,7 +138,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str
 		walker = *cachedWalker
 	}
 
-	discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
+	discoveredRooms, _, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
 
 	if err != nil {
 		switch err.(type) {
diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go
index 327ba9b08..dac9b1b34 100644
--- a/federationapi/routing/query.go
+++ b/federationapi/routing/query.go
@@ -146,7 +146,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques
 	}
 
 	walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1)
-	discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1)
+	discoveredRooms, inaccessibleRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1)
 
 	if err != nil {
 		switch err.(type) {
@@ -175,8 +175,9 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques
 	return util.JSONResponse{
 		Code: 200,
 		JSON: fclient.RoomHierarchyResponse{
-			Room:     discoveredRooms[0],
-			Children: discoveredRooms[1:],
+			Room:                 discoveredRooms[0],
+			Children:             discoveredRooms[1:],
+			InaccessibleChildren: inaccessibleRooms,
 		},
 	}
 }
diff --git a/go.mod b/go.mod
index 0dde02252..2d11985d7 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20240326183347-e8077abf519a
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.17
diff --git a/go.sum b/go.sum
index f526d6586..7d32fe964 100644
--- a/go.sum
+++ b/go.sum
@@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7 h1:EaUvK2ay6cxMxeshC1p6QswS9+rQFbUc2YerkRFyVXQ=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240326183347-e8077abf519a h1:K+lE7Bp2g62Ykfzd9nqxzdXZseOzVWZ494OhQsiLJ1U=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240326183347-e8077abf519a/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index dffb6d479..b2b319244 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -141,7 +141,12 @@ type QueryRoomHierarchyAPI interface {
 	//
 	// If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it
 	// can be cached.
-	QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *RoomHierarchyWalker, error)
+	QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) (
+		hierarchyRooms []fclient.RoomHierarchyRoom,
+		inaccessibleRooms []string,
+		hierarchyWalker *RoomHierarchyWalker,
+		err error,
+	)
 }
 
 type QueryMembershipAPI interface {
diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go
index 0ad5d2013..4979d18c7 100644
--- a/roomserver/api/wrapper.go
+++ b/roomserver/api/wrapper.go
@@ -189,7 +189,7 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkS
 			RoomID: roomID,
 		}
 		joinCount := 0
-		var joinRule, guestAccess string
+		var guestAccess string
 		for tuple, contentVal := range data {
 			if tuple.EventType == spec.MRoomMember && contentVal == "join" {
 				joinCount++
@@ -210,12 +210,12 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkS
 				pub.WorldReadable = contentVal == "world_readable"
 			// need both of these to determine whether guests can join
 			case joinRuleTuple:
-				joinRule = contentVal
+				pub.JoinRule = contentVal
 			case guestTuple:
 				guestAccess = contentVal
 			}
 		}
-		if joinRule == spec.Public && guestAccess == "can_join" {
+		if pub.JoinRule == spec.Public && guestAccess == "can_join" {
 			pub.GuestCanJoin = true
 		}
 		pub.JoinedMembersCount = joinCount
diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go
index 5f55980f0..3fc613192 100644
--- a/roomserver/internal/query/query_room_hierarchy.go
+++ b/roomserver/internal/query/query_room_hierarchy.go
@@ -39,9 +39,14 @@ import (
 //
 // If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it
 // can be cached.
-func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *roomserver.RoomHierarchyWalker, error) {
-	if authorised, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised {
-		return nil, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")}
+func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) (
+	[]fclient.RoomHierarchyRoom,
+	[]string,
+	*roomserver.RoomHierarchyWalker,
+	error,
+) {
+	if authorised, _, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised {
+		return nil, []string{walker.RootRoomID.String()}, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")}
 	}
 
 	discoveredRooms := []fclient.RoomHierarchyRoom{}
@@ -50,6 +55,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 	unvisited := make([]roomserver.RoomHierarchyWalkerQueuedRoom, len(walker.Unvisited))
 	copy(unvisited, walker.Unvisited)
 	processed := walker.Processed.Copy()
+	inaccessible := []string{}
 
 	// Depth first -> stack data structure
 	for len(unvisited) > 0 {
@@ -108,7 +114,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 				// as these children may be rooms we do know about.
 				roomType = spec.MSpace
 			}
-		} else if authorised, isJoinedOrInvited := authorised(ctx, querier, walker.Caller, queuedRoom.RoomID, queuedRoom.ParentRoomID); authorised {
+		} else if authorised, isJoinedOrInvited, allowedRoomIDs := authorised(ctx, querier, walker.Caller, queuedRoom.RoomID, queuedRoom.ParentRoomID); authorised {
 			// Get all `m.space.child` state events for this room
 			events, err := childReferences(ctx, querier, walker.SuggestedOnly, queuedRoom.RoomID)
 			if err != nil {
@@ -125,14 +131,18 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 			}
 
 			discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{
-				PublicRoom:    *pubRoom,
-				RoomType:      roomType,
-				ChildrenState: events,
+				PublicRoom:     *pubRoom,
+				RoomType:       roomType,
+				ChildrenState:  events,
+				AllowedRoomIDs: allowedRoomIDs,
 			})
 			// don't walk children if the user is not joined/invited to the space
 			if !isJoinedOrInvited {
 				continue
 			}
+		} else if !authorised {
+			inaccessible = append(inaccessible, queuedRoom.RoomID.String())
+			continue
 		} else {
 			// room exists but user is not authorised
 			continue
@@ -149,6 +159,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 		// We need to invert the order here because the child events are lo->hi on the timestamp,
 		// so we need to ensure we pop in the same lo->hi order, which won't be the case if we
 		// insert the highest timestamp last in a stack.
+	extendQueueLoop:
 		for i := len(discoveredChildEvents) - 1; i >= 0; i-- {
 			spaceContent := struct {
 				Via []string `json:"via"`
@@ -161,6 +172,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 			if err != nil {
 				util.GetLogger(ctx).WithError(err).WithField("invalid_room_id", ev.StateKey).WithField("parent_room_id", queuedRoom.RoomID).Warn("Invalid room ID in m.space.child state event")
 			} else {
+				// Make sure not to queue inaccessible rooms
+				for _, inaccessibleRoomID := range inaccessible {
+					if inaccessibleRoomID == childRoomID.String() {
+						continue extendQueueLoop
+					}
+				}
 				unvisited = append(unvisited, roomserver.RoomHierarchyWalkerQueuedRoom{
 					RoomID:       *childRoomID,
 					ParentRoomID: &queuedRoom.RoomID,
@@ -173,7 +190,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 
 	if len(unvisited) == 0 {
 		// If no more rooms to walk, then don't return a walker for future pages
-		return discoveredRooms, nil, nil
+		return discoveredRooms, inaccessible, nil, nil
 	} else {
 		// If there are more rooms to walk, then return a new walker to resume walking from (for querying more pages)
 		newWalker := roomserver.RoomHierarchyWalker{
@@ -185,22 +202,25 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r
 			Processed:     processed,
 		}
 
-		return discoveredRooms, &newWalker, nil
+		return discoveredRooms, inaccessible, &newWalker, nil
 	}
 
 }
 
 // authorised returns true iff the user is joined this room or the room is world_readable
-func authorised(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed, isJoinedOrInvited bool) {
+func authorised(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed, isJoinedOrInvited bool, resultAllowedRoomIDs []string) {
 	if clientCaller := caller.Device(); clientCaller != nil {
 		return authorisedUser(ctx, querier, clientCaller, roomID, parentRoomID)
-	} else {
-		return authorisedServer(ctx, querier, roomID, *caller.ServerName()), false
 	}
+	if serverCaller := caller.ServerName(); serverCaller != nil {
+		authed, resultAllowedRoomIDs = authorisedServer(ctx, querier, roomID, *serverCaller)
+		return authed, false, resultAllowedRoomIDs
+	}
+	return false, false, resultAllowedRoomIDs
 }
 
 // authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable
-func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, callerServerName spec.ServerName) bool {
+func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, callerServerName spec.ServerName) (bool, []string) {
 	// Check history visibility / join rules first
 	hisVisTuple := gomatrixserverlib.StateKeyTuple{
 		EventType: spec.MRoomHistoryVisibility,
@@ -219,13 +239,13 @@ func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID,
 	}, &queryRoomRes)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState")
-		return false
+		return false, []string{}
 	}
 	hisVisEv := queryRoomRes.StateEvents[hisVisTuple]
 	if hisVisEv != nil {
 		hisVis, _ := hisVisEv.HistoryVisibility()
 		if hisVis == "world_readable" {
-			return true
+			return true, []string{}
 		}
 	}
 
@@ -238,19 +258,23 @@ func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID,
 		rule, ruleErr := joinRuleEv.JoinRule()
 		if ruleErr != nil {
 			util.GetLogger(ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule")
-			return false
+			return false, []string{}
 		}
 
 		if rule == spec.Public || rule == spec.Knock {
-			return true
+			return true, []string{}
 		}
 
-		if rule == spec.Restricted {
+		if rule == spec.Restricted || rule == spec.KnockRestricted {
 			allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, restrictedJoinRuleAllowedRooms(ctx, joinRuleEv)...)
 		}
 	}
 
 	// check if server is joined to any allowed room
+	resultAllowedRoomIDs := make([]string, 0, len(allowJoinedToRoomIDs))
+	for _, allowedRoomID := range allowJoinedToRoomIDs {
+		resultAllowedRoomIDs = append(resultAllowedRoomIDs, allowedRoomID.String())
+	}
 	for _, allowedRoomID := range allowJoinedToRoomIDs {
 		var queryRes fs.QueryJoinedHostServerNamesInRoomResponse
 		err = querier.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{
@@ -262,18 +286,18 @@ func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID,
 		}
 		for _, srv := range queryRes.ServerNames {
 			if srv == callerServerName {
-				return true
+				return true, resultAllowedRoomIDs[1:]
 			}
 		}
 	}
 
-	return false
+	return false, resultAllowedRoomIDs[1:]
 }
 
 // authorisedUser returns true iff the user is invited/joined this room or the room is world_readable
 // or if the room has a public or knock join rule.
 // Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true.
-func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi.Device, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed bool, isJoinedOrInvited bool) {
+func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi.Device, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed bool, isJoinedOrInvited bool, resultAllowedRoomIDs []string) {
 	hisVisTuple := gomatrixserverlib.StateKeyTuple{
 		EventType: spec.MRoomHistoryVisibility,
 		StateKey:  "",
@@ -295,20 +319,20 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi
 	}, &queryRes)
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState")
-		return false, false
+		return false, false, resultAllowedRoomIDs
 	}
 	memberEv := queryRes.StateEvents[roomMemberTuple]
 	if memberEv != nil {
 		membership, _ := memberEv.Membership()
 		if membership == spec.Join || membership == spec.Invite {
-			return true, true
+			return true, true, resultAllowedRoomIDs
 		}
 	}
 	hisVisEv := queryRes.StateEvents[hisVisTuple]
 	if hisVisEv != nil {
 		hisVis, _ := hisVisEv.HistoryVisibility()
 		if hisVis == "world_readable" {
-			return true, false
+			return true, false, resultAllowedRoomIDs
 		}
 	}
 	joinRuleEv := queryRes.StateEvents[joinRuleTuple]
@@ -323,6 +347,7 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi
 			allowedRoomIDs := restrictedJoinRuleAllowedRooms(ctx, joinRuleEv)
 			// check parent is in the allowed set
 			for _, a := range allowedRoomIDs {
+				resultAllowedRoomIDs = append(resultAllowedRoomIDs, a.String())
 				if *parentRoomID == a {
 					allowed = true
 					break
@@ -345,13 +370,13 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi
 				if memberEv != nil {
 					membership, _ := memberEv.Membership()
 					if membership == spec.Join {
-						return true, false
+						return true, false, resultAllowedRoomIDs
 					}
 				}
 			}
 		}
 	}
-	return false, false
+	return false, false, resultAllowedRoomIDs
 }
 
 // helper function to fetch a state event

From 8aa088f7130d571ee6021d8547092162b3895b71 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 8 Apr 2024 07:51:04 +0200
Subject: [PATCH 32/46] Return correct Content-Type for unrecognized requests
 (#3355)

Fixes #3354
---
 internal/httputil/routing.go      | 4 ++--
 internal/httputil/routing_test.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal/httputil/routing.go b/internal/httputil/routing.go
index 2052c798f..f5f1c6528 100644
--- a/internal/httputil/routing.go
+++ b/internal/httputil/routing.go
@@ -66,15 +66,15 @@ func NewRouters() Routers {
 }
 
 var NotAllowedHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusMethodNotAllowed)
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusMethodNotAllowed)
 	unrecognizedErr, _ := json.Marshal(spec.Unrecognized("Unrecognized request")) // nolint:misspell
 	_, _ = w.Write(unrecognizedErr)                                               // nolint:misspell
 }))
 
 var NotFoundCORSHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusNotFound)
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusNotFound)
 	unrecognizedErr, _ := json.Marshal(spec.Unrecognized("Unrecognized request")) // nolint:misspell
 	_, _ = w.Write(unrecognizedErr)                                               // nolint:misspell
 }))
diff --git a/internal/httputil/routing_test.go b/internal/httputil/routing_test.go
index 21e2bf48a..39ccd6213 100644
--- a/internal/httputil/routing_test.go
+++ b/internal/httputil/routing_test.go
@@ -17,7 +17,7 @@ func TestRoutersError(t *testing.T) {
 	if rec.Code != http.StatusNotFound {
 		t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String())
 	}
-	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
+	if ct := rec.Result().Header.Get("Content-Type"); ct != "application/json" {
 		t.Fatalf("unexpected content-type: %s", ct)
 	}
 
@@ -32,7 +32,7 @@ func TestRoutersError(t *testing.T) {
 	if rec.Code != http.StatusMethodNotAllowed {
 		t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String())
 	}
-	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
+	if ct := rec.Result().Header.Get("Content-Type"); ct != "application/json" {
 		t.Fatalf("unexpected content-type: %s", ct)
 	}
 }

From 5c0ceec2a62e2c1003aa72b4fa78bb09e53c3a84 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 9 Apr 2024 07:49:56 +0200
Subject: [PATCH 33/46] Don't attempt to send transactions if Dendrite is
 shutting down (#3356)

This should avoid confusions with logs like:

```
time="2024-04-08T08:38:45.104235081Z" level=error msg="Failed to set \"scs.ems.host\" as assumed offline" func="github.com/matrix-org/dendrite/federationapi/statistics.(*ServerStatistics).Failure" file="github.com/matrix-org/dendrite/federationapi/statistics/statistics.go:204" error="sqlutil.WithTransaction.Begin: sql: database is closed"
time="2024-04-08T08:38:45.104239201Z" level=error msg="Failed to set \"obermui.de\" as assumed offline" func="github.com/matrix-org/dendrite/federationapi/statistics.(*ServerStatistics).Failure" file="github.com/matrix-org/dendrite/federationapi/statistics/statistics.go:204" error="sqlutil.WithTransaction.Begin: sql: database is closed"
```

or

```
time="2024-04-08T08:38:45.105235411Z" level=error msg="Failed to get pending EDUs for \"retro76.net\"" func="github.com/matrix-org/dendrite/federationapi/queue.(*destinationQueue).getPendingFromDatabase" file="github.com/matrix-org/dendritefederationapi/queue/destinationqueue.go:258" error="sqlutil.WithTransaction.Begin: sql: database is closed"
```

[skip ci]
---
 federationapi/queue/destinationqueue.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go
index f51e849fa..87a6fe559 100644
--- a/federationapi/queue/destinationqueue.go
+++ b/federationapi/queue/destinationqueue.go
@@ -294,6 +294,10 @@ func (oq *destinationQueue) checkNotificationsOnClose() {
 
 // backgroundSend is the worker goroutine for sending events.
 func (oq *destinationQueue) backgroundSend() {
+	// Don't try to send transactions if we are shutting down.
+	if oq.process.Context().Err() != nil {
+		return
+	}
 	// Check if a worker is already running, and if it isn't, then
 	// mark it as started.
 	if !oq.running.CompareAndSwap(false, true) {

From 14a6c100977be6484eec419fdc8f496eb27b0677 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 9 Apr 2024 10:24:27 +0200
Subject: [PATCH 34/46] Version 0.13.7 (#3349)

---
 CHANGES.md               |  17 ++++
 go.mod                   |  88 +++++++++---------
 go.sum                   | 192 ++++++++++++++++++---------------------
 helm/dendrite/Chart.yaml |   4 +-
 helm/dendrite/README.md  |   4 +-
 internal/version.go      |   2 +-
 6 files changed, 154 insertions(+), 153 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index c41025e91..9ebde9e36 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,22 @@
 # Changelog
 
+## Dendrite 0.13.7 (2024-04-09)
+
+### Fixes
+
+- Fixed an issue where the displayname/avatar of an invited user was replaced with the inviter's details
+- Improved server startup performance by avoiding unnecessary room ACL queries
+  - This change reduces memory footprint as it caches ACL regex patterns once instead of for each room
+  - Unnecessary Relay related queries have been removed. **Note**: To use relays, you now need to explicitly enable them using the `federation_api.enable_relays` config
+- Fixed space summaries over federation
+- Improved usage of external NATS JetStream by reusing existing connections instead of opening new ones unnecessarily
+
+### Features
+
+- Modernized Appservices (contributed by [tulir](https://github.com/tulir))
+- Added event reporting with Synapse Admin endpoints for querying them
+- Updated dependencies
+
 ## Dendrite 0.13.6 (2024-01-26)
 
 Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
diff --git a/go.mod b/go.mod
index 2d11985d7..0dce89222 100644
--- a/go.mod
+++ b/go.mod
@@ -6,14 +6,14 @@ require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/blevesearch/bleve/v2 v2.3.8
+	github.com/blevesearch/bleve/v2 v2.4.0
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1
 	github.com/docker/docker v24.0.9+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.9
+	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
@@ -22,10 +22,10 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20240326183347-e8077abf519a
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
-	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/nats-io/nats-server/v2 v2.10.7
 	github.com/nats-io/nats.go v1.31.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
@@ -33,26 +33,26 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_golang v1.19.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.2
-	github.com/tidwall/gjson v1.17.0
+	github.com/stretchr/testify v1.9.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/tidwall/sjson v1.2.5
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
-	go.uber.org/atomic v1.10.0
-	golang.org/x/crypto v0.17.0
-	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
+	go.uber.org/atomic v1.11.0
+	golang.org/x/crypto v0.22.0
+	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678
 	golang.org/x/image v0.10.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
-	golang.org/x/sync v0.3.0
-	golang.org/x/term v0.15.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/term v0.19.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
 	maunium.net/go/mautrix v0.15.1
-	modernc.org/sqlite v1.23.1
+	modernc.org/sqlite v1.29.5
 	nhooyr.io/websocket v1.8.7
 )
 
@@ -62,21 +62,23 @@ require (
 	github.com/RoaringBitmap/roaring v1.2.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.5.0 // indirect
-	github.com/blevesearch/bleve_index_api v1.0.5 // indirect
-	github.com/blevesearch/geo v0.1.17 // indirect
+	github.com/blevesearch/bleve_index_api v1.1.6 // indirect
+	github.com/blevesearch/geo v0.1.20 // indirect
+	github.com/blevesearch/go-faiss v1.0.13 // indirect
 	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
 	github.com/blevesearch/gtreap v0.1.1 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
-	github.com/blevesearch/scorch_segment_api/v2 v2.1.4 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.2.9 // indirect
 	github.com/blevesearch/segment v0.9.1 // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
 	github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
-	github.com/blevesearch/vellum v1.0.9 // indirect
-	github.com/blevesearch/zapx/v11 v11.3.7 // indirect
-	github.com/blevesearch/zapx/v12 v12.3.7 // indirect
-	github.com/blevesearch/zapx/v13 v13.3.7 // indirect
-	github.com/blevesearch/zapx/v14 v14.3.7 // indirect
-	github.com/blevesearch/zapx/v15 v15.3.10 // indirect
+	github.com/blevesearch/vellum v1.0.10 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.10 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.10 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.10 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.10 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.13 // indirect
+	github.com/blevesearch/zapx/v16 v16.0.12 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
@@ -91,56 +93,52 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/nats-io/jwt/v2 v2.5.3 // indirect
-	github.com/nats-io/nkeys v0.4.6 // indirect
+	github.com/nats-io/jwt/v2 v2.5.5 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
 	github.com/quic-go/quic-go v0.37.7 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/zerolog v1.29.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.12.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
 	maunium.net/go/maulogger/v2 v2.4.1 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.5 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.0.1 // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	modernc.org/libc v1.41.0 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
 )
 
 go 1.20
diff --git a/go.sum b/go.sum
index 7d32fe964..1aea553f4 100644
--- a/go.sum
+++ b/go.sum
@@ -36,38 +36,42 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
 github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/blevesearch/bleve/v2 v2.3.8 h1:IqFyMJ73n4gY8AmVqM8Sa6EtAZ5beE8yramVqCvs2kQ=
-github.com/blevesearch/bleve/v2 v2.3.8/go.mod h1:Lh9aZEHrLKxwPnW4z4lsBEGnflZQ1V/aWP/t+htsiDw=
-github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw=
-github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
-github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA=
-github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
+github.com/blevesearch/bleve/v2 v2.4.0 h1:2xyg+Wv60CFHYccXc+moGxbL+8QKT/dZK09AewHgKsg=
+github.com/blevesearch/bleve/v2 v2.4.0/go.mod h1:IhQHoFAbHgWKYavb9rQgQEJJVMuY99cKdQ0wPpst2aY=
+github.com/blevesearch/bleve_index_api v1.1.6 h1:orkqDFCBuNU2oHW9hN2YEJmet+TE9orml3FCGbl1cKk=
+github.com/blevesearch/bleve_index_api v1.1.6/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
+github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
+github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
+github.com/blevesearch/go-faiss v1.0.13 h1:zfFs7ZYD0NqXVSY37j0JZjZT1BhE9AE4peJfcx/NB4A=
+github.com/blevesearch/go-faiss v1.0.13/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
 github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
 github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
 github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.9 h1:3nBaSBRFokjE4FtPW3eUDgcAu3KphBg1GP07zy/6Uyk=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.9/go.mod h1:ckbeb7knyOOvAdZinn/ASbB7EA3HoagnJkmEV3J7+sg=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
 github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
 github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
-github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ=
-github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
-github.com/blevesearch/zapx/v11 v11.3.7 h1:Y6yIAF/DVPiqZUA/jNgSLXmqewfzwHzuwfKyfdG+Xaw=
-github.com/blevesearch/zapx/v11 v11.3.7/go.mod h1:Xk9Z69AoAWIOvWudNDMlxJDqSYGf90LS0EfnaAIvXCA=
-github.com/blevesearch/zapx/v12 v12.3.7 h1:DfQ6rsmZfEK4PzzJJRXjiM6AObG02+HWvprlXQ1Y7eI=
-github.com/blevesearch/zapx/v12 v12.3.7/go.mod h1:SgEtYIBGvM0mgIBn2/tQE/5SdrPXaJUaT/kVqpAPxm0=
-github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ubM4d2l8=
-github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s=
-github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20=
-github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w=
-github.com/blevesearch/zapx/v15 v15.3.10 h1:bQ9ZxJCj6rKp873EuVJu2JPxQ+EWQZI1cjJGeroovaQ=
-github.com/blevesearch/zapx/v15 v15.3.10/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY=
+github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
+github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
+github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
+github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
+github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
+github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
+github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
+github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
+github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
+github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
+github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
+github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
+github.com/blevesearch/zapx/v16 v16.0.12 h1:Uccxvjmn+hQ6ywQP+wIiTpdq9LnAviGoryJOmGwAo/I=
+github.com/blevesearch/zapx/v16 v16.0.12/go.mod h1:MYnOshRfSm4C4drxx1LGRI+MVFByykJ2anDY1fxdk9Q=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
@@ -141,7 +145,6 @@ github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -158,8 +161,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c=
 github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
@@ -174,6 +177,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -185,13 +190,11 @@ github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0=
 github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -208,8 +211,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20240326183347-e8077abf519a h1:K+lE7Bp2g62Ykfzd9nqxzdXZseOzVWZ494OhQsiLJ1U=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20240326183347-e8077abf519a/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5 h1:GuxmpyjZQoqb6UFQgKq8Td3wIITlXln/sItqp1jbTTA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -222,10 +225,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
@@ -242,16 +243,18 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
-github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo=
-github.com/nats-io/jwt/v2 v2.5.3/go.mod h1:iysuPemFcc7p4IoYots3IuELSI4EDe9Y0bQMe+I3Bf4=
+github.com/nats-io/jwt/v2 v2.5.5 h1:ROfXb50elFq5c9+1ztaUbdlrArNFl2+fQWP6B8HGEq4=
+github.com/nats-io/jwt/v2 v2.5.5/go.mod h1:ZdWS1nZa6WMZfFwwgpEaqBV8EPGVgOTDHN/wTbz0Y5A=
 github.com/nats-io/nats-server/v2 v2.10.7 h1:f5VDy+GMu7JyuFA0Fef+6TfulfCs5nBTgq7MMkFJx5Y=
 github.com/nats-io/nats-server/v2 v2.10.7/go.mod h1:V2JHOvPiPdtfDXTuEUsthUnCvSDeFrK4Xn9hRo6du7c=
 github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
 github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
-github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
-github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
@@ -276,23 +279,21 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
-github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
-github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
-github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
 github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
 github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU=
 github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
@@ -304,22 +305,18 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
@@ -343,10 +340,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
+go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -354,15 +351,15 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U=
-golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -377,8 +374,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -388,24 +385,22 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -422,13 +417,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -452,8 +447,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
-golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -485,34 +480,25 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
-lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
-lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8=
 maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho=
 maunium.net/go/mautrix v0.15.1 h1:pmCtMjYRpd83+2UL+KTRFYQo5to0373yulimvLK+1k0=
 maunium.net/go/mautrix v0.15.1/go.mod h1:icQIrvz2NldkRLTuzSGzmaeuMUmw+fzO7UVycPeauN8=
-modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
-modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
-modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
-modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
-modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
-modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
-modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
-modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
-modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
-modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
-modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
-modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
+modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.29.5 h1:8l/SQKAjDtZFo9lkJLdk8g9JEOeYRG4/ghStDCCTiTE=
+modernc.org/sqlite v1.29.5/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml
index 2ede0cfe9..a4088872e 100644
--- a/helm/dendrite/Chart.yaml
+++ b/helm/dendrite/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: dendrite
-version: "0.14.0"
-appVersion: "0.13.6"
+version: "0.14.1"
+appVersion: "0.13.7"
 description: Dendrite Matrix Homeserver
 type: application
 icon: https://avatars.githubusercontent.com/u/8418310?s=48&v=4
diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md
index 091f575f0..9259c7903 100644
--- a/helm/dendrite/README.md
+++ b/helm/dendrite/README.md
@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.14.0](https://img.shields.io/badge/Version-0.14.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
+![Version: 0.14.0](https://img.shields.io/badge/Version-0.14.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.7](https://img.shields.io/badge/AppVersion-0.13.7-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -190,4 +190,4 @@ grafana:
 PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`)
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.13.0](https://github.com/norwoodj/helm-docs/releases/v1.13.0)
\ No newline at end of file
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)
\ No newline at end of file
diff --git a/internal/version.go b/internal/version.go
index 1a7575090..8616b82af 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -18,7 +18,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 13
-	VersionPatch = 6
+	VersionPatch = 7
 	VersionTag   = "" // example: "rc1"
 
 	gitRevLen = 7 // 7 matches the displayed characters on github.com

From 5547bf8ca67af325df6560ab13e54d04c39c629b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 30 Apr 2024 23:12:52 +0000
Subject: [PATCH 35/46] Bump golang.org/x/net from 0.21.0 to 0.23.0 (#3365)

Bumps [golang.org/x/net](https://github.com/golang/net) from 0.21.0 to
0.23.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/net/commit/c48da131589f122489348be5dfbcb6457640046f"><code>c48da13</code></a>
http2: fix TestServerContinuationFlood flakes</li>
<li><a
href="https://github.com/golang/net/commit/762b58d1cf6e0779780decad89c6c1523386638d"><code>762b58d</code></a>
http2: fix tipos in comment</li>
<li><a
href="https://github.com/golang/net/commit/ba872109ef2dc8f1da778651bd1fd3792d0e4587"><code>ba87210</code></a>
http2: close connections when receiving too many headers</li>
<li><a
href="https://github.com/golang/net/commit/ebc8168ac8ac742194df729305175940790c55a2"><code>ebc8168</code></a>
all: fix some typos</li>
<li><a
href="https://github.com/golang/net/commit/3678185f8a652e52864c44049a9ea96b7bcc066a"><code>3678185</code></a>
http2: make TestCanonicalHeaderCacheGrowth faster</li>
<li><a
href="https://github.com/golang/net/commit/448c44f9287b6745f958d74aa2a17ec7761c2f13"><code>448c44f</code></a>
http2: remove clientTester</li>
<li><a
href="https://github.com/golang/net/commit/c7877ac4213b2f859831366f5a35b353e0dc9f66"><code>c7877ac</code></a>
http2: convert the remaining clientTester tests to testClientConn</li>
<li><a
href="https://github.com/golang/net/commit/d8870b0bf2f2426fc8d19a9332f652da5c25418f"><code>d8870b0</code></a>
http2: use synthetic time in TestIdleConnTimeout</li>
<li><a
href="https://github.com/golang/net/commit/d73acffdc9493532acb85777105bb4a351eea702"><code>d73acff</code></a>
http2: only set up deadline when Server.IdleTimeout is positive</li>
<li><a
href="https://github.com/golang/net/commit/89f602b7bbf237abe0467031a18b42fc742ced08"><code>89f602b</code></a>
http2: validate client/outgoing trailers</li>
<li>Additional commits viewable in <a
href="https://github.com/golang/net/compare/v0.21.0...v0.23.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/net&package-manager=go_modules&previous-version=0.21.0&new-version=0.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 0dce89222..5c8155281 100644
--- a/go.mod
+++ b/go.mod
@@ -124,7 +124,7 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
diff --git a/go.sum b/go.sum
index 1aea553f4..2ffa2c873 100644
--- a/go.sum
+++ b/go.sum
@@ -385,8 +385,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

From 46902e5766cc8efcc4f1abb38789f8179893c17d Mon Sep 17 00:00:00 2001
From: 0x1a8510f2 <admin@0x1a8510f2.space>
Date: Wed, 1 May 2024 01:38:36 +0100
Subject: [PATCH 36/46] Take advantage of changes in recent Go versions (#3361)

Given that #2714 wasn't merged but we are now at a minimum supported Go
version of 1.20 (soon to be 1.21), I wanted to carry over some of the
changes. Namely:
- Fix the log typo
- Simplify build constraints for unix
- Use stdlib atomic package

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [x] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [x] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `0x1a8510f2 <admin@0x1a8510f2.space>`

---------

Co-authored-by: devonh <devon.dmytro@gmail.com>
---
 clientapi/routing/routing.go                  |  2 +-
 cmd/dendrite-demo-pinecone/relay/retriever.go |  4 ++--
 federationapi/queue/destinationqueue.go       |  2 +-
 federationapi/queue/queue_test.go             |  6 +++---
 federationapi/statistics/statistics.go        |  6 +++---
 internal/sqlutil/writer_exclusive.go          |  3 +--
 internal/transactionrequest_test.go           | 14 +++++++-------
 setup/base/base.go                            |  2 +-
 setup/base/sanity_other.go                    |  4 ++--
 setup/base/sanity_unix.go                     |  4 ++--
 10 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index c96c6538c..60dad5433 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -255,7 +255,7 @@ func Setup(
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
 		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
 		if err != nil {
-			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+			logrus.WithError(err).Fatal("unable to get account for sending server notices")
 		}
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
diff --git a/cmd/dendrite-demo-pinecone/relay/retriever.go b/cmd/dendrite-demo-pinecone/relay/retriever.go
index 3c76ad600..9c918fb67 100644
--- a/cmd/dendrite-demo-pinecone/relay/retriever.go
+++ b/cmd/dendrite-demo-pinecone/relay/retriever.go
@@ -17,13 +17,13 @@ package relay
 import (
 	"context"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 )
 
 const (
@@ -54,7 +54,7 @@ func NewRelayServerRetriever(
 		federationAPI:       federationAPI,
 		relayAPI:            relayAPI,
 		relayServersQueried: make(map[spec.ServerName]bool),
-		running:             *atomic.NewBool(false),
+		running:             atomic.Bool{},
 		quit:                quit,
 	}
 }
diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go
index 87a6fe559..be43aaf1c 100644
--- a/federationapi/queue/destinationqueue.go
+++ b/federationapi/queue/destinationqueue.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/matrix-org/gomatrix"
@@ -26,7 +27,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go
index 6da863427..7d21a3bb3 100644
--- a/federationapi/queue/queue_test.go
+++ b/federationapi/queue/queue_test.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -26,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
-	"go.uber.org/atomic"
 	"gotest.tools/v3/poll"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -113,8 +113,8 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 	fc := &stubFederationClient{
 		shouldTxSucceed:      shouldTxSucceed,
 		shouldTxRelaySucceed: shouldTxRelaySucceed,
-		txCount:              *atomic.NewUint32(0),
-		txRelayCount:         *atomic.NewUint32(0),
+		txCount:              atomic.Uint32{},
+		txRelayCount:         atomic.Uint32{},
 	}
 
 	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline, false)
diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go
index e133fc9c9..750c57fd7 100644
--- a/federationapi/statistics/statistics.go
+++ b/federationapi/statistics/statistics.go
@@ -5,10 +5,10 @@ import (
 	"math"
 	"math/rand"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -169,7 +169,7 @@ func (s *ServerStatistics) Success(method SendMethod) {
 	// NOTE : Sending to the final destination vs. a relay server has
 	// slightly different semantics.
 	if method == SendDirect {
-		s.successCounter.Inc()
+		s.successCounter.Add(1)
 		if s.blacklisted.Load() && s.statistics.DB != nil {
 			if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil {
 				logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName)
@@ -195,7 +195,7 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 	// start a goroutine which will wait out the backoff and
 	// unset the backoffStarted flag when done.
 	if s.backoffStarted.CompareAndSwap(false, true) {
-		backoffCount := s.backoffCount.Inc()
+		backoffCount := s.backoffCount.Add(1)
 
 		if backoffCount >= s.statistics.FailuresUntilAssumedOffline {
 			s.assumedOffline.CompareAndSwap(false, true)
diff --git a/internal/sqlutil/writer_exclusive.go b/internal/sqlutil/writer_exclusive.go
index c6a271c1c..69eb8609c 100644
--- a/internal/sqlutil/writer_exclusive.go
+++ b/internal/sqlutil/writer_exclusive.go
@@ -3,8 +3,7 @@ package sqlutil
 import (
 	"database/sql"
 	"errors"
-
-	"go.uber.org/atomic"
+	"sync/atomic"
 )
 
 // ExclusiveWriter implements sqlutil.Writer.
diff --git a/internal/transactionrequest_test.go b/internal/transactionrequest_test.go
index ffc1cd89a..8dd100d11 100644
--- a/internal/transactionrequest_test.go
+++ b/internal/transactionrequest_test.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strconv"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -26,7 +27,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
-	"go.uber.org/atomic"
 	"gotest.tools/v3/poll"
 
 	"github.com/matrix-org/dendrite/federationapi/producers"
@@ -228,7 +228,7 @@ func TestProcessTransactionRequestEDUTyping(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 		room := msg.Header.Get(jetstream.RoomID)
@@ -294,7 +294,7 @@ func TestProcessTransactionRequestEDUToDevice(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -371,7 +371,7 @@ func TestProcessTransactionRequestEDUDeviceListUpdate(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -468,7 +468,7 @@ func TestProcessTransactionRequestEDUReceipt(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -512,7 +512,7 @@ func TestProcessTransactionRequestEDUSigningKeyUpdate(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
@@ -569,7 +569,7 @@ func TestProcessTransactionRequestEDUPresence(t *testing.T) {
 	ctx := process.NewProcessContext()
 	defer ctx.ShutdownDendrite()
 	txn, js, cfg := createTransactionWithEDU(ctx, edus)
-	received := atomic.NewBool(false)
+	received := atomic.Bool{}
 	onMessage := func(ctx context.Context, msgs []*nats.Msg) bool {
 		msg := msgs[0] // Guaranteed to exist if onMessage is called
 
diff --git a/setup/base/base.go b/setup/base/base.go
index 455337e59..82068aa92 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -28,13 +28,13 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"sync/atomic"
 	"syscall"
 	"time"
 
 	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"go.uber.org/atomic"
 
 	"github.com/gorilla/mux"
 	"github.com/kardianos/minwinsvc"
diff --git a/setup/base/sanity_other.go b/setup/base/sanity_other.go
index d35c2e872..38e2b941f 100644
--- a/setup/base/sanity_other.go
+++ b/setup/base/sanity_other.go
@@ -1,5 +1,5 @@
-//go:build !linux && !darwin && !netbsd && !freebsd && !openbsd && !solaris && !dragonfly && !aix
-// +build !linux,!darwin,!netbsd,!freebsd,!openbsd,!solaris,!dragonfly,!aix
+//go:build !unix
+// +build !unix
 
 package base
 
diff --git a/setup/base/sanity_unix.go b/setup/base/sanity_unix.go
index 0403df1a8..90e38a6db 100644
--- a/setup/base/sanity_unix.go
+++ b/setup/base/sanity_unix.go
@@ -1,5 +1,5 @@
-//go:build linux || darwin || netbsd || freebsd || openbsd || solaris || dragonfly || aix
-// +build linux darwin netbsd freebsd openbsd solaris dragonfly aix
+//go:build unix
+// +build unix
 
 package base
 

From 3e62b986d140de291fbccf5a22f637c2576b7584 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Fri, 14 Jun 2024 00:55:02 +0100
Subject: [PATCH 37/46] Blacklist sytests that require MSC3967 (#3384)

https://github.com/matrix-org/sytest/pull/1383 updates some sytests in
line with MSC3967. Dendrite does not support MSC3967, so these tests
fail.
---
 sytest-whitelist | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sytest-whitelist b/sytest-whitelist
index 492c756ba..35d700d0a 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -540,7 +540,6 @@ Will not back up to an old backup version
 Can create more than 10 backup versions
 Can delete backup
 Deleted & recreated backups are empty
-Can upload self-signing keys
 Fails to upload self-signing keys with no auth
 Fails to upload self-signing key without master key
 can fetch self-signing keys over federation
@@ -633,7 +632,6 @@ Trying to add push rule with no scope fails with 400
 Trying to add push rule with invalid scope fails with 400
 Forward extremities remain so even after the next events are populated as outliers
 uploading self-signing key notifies over federation
-uploading signed devices gets propagated over federation
 Device list doesn't change if remote server is down
 /context/ on joined room works
 /context/ on non world readable room does not work

From acfc36f697b35c29aa5d0497f5605c655f746614 Mon Sep 17 00:00:00 2001
From: Josh Simmons <josh@openchapters.tech>
Date: Thu, 25 Jul 2024 08:24:35 -0700
Subject: [PATCH 38/46] add DCO (#3395)

The Matrix.org Foundation is rolling out this DCO across all public
repositories in its namespace. Of note, this DCO does not require the
use of "real" or "legally identifiable" names.

### Pull Request Checklist

<!-- Please read
https://matrix-org.github.io/dendrite/development/contributing before
submitting your pull request -->

* [X] I have added Go unit tests or [Complement integration
tests](https://github.com/matrix-org/complement) for this PR _or_ I have
justified why this PR doesn't need tests
* [X] Pull request includes a [sign off below using a legally
identifiable
name](https://matrix-org.github.io/dendrite/development/contributing#sign-off)
_or_ I have already signed off privately

Signed-off-by: `Josh Simmons <josh@foundation.matrix.org>`

Co-authored-by: Josh Simmons <git@josh.tel>
---
 CONTRIBUTING.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 CONTRIBUTING.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000..028424406
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,59 @@
+# Contributing to Dendrite
+
+Thank you for taking the time to contribute to Matrix!
+
+This is the repository for Dendrite, a second-generation Matrix homeserver written in Go.
+
+## Sign off
+
+We ask that everybody who contributes to this project signs off their contributions, as explained below.
+
+We follow a simple 'inbound=outbound' model for contributions: the act of submitting an 'inbound' contribution means that the contributor agrees to license their contribution under the same terms as the project's overall 'outbound' license - in our case, this is Apache Software License v2 (see [LICENSE](./LICENSE)).
+
+In order to have a concrete record that your contribution is intentional and you agree to license it under the same terms as the project's license, we've adopted the same lightweight approach used by the [Linux Kernel](https://www.kernel.org/doc/html/latest/process/submitting-patches.html), [Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other projects: the [Developer Certificate of Origin](https://developercertificate.org/) (DCO). This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix:
+
+```
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+660 York Street, Suite 102,
+San Francisco, CA 94110 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+```
+
+If you agree to this for your contribution, then all that's needed is to include the line in your commit or pull request comment:
+
+```
+Signed-off-by: Your Name <your@email.example.org>
+```
+
+Git allows you to add this signoff automatically when using the `-s` flag to `git commit`, which uses the name and email set in your `user.name` and `user.email` git configs.
\ No newline at end of file

From 9566cc63594d6cc86dcd9bc6f26942158e65b4a9 Mon Sep 17 00:00:00 2001
From: Tulir Asokan <tulir@maunium.net>
Date: Sat, 27 Jul 2024 21:48:15 +0300
Subject: [PATCH 39/46] Fix parsing ?ts query param (#3396)

Signed-off-by: Tulir Asokan <tulir@maunium.net>
---
 clientapi/httputil/parse.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clientapi/httputil/parse.go b/clientapi/httputil/parse.go
index c83583345..a952d1778 100644
--- a/clientapi/httputil/parse.go
+++ b/clientapi/httputil/parse.go
@@ -35,5 +35,5 @@ func ParseTSParam(req *http.Request) (time.Time, error) {
 		return time.Time{}, fmt.Errorf("param 'ts' is no valid int (%s)", err.Error())
 	}
 
-	return time.Unix(ts/1000, 0), nil
+	return time.UnixMilli(ts), nil
 }

From b79bafb702cf47f6975c84b54b747b0e3b5a6a14 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 27 Jul 2024 20:48:51 +0200
Subject: [PATCH 40/46] Bump ws from 7.5.2 to 7.5.10 in /test/wasm (#3388)

Bumps [ws](https://github.com/websockets/ws) from 7.5.2 to 7.5.10.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/websockets/ws/releases">ws's
releases</a>.</em></p>
<blockquote>
<h2>7.5.10</h2>
<h1>Bug fixes</h1>
<ul>
<li>Backported e55e5106 to the 7.x release line (22c28763).</li>
</ul>
<h2>7.5.9</h2>
<h1>Bug fixes</h1>
<ul>
<li>Backported bc8bd34e to the 7.x release line (0435e6e1).</li>
</ul>
<h2>7.5.8</h2>
<h1>Bug fixes</h1>
<ul>
<li>Backported 0fdcc0af to the 7.x release line (2758ed35).</li>
<li>Backported d68ba9e1 to the 7.x release line (dc1781bc).</li>
</ul>
<h2>7.5.7</h2>
<h1>Bug fixes</h1>
<ul>
<li>Backported 6946f5fe to the 7.x release line (1f72e2e1).</li>
</ul>
<h2>7.5.6</h2>
<h1>Bug fixes</h1>
<ul>
<li>Backported b8186dd1 to the 7.x release line (73dec34b).</li>
<li>Backported ed2b8039 to the 7.x release line (22a26afb).</li>
</ul>
<h2>7.5.5</h2>
<h1>Bug fixes</h1>
<ul>
<li>Backported ec9377ca to the 7.x release line (0e274acd).</li>
</ul>
<h2>7.5.4</h2>
<h1>Bug fixes</h1>
<ul>
<li>Backported 6a72da3e to the 7.x release line (76087fbf).</li>
<li>Backported 869c9892 to the 7.x release line (27997933).</li>
</ul>
<h2>7.5.3</h2>
<h1>Bug fixes</h1>
<ul>
<li>The <code>WebSocketServer</code> constructor now throws an error if
more than one of the
<code>noServer</code>, <code>server</code>, and <code>port</code>
options are specefied (66e58d27).</li>
<li>Fixed a bug where a <code>'close'</code> event was emitted by a
<code>WebSocketServer</code> before
the internal HTTP/S server was actually closed (5a587304).</li>
<li>Fixed a bug that allowed WebSocket connections to be established
after
<code>WebSocketServer.prototype.close()</code> was called
(772236a1).</li>
</ul>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/websockets/ws/commit/d962d70649e393841ee1ed726a8f7ffbe90d0c06"><code>d962d70</code></a>
[dist] 7.5.10</li>
<li><a
href="https://github.com/websockets/ws/commit/22c28763234aa75a7e1b76f5c01c181260d7917f"><code>22c2876</code></a>
[security] Fix crash when the Upgrade header cannot be read (<a
href="https://redirect.github.com/websockets/ws/issues/2231">#2231</a>)</li>
<li><a
href="https://github.com/websockets/ws/commit/8a78f8770618cc5a1ade485a7445cb6d6f46e2f2"><code>8a78f87</code></a>
[dist] 7.5.9</li>
<li><a
href="https://github.com/websockets/ws/commit/0435e6e12b8d38992cf0651cb8605dde2294bd25"><code>0435e6e</code></a>
[security] Fix same host check for ws+unix: redirects</li>
<li><a
href="https://github.com/websockets/ws/commit/4271f07cfc95cf7e1936388fb69e22a3731fa260"><code>4271f07</code></a>
[dist] 7.5.8</li>
<li><a
href="https://github.com/websockets/ws/commit/dc1781bc319cb347878d11cf730947d0bef69a51"><code>dc1781b</code></a>
[security] Drop sensitive headers when following insecure redirects</li>
<li><a
href="https://github.com/websockets/ws/commit/2758ed355073105a60b8b836b25265b8cdcb3b42"><code>2758ed3</code></a>
[fix] Abort the handshake if the Upgrade header is invalid</li>
<li><a
href="https://github.com/websockets/ws/commit/a370613fab74b82990582fa7728e130c5e87ee4c"><code>a370613</code></a>
[dist] 7.5.7</li>
<li><a
href="https://github.com/websockets/ws/commit/1f72e2e14f4fbb20265c228a43bb64ab915d8046"><code>1f72e2e</code></a>
[security] Drop sensitive headers when following redirects (<a
href="https://redirect.github.com/websockets/ws/issues/2013">#2013</a>)</li>
<li><a
href="https://github.com/websockets/ws/commit/8ecd890800dfbe210298438ab9bb8dbe328f3e0b"><code>8ecd890</code></a>
[dist] 7.5.6</li>
<li>Additional commits viewable in <a
href="https://github.com/websockets/ws/compare/7.5.2...7.5.10">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ws&package-manager=npm_and_yarn&previous-version=7.5.2&new-version=7.5.10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 test/wasm/package-lock.json | 14 +++++++-------
 test/wasm/package.json      |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/wasm/package-lock.json b/test/wasm/package-lock.json
index f26d55ab7..c9ea15407 100644
--- a/test/wasm/package-lock.json
+++ b/test/wasm/package-lock.json
@@ -7,7 +7,7 @@
       "dependencies": {
         "isomorphic-ws": "^4.0.1",
         "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
-        "ws": "^7.5.2"
+        "ws": "^7.5.10"
       }
     },
     "node_modules/isomorphic-ws": {
@@ -25,9 +25,9 @@
       "license": "MIT"
     },
     "node_modules/ws": {
-      "version": "7.5.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
-      "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
+      "version": "7.5.10",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz",
+      "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==",
       "engines": {
         "node": ">=8.3.0"
       },
@@ -58,9 +58,9 @@
       "from": "sql.js@github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae"
     },
     "ws": {
-      "version": "7.5.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
-      "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
+      "version": "7.5.10",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz",
+      "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==",
       "requires": {}
     }
   }
diff --git a/test/wasm/package.json b/test/wasm/package.json
index b28c30b1d..6a2cc4363 100644
--- a/test/wasm/package.json
+++ b/test/wasm/package.json
@@ -2,6 +2,6 @@
   "dependencies": {
     "isomorphic-ws": "^4.0.1",
     "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
-    "ws": "^7.5.2"
+    "ws": "^7.5.10"
   }
 }

From 7d8516838d4904e1c78f811be440bbb2a644cbad Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 27 Jul 2024 20:49:18 +0200
Subject: [PATCH 41/46] Bump golang.org/x/image from 0.10.0 to 0.18.0 (#3390)

Bumps [golang.org/x/image](https://github.com/golang/image) from 0.10.0
to 0.18.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/image/commit/3bbf4a659e56fde394e7214ddd17673223aca672"><code>3bbf4a6</code></a>
tiff: Validate palette indices when parsing palette-color images</li>
<li><a
href="https://github.com/golang/image/commit/6c5fa462eb87ac98bad9b09ea3b041dd770fa611"><code>6c5fa46</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/image/commit/55c4ab6bd625a2e8433671ec9f9b6c46daddf2cf"><code>55c4ab6</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/image/commit/0057a939a541e6068d04ccf947c61a708378101d"><code>0057a93</code></a>
tiff: fix function name in comment</li>
<li><a
href="https://github.com/golang/image/commit/9e190ae4a3c5edc736fd99ba38be1c9d08ea5320"><code>9e190ae</code></a>
webp: disallow multiple VP8X chunks</li>
<li><a
href="https://github.com/golang/image/commit/445ab0e75e6df9b6a4d3d5437bda2a7cac74eb72"><code>445ab0e</code></a>
go.mod: update golang.org/x dependencies</li>
<li><a
href="https://github.com/golang/image/commit/240a51ac9f088c1c81cad2cf80a37b99c52abcde"><code>240a51a</code></a>
font/sfnt: support early version 0 OS/2 tables</li>
<li><a
href="https://github.com/golang/image/commit/c20bbc37136f3a0b463478dd8e699c51139af48c"><code>c20bbc3</code></a>
draw: simplify some calls to fmt.Fprintf</li>
<li><a
href="https://github.com/golang/image/commit/491771c681427e82b9843267b24b32cf56743e83"><code>491771c</code></a>
draw: merge draw_go117.go into draw.go</li>
<li><a
href="https://github.com/golang/image/commit/4aa0222fac539f4a04a4378b75b70ad6e1820ecb"><code>4aa0222</code></a>
go.mod: update go directive to 1.18</li>
<li>Additional commits viewable in <a
href="https://github.com/golang/image/compare/v0.10.0...v0.18.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/image&package-manager=go_modules&previous-version=0.10.0&new-version=0.18.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 16 ++++++++--------
 go.sum | 50 ++++++++++++++++----------------------------------
 2 files changed, 24 insertions(+), 42 deletions(-)

diff --git a/go.mod b/go.mod
index 5c8155281..af1cb68cc 100644
--- a/go.mod
+++ b/go.mod
@@ -42,12 +42,12 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
 	go.uber.org/atomic v1.11.0
-	golang.org/x/crypto v0.22.0
+	golang.org/x/crypto v0.23.0
 	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678
-	golang.org/x/image v0.10.0
+	golang.org/x/image v0.18.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
 	golang.org/x/sync v0.7.0
-	golang.org/x/term v0.19.0
+	golang.org/x/term v0.20.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
@@ -123,12 +123,12 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index 2ffa2c873..f57f3d4b0 100644
--- a/go.sum
+++ b/go.sum
@@ -339,7 +339,6 @@ github.com/yggdrasil-network/yggdrasil-go v0.4.6/go.mod h1:PBMoAOvQjA9geNEeGyMXA
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
@@ -350,9 +349,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -363,8 +361,8 @@ golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUU
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M=
-golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0=
+golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
+golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4=
 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY=
@@ -372,10 +370,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -383,16 +379,12 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -407,31 +399,23 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
@@ -445,10 +429,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

From 38549e649ed873257ca958beff272bf3f38288f8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 27 Jul 2024 21:33:38 +0200
Subject: [PATCH 42/46] Bump rexml from 3.2.5 to 3.3.2 in /docs (#3398)

Bumps [rexml](https://github.com/ruby/rexml) from 3.2.5 to 3.3.2.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/ruby/rexml/releases">rexml's
releases</a>.</em></p>
<blockquote>
<h2>REXML 3.3.2 - 2024-07-16</h2>
<h3>Improvements</h3>
<ul>
<li>
<p>Improved parse performance.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/160">GH-160</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Improved parse performance.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/169">GH-169</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/170">GH-170</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/171">GH-171</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/172">GH-172</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/173">GH-173</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/174">GH-174</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/175">GH-175</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/176">GH-176</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/177">GH-177</a></li>
<li>Patch by Watson.</li>
</ul>
</li>
<li>
<p>Added support for raising a parse exception when an XML has extra
content after the root element.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/161">GH-161</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Added support for raising a parse exception when an XML
declaration exists in wrong position.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/162">GH-162</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Removed needless a space after XML declaration in pretty print
mode.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/164">GH-164</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Stopped to emit <code>:text</code> event after the root element.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/167">GH-167</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
</ul>
<h3>Fixes</h3>
<ul>
<li>Fixed a bug that SAX2 parser doesn't expand predefined entities for
<code>characters</code> callback.
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/168">GH-168</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
</ul>
<h3>Thanks</h3>
<ul>
<li>
<p>NAITOH Jun</p>
</li>
<li>
<p>Watson</p>
</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/ruby/rexml/blob/master/NEWS.md">rexml's
changelog</a>.</em></p>
<blockquote>
<h2>3.3.2 - 2024-07-16 {#version-3-3-2}</h2>
<h3>Improvements</h3>
<ul>
<li>
<p>Improved parse performance.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/160">GH-160</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Improved parse performance.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/169">GH-169</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/170">GH-170</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/171">GH-171</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/172">GH-172</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/173">GH-173</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/174">GH-174</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/175">GH-175</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/176">GH-176</a></li>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/177">GH-177</a></li>
<li>Patch by Watson.</li>
</ul>
</li>
<li>
<p>Added support for raising a parse exception when an XML has extra
content after the root element.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/161">GH-161</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Added support for raising a parse exception when an XML
declaration exists in wrong position.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/162">GH-162</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Removed needless a space after XML declaration in pretty print
mode.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/164">GH-164</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
<li>
<p>Stopped to emit <code>:text</code> event after the root element.</p>
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/167">GH-167</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
</ul>
<h3>Fixes</h3>
<ul>
<li>Fixed a bug that SAX2 parser doesn't expand predefined entities for
<code>characters</code> callback.
<ul>
<li><a
href="https://redirect.github.com/ruby/rexml/issues/168">GH-168</a></li>
<li>Patch by NAITOH Jun.</li>
</ul>
</li>
</ul>
<h3>Thanks</h3>
<ul>
<li>
<p>NAITOH Jun</p>
</li>
<li>
<p>Watson</p>
</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/ruby/rexml/commit/2b285ac0804f2918de642f7ed4646dc6d645a7fc"><code>2b285ac</code></a>
Add 3.3.2 entry</li>
<li><a
href="https://github.com/ruby/rexml/commit/0e33d3adfb5069b20622e5ed9393d10b8cc17b40"><code>0e33d3a</code></a>
test: improve linear performance test names</li>
<li><a
href="https://github.com/ruby/rexml/commit/910e5a2b487cb5a30989884a39f9cad2cc499cfc"><code>910e5a2</code></a>
Fix performance issue caused by using repeated <code>&gt;</code>
characters inside `&lt;xml&gt;&lt;!...</li>
<li><a
href="https://github.com/ruby/rexml/commit/1f1e6e9b40bf339894e843dfd679c2fb1a5ddbf2"><code>1f1e6e9</code></a>
Fix ReDoS by using repeated space characters inside `&lt;!DOCTYPE name
[&lt;!ATTLIS...</li>
<li><a
href="https://github.com/ruby/rexml/commit/1cc1d9a74ede52f3d9ce774cafb11c57b3905165"><code>1cc1d9a</code></a>
Suppress have_root not initialized warnings on Ruby &lt; 3</li>
<li><a
href="https://github.com/ruby/rexml/commit/67efb5951ed09dbb575c375b130a1e469f437d1f"><code>67efb59</code></a>
Fix performance issue caused by using repeated <code>&gt;</code>
characters inside `&lt;!DOCTY...</li>
<li><a
href="https://github.com/ruby/rexml/commit/a79ac8b4b42a9efabe33a0be31bd82d33fd50347"><code>a79ac8b</code></a>
Fix performance issue caused by using repeated <code>&gt;</code>
characters inside `&lt;!DOCTY...</li>
<li><a
href="https://github.com/ruby/rexml/commit/c33ea498102be65082940e8b7d6d31cb2c6e6ee2"><code>c33ea49</code></a>
Fix performance issue caused by using repeated <code>&gt;</code>
characters after ` &lt;!DOCTY...</li>
<li><a
href="https://github.com/ruby/rexml/commit/9f1415a2616c77cad44a176eee90e8457b4774b6"><code>9f1415a</code></a>
Fix performance issue caused by using repeated <code>&gt;</code>
characters inside `CDATA [...</li>
<li><a
href="https://github.com/ruby/rexml/commit/c1b64c174ec2e8ca2174c51332670e3be30c865f"><code>c1b64c1</code></a>
Fix performance issue caused by using repeated <code>&gt;</code>
characters inside comments...</li>
<li>Additional commits viewable in <a
href="https://github.com/ruby/rexml/compare/v3.2.5...v3.3.2">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=rexml&package-manager=bundler&previous-version=3.2.5&new-version=3.3.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 docs/Gemfile.lock | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
index 791d0f857..bf123b165 100644
--- a/docs/Gemfile.lock
+++ b/docs/Gemfile.lock
@@ -245,7 +245,8 @@ GEM
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rexml (3.2.5)
+    rexml (3.3.2)
+      strscan
     rouge (3.26.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
@@ -260,6 +261,7 @@ GEM
       faraday (> 0.8, < 2.0)
     simpleidn (0.2.1)
       unf (~> 0.1.4)
+    strscan (3.1.0)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
     thread_safe (0.3.6)

From a2e56dccb0a6c525e1634ec35be5c8ff703cb70d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Sat, 27 Jul 2024 22:28:52 +0200
Subject: [PATCH 43/46] Don't bother hitting the DB if we got everything from
 cache (#3376)

Yet another finding where we hit the DB with a useless query. [skip ci]
---
 federationapi/storage/cache/keydb.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/federationapi/storage/cache/keydb.go b/federationapi/storage/cache/keydb.go
index b53695ca4..d63c889d5 100644
--- a/federationapi/storage/cache/keydb.go
+++ b/federationapi/storage/cache/keydb.go
@@ -46,6 +46,10 @@ func (d *KeyDatabase) FetchKeys(
 			delete(requests, req)
 		}
 	}
+	// Don't bother hitting the DB if we got everything from cache.
+	if len(requests) == 0 {
+		return results, nil
+	}
 	fromDB, err := d.inner.FetchKeys(ctx, requests)
 	if err != nil {
 		return results, err

From 795c4a9453e6775714ae73099e2a64df1c41743b Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Sat, 27 Jul 2024 22:29:19 +0200
Subject: [PATCH 44/46] Fix media DB possibly leaking connections (#3372)

Grafana Pyroscope unveiled that we are hitting
https://github.com/golang/go/blob/ad10fbd3c4ec7413775028213f4d5089b18926f7/src/database/sql/sql.go#L2739-L2742
for media DB queries.

Making the methods pointer receivers fixes this.

(Also some minor `error` cosmetic updates)
---
 mediaapi/storage/shared/mediaapi.go | 31 ++++++++++++-----------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/mediaapi/storage/shared/mediaapi.go b/mediaapi/storage/shared/mediaapi.go
index 867405fb3..bdd7f317b 100644
--- a/mediaapi/storage/shared/mediaapi.go
+++ b/mediaapi/storage/shared/mediaapi.go
@@ -17,6 +17,7 @@ package shared
 import (
 	"context"
 	"database/sql"
+	"errors"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
@@ -33,7 +34,7 @@ type Database struct {
 
 // StoreMediaMetadata inserts the metadata about the uploaded media into the database.
 // Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error {
+func (d *Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.MediaRepository.InsertMedia(ctx, txn, mediaMetadata)
 	})
@@ -42,9 +43,9 @@ func (d Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.M
 // GetMediaMetadata returns metadata about media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this media.
-func (d Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) {
+func (d *Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) {
 	mediaMetadata, err := d.MediaRepository.SelectMedia(ctx, nil, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
+	if errors.Is(err, sql.ErrNoRows) {
 		return nil, nil
 	}
 	return mediaMetadata, err
@@ -53,9 +54,9 @@ func (d Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, m
 // GetMediaMetadataByHash returns metadata about media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this media.
-func (d Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) {
+func (d *Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) {
 	mediaMetadata, err := d.MediaRepository.SelectMediaByHash(ctx, nil, mediaHash, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
+	if errors.Is(err, sql.ErrNoRows) {
 		return nil, nil
 	}
 	return mediaMetadata, err
@@ -63,7 +64,7 @@ func (d Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Ba
 
 // StoreThumbnail inserts the metadata about the thumbnail into the database.
 // Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error {
+func (d *Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.Thumbnails.InsertThumbnail(ctx, txn, thumbnailMetadata)
 	})
@@ -72,13 +73,10 @@ func (d Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.T
 // GetThumbnail returns metadata about a specific thumbnail.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this thumbnail.
-func (d Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) {
+func (d *Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) {
 	metadata, err := d.Thumbnails.SelectThumbnail(ctx, nil, mediaID, mediaOrigin, width, height, resizeMethod)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil
-		}
-		return nil, err
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
 	}
 	return metadata, err
 }
@@ -86,13 +84,10 @@ func (d Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, media
 // GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there are no thumbnails associated with this media.
-func (d Database) GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) ([]*types.ThumbnailMetadata, error) {
+func (d *Database) GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) ([]*types.ThumbnailMetadata, error) {
 	metadatas, err := d.Thumbnails.SelectThumbnails(ctx, nil, mediaID, mediaOrigin)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil
-		}
-		return nil, err
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
 	}
 	return metadatas, err
 }

From affb6977e43ad5051761d0de650370f421f751b5 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Sat, 27 Jul 2024 22:29:34 +0200
Subject: [PATCH 45/46] Fix nil pointer derefernce issues (#3379)

Discovered while running
https://gitlab.futo.org/load-testing/matrix-goose.

Dendrite locks up and runs into `context cancelled`, so the error is not
`sql.ErrNoRows` nor "default" (and definitely shouldn't return that the
account exists in this case)
---
 roomserver/storage/postgres/user_room_keys_table.go | 3 +++
 roomserver/storage/sqlite3/user_room_keys_table.go  | 3 +++
 userapi/internal/user_api.go                        | 3 ++-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go
index 57e8f213b..f8befc46b 100644
--- a/roomserver/storage/postgres/user_room_keys_table.go
+++ b/roomserver/storage/postgres/user_room_keys_table.go
@@ -162,6 +162,9 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context,
 	if errors.Is(err, sql.ErrNoRows) {
 		return nil, nil
 	}
+	if err != nil {
+		return nil, err
+	}
 	defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows")
 
 	resultMap := make(map[types.RoomNID]ed25519.PublicKey)
diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go
index 13906f771..ef3b8fe20 100644
--- a/roomserver/storage/sqlite3/user_room_keys_table.go
+++ b/roomserver/storage/sqlite3/user_room_keys_table.go
@@ -177,6 +177,9 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context,
 	if errors.Is(err, sql.ErrNoRows) {
 		return nil, nil
 	}
+	if err != nil {
+		return nil, err
+	}
 	defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows")
 
 	resultMap := make(map[types.RoomNID]ed25519.PublicKey)
diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go
index a126dc871..fd73bf62f 100644
--- a/userapi/internal/user_api.go
+++ b/userapi/internal/user_api.go
@@ -939,11 +939,12 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q
 		return nil
 	case bcrypt.ErrHashTooShort: // user exists, but probably a passwordless account
 		return nil
-	default:
+	case nil:
 		res.Exists = true
 		res.Account = acc
 		return nil
 	}
+	return err
 }
 
 func (a *UserInternalAPI) SetDisplayName(ctx context.Context, localpart string, serverName spec.ServerName, displayName string) (*authtypes.Profile, bool, error) {

From 989795973103c463a33f053663c6a8616177186c Mon Sep 17 00:00:00 2001
From: Alex <d.lexand@gmail.com>
Date: Sat, 27 Jul 2024 22:30:17 +0200
Subject: [PATCH 46/46] Fix: Edited messages appear twice in fulltext search
 (#3363)

As stated in https://github.com/matrix-org/dendrite/issues/3358 the
search response contains both original and edited message.
This PR fixes it by removing of the original message from the fulltext
index after indexing the edit message event.
I also made some cosmetic changes/fixes i found in the code

Signed-off-by: `Alexander Dubovikov <d.lexand@gmail.com>`
---
 clientapi/routing/admin.go          |  4 +-
 clientapi/routing/pushrules.go      |  2 +-
 clientapi/routing/sendevent_test.go |  2 +-
 syncapi/consumers/roomserver.go     | 18 ++++++
 syncapi/syncapi_test.go             | 92 +++++++++++++++++++++++++++++
 test/testrig/base.go                |  1 +
 6 files changed, 115 insertions(+), 4 deletions(-)

diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index 68e62b08f..73a0afc32 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -328,7 +328,7 @@ func AdminPurgeRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI)
 	}
 }
 
-func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI api.ClientUserAPI) util.JSONResponse {
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	if req.Body == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -423,7 +423,7 @@ func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *api.Device,
 	}
 }
 
-func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.ClientKeyAPI) util.JSONResponse {
+func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI userapi.ClientKeyAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
diff --git a/clientapi/routing/pushrules.go b/clientapi/routing/pushrules.go
index 74873d5c9..43c034f9d 100644
--- a/clientapi/routing/pushrules.go
+++ b/clientapi/routing/pushrules.go
@@ -70,7 +70,7 @@ func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi
 	}
 	rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind))
 	// Even if rulesPtr is not nil, there may not be any rules for this kind
-	if rulesPtr == nil || (rulesPtr != nil && len(*rulesPtr) == 0) {
+	if rulesPtr == nil || len(*rulesPtr) == 0 {
 		return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed")
 	}
 	return util.JSONResponse{
diff --git a/clientapi/routing/sendevent_test.go b/clientapi/routing/sendevent_test.go
index 9cdd75358..00d19154a 100644
--- a/clientapi/routing/sendevent_test.go
+++ b/clientapi/routing/sendevent_test.go
@@ -265,7 +265,7 @@ func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([
 	for i, eventJSON := range eventsJSON {
 		pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false)
 		if evErr != nil {
-			return nil, fmt.Errorf("failed to make event: %s", err.Error())
+			return nil, fmt.Errorf("failed to make event: %s", evErr.Error())
 		}
 		ev := types.HeaderedEvent{PDU: pdu}
 		events[i] = &ev
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 81c532f19..abf888829 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -601,9 +601,11 @@ func (s *OutputRoomEventConsumer) writeFTS(ev *rstypes.HeaderedEvent, pduPositio
 	}
 	e.SetContentType(ev.Type())
 
+	var relatesTo gjson.Result
 	switch ev.Type() {
 	case "m.room.message":
 		e.Content = gjson.GetBytes(ev.Content(), "body").String()
+		relatesTo = gjson.GetBytes(ev.Content(), "m\\.relates_to")
 	case spec.MRoomName:
 		e.Content = gjson.GetBytes(ev.Content(), "name").String()
 	case spec.MRoomTopic:
@@ -622,6 +624,22 @@ func (s *OutputRoomEventConsumer) writeFTS(ev *rstypes.HeaderedEvent, pduPositio
 		if err := s.fts.Index(e); err != nil {
 			return err
 		}
+		// If the event is an edited message we remove the original event from the index
+		// to avoid duplicates in the search results.
+		if relatesTo.Exists() {
+			relatedData := relatesTo.Map()
+			if _, ok := relatedData["rel_type"]; ok && relatedData["rel_type"].Str == "m.replace" {
+				// We remove the original event from the index
+				if srcEventID, ok := relatedData["event_id"]; ok {
+					if err := s.fts.Delete(srcEventID.Str); err != nil {
+						log.WithFields(log.Fields{
+							"event_id": ev.EventID(),
+							"src_id":   srcEventID.Str,
+						}).WithError(err).Error("Failed to delete edited message from the fulltext index")
+					}
+				}
+			}
+		}
 	}
 	return nil
 }
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 0392f209a..d360e10d9 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -4,12 +4,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -17,6 +19,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
 	"github.com/tidwall/gjson"
 
 	rstypes "github.com/matrix-org/dendrite/roomserver/types"
@@ -1324,6 +1327,95 @@ func TestUpdateRelations(t *testing.T) {
 	})
 }
 
+func TestRemoveEditedEventFromSearchIndex(t *testing.T) {
+	user := test.NewUser(t)
+	alice := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      user.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "Alice",
+		AccountType: userapi.AccountTypeUser,
+	}
+
+	routers := httputil.NewRouters()
+
+	cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypeSQLite)
+	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	defer close()
+
+	// Use an actual roomserver for this
+	natsInstance := jetstream.NATSInstance{}
+	jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+	rsAPI.SetFederationAPI(nil, nil)
+
+	room := test.NewRoom(t, user)
+	AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics)
+
+	if err := api.SendEvents(processCtx.Context(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+		t.Fatalf("failed to send events: %v", err)
+	}
+
+	ev1 := room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{"body": "first"})
+	ev2 := room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{
+		"body": " * first",
+		"m.new_content": map[string]interface{}{
+			"body":    "first",
+			"msgtype": "m.text",
+		},
+		"m.relates_to": map[string]interface{}{
+			"event_id": ev1.EventID(),
+			"rel_type": "m.replace",
+		},
+	})
+	events := []*rstypes.HeaderedEvent{ev1, ev2}
+
+	for _, e := range events {
+		roomEvents := append([]*rstypes.HeaderedEvent{}, e)
+		if err := api.SendEvents(processCtx.Context(), rsAPI, api.KindNew, roomEvents, "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool {
+			// wait for the last sent eventID to come down sync
+			path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, e.EventID())
+
+			return gjson.Get(syncBody, path).Exists()
+		})
+
+		// We search that event is the only one nad is the exact event we sent
+		searchResult := searchRequest(t, routers.Client, alice.AccessToken, "first", []string{room.ID})
+		results := gjson.GetBytes(searchResult, fmt.Sprintf(`search_categories.room_events.groups.room_id.%s.results`, room.ID))
+		assert.True(t, results.Exists(), "Should be a search response")
+		assert.Equal(t, 1, len(results.Array()), "Should be exactly one result")
+		assert.Equal(t, e.EventID(), results.Array()[0].String(), "Should be only found exact event")
+	}
+}
+
+func searchRequest(t *testing.T, router *mux.Router, accessToken, searchTerm string, roomList []string) []byte {
+	t.Helper()
+	w := httptest.NewRecorder()
+	rq := test.NewRequest(t, "POST", "/_matrix/client/v3/search", test.WithQueryParams(map[string]string{
+		"access_token": accessToken,
+	}), test.WithJSONBody(t, map[string]interface{}{
+		"search_categories": map[string]interface{}{
+			"room_events": map[string]interface{}{
+				"filters":     roomList,
+				"search_term": searchTerm,
+			},
+		},
+	}))
+
+	router.ServeHTTP(w, rq)
+	assert.Equal(t, 200, w.Code)
+	defer w.Result().Body.Close()
+	body, err := io.ReadAll(w.Result().Body)
+	assert.NoError(t, err)
+	return body
+}
 func syncUntil(t *testing.T,
 	routers httputil.Routers, accessToken string,
 	skip bool,
diff --git a/test/testrig/base.go b/test/testrig/base.go
index 953704595..a21cfe802 100644
--- a/test/testrig/base.go
+++ b/test/testrig/base.go
@@ -71,6 +71,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process.
 			SingleDatabase: false,
 		})
 		cfg.Global.ServerName = "test"
+		cfg.SyncAPI.Fulltext.Enabled = true
 		cfg.SyncAPI.Fulltext.InMemory = true
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(