From 69f2ff7c82abe0731a05febde88098f4cd34ab8d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 11 Apr 2022 09:05:23 +0200
Subject: [PATCH 001/391] Correctly use provided filters (#2339)

* Apply filters correctly

* Fix issues; Use prepareWithFilters

* Update gmsl & tests

* go.mod..

* PR comments
---
 go.mod                                        |  3 +-
 go.sum                                        |  7 +--
 syncapi/routing/context.go                    |  4 +-
 .../postgres/current_room_state_table.go      |  5 +-
 syncapi/storage/postgres/filtering.go         | 32 ++++++++++--
 .../postgres/output_room_events_table.go      | 26 ++++++----
 syncapi/storage/sqlite3/account_data_table.go | 39 +++++----------
 syncapi/storage/sqlite3/filtering.go          | 50 ++++++++++++-------
 syncapi/streams/stream_pdu.go                 |  6 ++-
 sytest-whitelist                              |  4 +-
 10 files changed, 109 insertions(+), 67 deletions(-)

diff --git a/go.mod b/go.mod
index f4ac8d123..070f2b5b9 100644
--- a/go.mod
+++ b/go.mod
@@ -38,7 +38,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
@@ -64,6 +64,7 @@ require (
 	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
 	golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2
 	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3
+	golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
diff --git a/go.sum b/go.sum
index 063365168..464f774be 100644
--- a/go.sum
+++ b/go.sum
@@ -1112,8 +1112,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5 h1:Fkennny7+Z/5pygrhjFMZbz1j++P2hhhLoT7NO3p8DQ=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220405134050-301e340659d5/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f h1:MZrl4TgTnlaOn2Cu9gJCoJ3oyW5mT4/3QIZGgZXzKl4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -1977,8 +1977,9 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 h1:QyVthZKMsyaQwBTJE04jdNN0Pp5Fn9Qga0mrgxyERQM=
+golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 2412bc2ae..aaa0c61bf 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -60,7 +60,9 @@ func Context(
 			Headers: nil,
 		}
 	}
-	filter.Rooms = append(filter.Rooms, roomID)
+	if filter.Rooms != nil {
+		*filter.Rooms = append(*filter.Rooms, roomID)
+	}
 
 	ctx := req.Context()
 	membershipRes := roomserver.QueryMembershipForUserResponse{}
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 69e6e30ec..fe68788d1 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -233,9 +233,10 @@ func (s *currentRoomStateStatements) SelectCurrentState(
 	excludeEventIDs []string,
 ) ([]*gomatrixserverlib.HeaderedEvent, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt)
+	senders, notSenders := getSendersStateFilterFilter(stateFilter)
 	rows, err := stmt.QueryContext(ctx, roomID,
-		pq.StringArray(stateFilter.Senders),
-		pq.StringArray(stateFilter.NotSenders),
+		pq.StringArray(senders),
+		pq.StringArray(notSenders),
 		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)),
 		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)),
 		stateFilter.ContainsURL,
diff --git a/syncapi/storage/postgres/filtering.go b/syncapi/storage/postgres/filtering.go
index dcc421362..a2ca42156 100644
--- a/syncapi/storage/postgres/filtering.go
+++ b/syncapi/storage/postgres/filtering.go
@@ -16,21 +16,45 @@ package postgres
 
 import (
 	"strings"
+
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // filterConvertWildcardToSQL converts wildcards as defined in
 // https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter
 // to SQL wildcards that can be used with LIKE()
-func filterConvertTypeWildcardToSQL(values []string) []string {
+func filterConvertTypeWildcardToSQL(values *[]string) []string {
 	if values == nil {
 		// Return nil instead of []string{} so IS NULL can work correctly when
 		// the return value is passed into SQL queries
 		return nil
 	}
 
-	ret := make([]string, len(values))
-	for i := range values {
-		ret[i] = strings.Replace(values[i], "*", "%", -1)
+	v := *values
+	ret := make([]string, len(v))
+	for i := range v {
+		ret[i] = strings.Replace(v[i], "*", "%", -1)
 	}
 	return ret
 }
+
+// TODO: Replace when Dendrite uses Go 1.18
+func getSendersRoomEventFilter(filter *gomatrixserverlib.RoomEventFilter) (senders []string, notSenders []string) {
+	if filter.Senders != nil {
+		senders = *filter.Senders
+	}
+	if filter.NotSenders != nil {
+		notSenders = *filter.NotSenders
+	}
+	return senders, notSenders
+}
+
+func getSendersStateFilterFilter(filter *gomatrixserverlib.StateFilter) (senders []string, notSenders []string) {
+	if filter.Senders != nil {
+		senders = *filter.Senders
+	}
+	if filter.NotSenders != nil {
+		notSenders = *filter.NotSenders
+	}
+	return senders, notSenders
+}
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index a30e220ba..269cd4494 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -204,11 +204,11 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 	stateFilter *gomatrixserverlib.StateFilter, roomIDs []string,
 ) (map[string]map[string]bool, map[string]types.StreamEvent, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt)
-
+	senders, notSenders := getSendersStateFilterFilter(stateFilter)
 	rows, err := stmt.QueryContext(
 		ctx, r.Low(), r.High(), pq.StringArray(roomIDs),
-		pq.StringArray(stateFilter.Senders),
-		pq.StringArray(stateFilter.NotSenders),
+		pq.StringArray(senders),
+		pq.StringArray(notSenders),
 		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)),
 		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)),
 		stateFilter.ContainsURL,
@@ -353,10 +353,11 @@ func (s *outputRoomEventsStatements) SelectRecentEvents(
 	} else {
 		stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt)
 	}
+	senders, notSenders := getSendersRoomEventFilter(eventFilter)
 	rows, err := stmt.QueryContext(
 		ctx, roomID, r.Low(), r.High(),
-		pq.StringArray(eventFilter.Senders),
-		pq.StringArray(eventFilter.NotSenders),
+		pq.StringArray(senders),
+		pq.StringArray(notSenders),
 		pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.Types)),
 		pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.NotTypes)),
 		eventFilter.Limit+1,
@@ -398,11 +399,12 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents(
 	ctx context.Context, txn *sql.Tx,
 	roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter,
 ) ([]types.StreamEvent, error) {
+	senders, notSenders := getSendersRoomEventFilter(eventFilter)
 	stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt)
 	rows, err := stmt.QueryContext(
 		ctx, roomID, r.Low(), r.High(),
-		pq.StringArray(eventFilter.Senders),
-		pq.StringArray(eventFilter.NotSenders),
+		pq.StringArray(senders),
+		pq.StringArray(notSenders),
 		pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.Types)),
 		pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.NotTypes)),
 		eventFilter.Limit,
@@ -480,10 +482,11 @@ func (s *outputRoomEventsStatements) SelectContextEvent(ctx context.Context, txn
 func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
 	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
 ) (evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	senders, notSenders := getSendersRoomEventFilter(filter)
 	rows, err := sqlutil.TxStmt(txn, s.selectContextBeforeEventStmt).QueryContext(
 		ctx, roomID, id, filter.Limit,
-		pq.StringArray(filter.Senders),
-		pq.StringArray(filter.NotSenders),
+		pq.StringArray(senders),
+		pq.StringArray(notSenders),
 		pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)),
 		pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)),
 	)
@@ -512,10 +515,11 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
 func (s *outputRoomEventsStatements) SelectContextAfterEvent(
 	ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter,
 ) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) {
+	senders, notSenders := getSendersRoomEventFilter(filter)
 	rows, err := sqlutil.TxStmt(txn, s.selectContextAfterEventStmt).QueryContext(
 		ctx, roomID, id, filter.Limit,
-		pq.StringArray(filter.Senders),
-		pq.StringArray(filter.NotSenders),
+		pq.StringArray(senders),
+		pq.StringArray(notSenders),
 		pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)),
 		pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)),
 	)
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index 5b2287e6d..b0aeb70f2 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -41,10 +41,10 @@ const insertAccountDataSQL = "" +
 	" ON CONFLICT (user_id, room_id, type) DO UPDATE" +
 	" SET id = $5"
 
+// further parameters are added by prepareWithFilters
 const selectAccountDataInRangeSQL = "" +
 	"SELECT room_id, type FROM syncapi_account_data_type" +
-	" WHERE user_id = $1 AND id > $2 AND id <= $3" +
-	" ORDER BY id ASC"
+	" WHERE user_id = $1 AND id > $2 AND id <= $3"
 
 const selectMaxAccountDataIDSQL = "" +
 	"SELECT MAX(id) FROM syncapi_account_data_type"
@@ -94,18 +94,25 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	ctx context.Context,
 	userID string,
 	r types.Range,
-	accountDataFilterPart *gomatrixserverlib.EventFilter,
+	filter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, err error) {
 	data = make(map[string][]string)
+	stmt, params, err := prepareWithFilters(
+		s.db, nil, selectAccountDataInRangeSQL,
+		[]interface{}{
+			userID, r.Low(), r.High(),
+		},
+		filter.Senders, filter.NotSenders,
+		filter.Types, filter.NotTypes,
+		[]string{}, filter.Limit, FilterOrderAsc,
+	)
 
-	rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High())
+	rows, err := stmt.QueryContext(ctx, params...)
 	if err != nil {
 		return
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed")
 
-	var entries int
-
 	for rows.Next() {
 		var dataType string
 		var roomID string
@@ -114,31 +121,11 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 			return
 		}
 
-		// check if we should add this by looking at the filter.
-		// It would be nice if we could do this in SQL-land, but the mix of variadic
-		// and positional parameters makes the query annoyingly hard to do, it's easier
-		// and clearer to do it in Go-land. If there are no filters for [not]types then
-		// this gets skipped.
-		for _, includeType := range accountDataFilterPart.Types {
-			if includeType != dataType { // TODO: wildcard support
-				continue
-			}
-		}
-		for _, excludeType := range accountDataFilterPart.NotTypes {
-			if excludeType == dataType { // TODO: wildcard support
-				continue
-			}
-		}
-
 		if len(data[roomID]) > 0 {
 			data[roomID] = append(data[roomID], dataType)
 		} else {
 			data[roomID] = []string{dataType}
 		}
-		entries++
-		if entries >= accountDataFilterPart.Limit {
-			break
-		}
 	}
 
 	return data, nil
diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go
index 11f3e647b..54b12ddf8 100644
--- a/syncapi/storage/sqlite3/filtering.go
+++ b/syncapi/storage/sqlite3/filtering.go
@@ -25,32 +25,48 @@ const (
 // parts.
 func prepareWithFilters(
 	db *sql.DB, txn *sql.Tx, query string, params []interface{},
-	senders, notsenders, types, nottypes []string, excludeEventIDs []string,
+	senders, notsenders, types, nottypes *[]string, excludeEventIDs []string,
 	limit int, order FilterOrder,
 ) (*sql.Stmt, []interface{}, error) {
 	offset := len(params)
-	if count := len(senders); count > 0 {
-		query += " AND sender IN " + sqlutil.QueryVariadicOffset(count, offset)
-		for _, v := range senders {
-			params, offset = append(params, v), offset+1
+	if senders != nil {
+		if count := len(*senders); count > 0 {
+			query += " AND sender IN " + sqlutil.QueryVariadicOffset(count, offset)
+			for _, v := range *senders {
+				params, offset = append(params, v), offset+1
+			}
+		} else {
+			query += ` AND sender = ""`
 		}
 	}
-	if count := len(notsenders); count > 0 {
-		query += " AND sender NOT IN " + sqlutil.QueryVariadicOffset(count, offset)
-		for _, v := range notsenders {
-			params, offset = append(params, v), offset+1
+	if notsenders != nil {
+		if count := len(*notsenders); count > 0 {
+			query += " AND sender NOT IN " + sqlutil.QueryVariadicOffset(count, offset)
+			for _, v := range *notsenders {
+				params, offset = append(params, v), offset+1
+			}
+		} else {
+			query += ` AND sender NOT = ""`
 		}
 	}
-	if count := len(types); count > 0 {
-		query += " AND type IN " + sqlutil.QueryVariadicOffset(count, offset)
-		for _, v := range types {
-			params, offset = append(params, v), offset+1
+	if types != nil {
+		if count := len(*types); count > 0 {
+			query += " AND type IN " + sqlutil.QueryVariadicOffset(count, offset)
+			for _, v := range *types {
+				params, offset = append(params, v), offset+1
+			}
+		} else {
+			query += ` AND type = ""`
 		}
 	}
-	if count := len(nottypes); count > 0 {
-		query += " AND type NOT IN " + sqlutil.QueryVariadicOffset(count, offset)
-		for _, v := range nottypes {
-			params, offset = append(params, v), offset+1
+	if nottypes != nil {
+		if count := len(*nottypes); count > 0 {
+			query += " AND type NOT IN " + sqlutil.QueryVariadicOffset(count, offset)
+			for _, v := range *nottypes {
+				params, offset = append(params, v), offset+1
+			}
+		} else {
+			query += ` AND type NOT = ""`
 		}
 	}
 	if count := len(excludeEventIDs); count > 0 {
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index ab200e007..bcaf6ca31 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -423,8 +423,12 @@ func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, req *ty
 		return err
 	}
 	req.IgnoredUsers = *ignores
+	userList := make([]string, 0, len(ignores.List))
 	for userID := range ignores.List {
-		eventFilter.NotSenders = append(eventFilter.NotSenders, userID)
+		userList = append(userList, userID)
+	}
+	if len(userList) > 0 {
+		eventFilter.NotSenders = &userList
 	}
 	return nil
 }
diff --git a/sytest-whitelist b/sytest-whitelist
index dc67c9935..a7aea05ed 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -696,4 +696,6 @@ Room state after a rejected message event is the same as before
 Room state after a rejected state event is the same as before
 Ignore user in existing room
 Ignore invite in full sync
-Ignore invite in incremental sync
\ No newline at end of file
+Ignore invite in incremental sync
+A filtered timeline reaches its limit
+A change to displayname should not result in a full state sync
\ No newline at end of file

From ea92f80c128bcff0fd4e02df40b272f7d90a97da Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Mon, 11 Apr 2022 10:23:01 +0100
Subject: [PATCH 002/391] Add database namespacing for unit tests (#2340)

* Add database namespacing for unit tests

Background: Running `go test ./...` will run tests in different packages concurrently.
This can be stopped or limited by using `-p 1` (no concurrency). We want concurrency,
but this causes problems when running Postgres DBs in CI. The problem is that, in CI,
we have 1x postgres server exposing 1x postgres DB, which we wipe clean at the end of
each test via `defer close()`. When tests run concurrently, calls to `close()` will
delete data/tables which other tests are currently using, causing havoc.

Fix this by:
 - Creating a database per package.
 - Namespacing the database name by a hash of the current working directory (the directory containing those `_test.go` files)

This is exactly what SQLite does, quite unintentionally, via the use of `file:dendrite_test.db`,
which dumps the file into the current working directory which is the package running the tests,
hence deleting the file is safe when running concurrently.

* Linting

* Don't create the database in a txn

* dupe db is not an error
---
 .github/workflows/dendrite.yml |  2 +-
 test/db.go                     | 58 +++++++++++++++++++++++++++++-----
 2 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 8221bff96..4f337a866 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -111,7 +111,7 @@ jobs:
           key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go${{ matrix.go }}-test-
-      - run: go test -p 1 ./...
+      - run: go test ./...
         env:
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
diff --git a/test/db.go b/test/db.go
index 674fdf5c3..6412feaa6 100644
--- a/test/db.go
+++ b/test/db.go
@@ -15,12 +15,16 @@
 package test
 
 import (
+	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
 	"fmt"
 	"os"
 	"os/exec"
 	"os/user"
 	"testing"
+
+	"github.com/lib/pq"
 )
 
 type DBType int
@@ -30,7 +34,7 @@ var DBTypePostgres DBType = 2
 
 var Quiet = false
 
-func createLocalDB(dbName string) string {
+func createLocalDB(dbName string) {
 	if !Quiet {
 		fmt.Println("Note: tests require a postgres install accessible to the current user")
 	}
@@ -43,7 +47,29 @@ func createLocalDB(dbName string) string {
 	if err != nil && !Quiet {
 		fmt.Println("createLocalDB returned error:", err)
 	}
-	return dbName
+}
+
+func createRemoteDB(t *testing.T, dbName, user, connStr string) {
+	db, err := sql.Open("postgres", connStr+" dbname=postgres")
+	if err != nil {
+		t.Fatalf("failed to open postgres conn with connstr=%s : %s", connStr, err)
+	}
+	_, err = db.Exec(fmt.Sprintf(`CREATE DATABASE %s;`, dbName))
+	if err != nil {
+		pqErr, ok := err.(*pq.Error)
+		if !ok {
+			t.Fatalf("failed to CREATE DATABASE: %s", err)
+		}
+		// we ignore duplicate database error as we expect this
+		if pqErr.Code != "42P04" {
+			t.Fatalf("failed to CREATE DATABASE with code=%s msg=%s", pqErr.Code, pqErr.Message)
+		}
+	}
+	_, err = db.Exec(fmt.Sprintf(`GRANT ALL PRIVILEGES ON DATABASE %s TO %s`, dbName, user))
+	if err != nil {
+		t.Fatalf("failed to GRANT: %s", err)
+	}
+	_ = db.Close()
 }
 
 func currentUser() string {
@@ -64,6 +90,7 @@ func currentUser() string {
 // TODO: namespace for concurrent package tests
 func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, close func()) {
 	if dbType == DBTypeSQLite {
+		// this will be made in the current working directory which namespaces concurrent package runs correctly
 		dbname := "dendrite_test.db"
 		return fmt.Sprintf("file:%s", dbname), func() {
 			err := os.Remove(dbname)
@@ -79,13 +106,9 @@ func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, clo
 	if user == "" {
 		user = currentUser()
 	}
-	dbName := os.Getenv("POSTGRES_DB")
-	if dbName == "" {
-		dbName = createLocalDB("dendrite_test")
-	}
 	connStr = fmt.Sprintf(
-		"user=%s dbname=%s sslmode=disable",
-		user, dbName,
+		"user=%s sslmode=disable",
+		user,
 	)
 	// optional vars, used in CI
 	password := os.Getenv("POSTGRES_PASSWORD")
@@ -97,6 +120,25 @@ func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, clo
 		connStr += fmt.Sprintf(" host=%s", host)
 	}
 
+	// superuser database
+	postgresDB := os.Getenv("POSTGRES_DB")
+	// we cannot use 'dendrite_test' here else 2x concurrently running packages will try to use the same db.
+	// instead, hash the current working directory, snaffle the first 16 bytes and append that to dendrite_test
+	// and use that as the unique db name. We do this because packages are per-directory hence by hashing the
+	// working (test) directory we ensure we get a consistent hash and don't hash against concurrent packages.
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("cannot get working directory: %s", err)
+	}
+	hash := sha256.Sum256([]byte(wd))
+	dbName := fmt.Sprintf("dendrite_test_%s", hex.EncodeToString(hash[:16]))
+	if postgresDB == "" { // local server, use createdb
+		createLocalDB(dbName)
+	} else { // remote server, shell into the postgres user and CREATE DATABASE
+		createRemoteDB(t, dbName, user, connStr)
+	}
+	connStr += fmt.Sprintf(" dbname=%s", dbName)
+
 	return connStr, func() {
 		// Drop all tables on the database to get a fresh instance
 		db, err := sql.Open("postgres", connStr)

From 29f216878994dccc68e34c90e5a0240c7698589f Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 13 Apr 2022 13:16:02 +0200
Subject: [PATCH 003/391] Make `/messages` filterable (#2347)

* Make /messages filterable
Fix bug when determining if an event contains an URL

* Add newly passing test

* Fix test
---
 syncapi/routing/messages.go                   |  6 +--
 syncapi/storage/interface.go                  |  2 +-
 .../postgres/output_room_events_table.go      | 38 +++++++++++++++++--
 syncapi/storage/shared/syncserver.go          | 13 ++++---
 syncapi/storage/sqlite3/account_data_table.go |  3 +-
 .../sqlite3/current_room_state_table.go       |  2 +-
 syncapi/storage/sqlite3/filtering.go          |  5 ++-
 .../sqlite3/output_room_events_table.go       | 33 +++++++++-------
 syncapi/storage/storage_test.go               |  3 +-
 syncapi/storage/tables/interface.go           |  2 +-
 .../storage/tables/output_room_events_test.go | 25 +++++++++++-
 sytest-whitelist                              |  3 +-
 12 files changed, 98 insertions(+), 37 deletions(-)

diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 36ba3a3e6..519aeff68 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -262,12 +262,8 @@ func (r *messagesReq) retrieveEvents() (
 	clientEvents []gomatrixserverlib.ClientEvent, start,
 	end types.TopologyToken, err error,
 ) {
-	eventFilter := r.filter
-
 	// Retrieve the events from the local database.
-	streamEvents, err := r.db.GetEventsInTopologicalRange(
-		r.ctx, r.from, r.to, r.roomID, eventFilter.Limit, r.backwardOrdering,
-	)
+	streamEvents, err := r.db.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering)
 	if err != nil {
 		err = fmt.Errorf("GetEventsInRange: %w", err)
 		return
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index cf3fd5532..14cb08a52 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -105,7 +105,7 @@ type Database interface {
 	// Returns an error if there was a problem communicating with the database.
 	DeletePeeks(ctx context.Context, RoomID, UserID string) (types.StreamPosition, error)
 	// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last.
-	GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error)
+	GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error)
 	// EventPositionInTopology returns the depth and stream position of the given event.
 	EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error)
 	// BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events.
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 269cd4494..17e2feab6 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -81,6 +81,15 @@ const insertEventSQL = "" +
 const selectEventsSQL = "" +
 	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)"
 
+const selectEventsWithFilterSQL = "" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" +
+	" AND ( $2::text[] IS NULL OR     sender  = ANY($2)  )" +
+	" AND ( $3::text[] IS NULL OR NOT(sender  = ANY($3)) )" +
+	" AND ( $4::text[] IS NULL OR     type LIKE ANY($4)  )" +
+	" AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" +
+	" AND ( $6::bool   IS NULL OR     contains_url = $6 )" +
+	" LIMIT $7"
+
 const selectRecentEventsSQL = "" +
 	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3" +
@@ -153,6 +162,7 @@ const selectContextAfterEventSQL = "" +
 type outputRoomEventsStatements struct {
 	insertEventStmt               *sql.Stmt
 	selectEventsStmt              *sql.Stmt
+	selectEventsWitFilterStmt     *sql.Stmt
 	selectMaxEventIDStmt          *sql.Stmt
 	selectRecentEventsStmt        *sql.Stmt
 	selectRecentEventsForSyncStmt *sql.Stmt
@@ -174,6 +184,7 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
 	return s, sqlutil.StatementList{
 		{&s.insertEventStmt, insertEventSQL},
 		{&s.selectEventsStmt, selectEventsSQL},
+		{&s.selectEventsWitFilterStmt, selectEventsWithFilterSQL},
 		{&s.selectMaxEventIDStmt, selectMaxEventIDSQL},
 		{&s.selectRecentEventsStmt, selectRecentEventsSQL},
 		{&s.selectRecentEventsForSyncStmt, selectRecentEventsForSyncSQL},
@@ -310,7 +321,7 @@ func (s *outputRoomEventsStatements) InsertEvent(
 	// Parse content as JSON and search for an "url" key
 	containsURL := false
 	var content map[string]interface{}
-	if json.Unmarshal(event.Content(), &content) != nil {
+	if json.Unmarshal(event.Content(), &content) == nil {
 		// Set containsURL to true if url is present
 		_, containsURL = content["url"]
 	}
@@ -429,10 +440,29 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents(
 // selectEvents returns the events for the given event IDs. If an event is
 // missing from the database, it will be omitted.
 func (s *outputRoomEventsStatements) SelectEvents(
-	ctx context.Context, txn *sql.Tx, eventIDs []string, preserveOrder bool,
+	ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool,
 ) ([]types.StreamEvent, error) {
-	stmt := sqlutil.TxStmt(txn, s.selectEventsStmt)
-	rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs))
+	var (
+		stmt *sql.Stmt
+		rows *sql.Rows
+		err  error
+	)
+	if filter == nil {
+		stmt = sqlutil.TxStmt(txn, s.selectEventsStmt)
+		rows, err = stmt.QueryContext(ctx, pq.StringArray(eventIDs))
+	} else {
+		senders, notSenders := getSendersRoomEventFilter(filter)
+		stmt = sqlutil.TxStmt(txn, s.selectEventsWitFilterStmt)
+		rows, err = stmt.QueryContext(ctx,
+			pq.StringArray(eventIDs),
+			pq.StringArray(senders),
+			pq.StringArray(notSenders),
+			pq.StringArray(filterConvertTypeWildcardToSQL(filter.Types)),
+			pq.StringArray(filterConvertTypeWildcardToSQL(filter.NotTypes)),
+			filter.ContainsURL,
+			filter.Limit,
+		)
+	}
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 14db5795c..91eba44e1 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -150,7 +150,7 @@ func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, stre
 // Returns an error if there was a problem talking with the database.
 // Does not include any transaction IDs in the returned events.
 func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) {
-	streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs, false)
+	streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs, nil, false)
 	if err != nil {
 		return nil, err
 	}
@@ -312,7 +312,7 @@ func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, e
 
 	// Check if we have all of the event's previous events. If an event is
 	// missing, add it to the room's backward extremities.
-	prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs(), false)
+	prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs(), nil, false)
 	if err != nil {
 		return err
 	}
@@ -429,7 +429,8 @@ func (d *Database) updateRoomState(
 func (d *Database) GetEventsInTopologicalRange(
 	ctx context.Context,
 	from, to *types.TopologyToken,
-	roomID string, limit int,
+	roomID string,
+	filter *gomatrixserverlib.RoomEventFilter,
 	backwardOrdering bool,
 ) (events []types.StreamEvent, err error) {
 	var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition
@@ -450,14 +451,14 @@ func (d *Database) GetEventsInTopologicalRange(
 	// Select the event IDs from the defined range.
 	var eIDs []string
 	eIDs, err = d.Topology.SelectEventIDsInRange(
-		ctx, nil, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, limit, !backwardOrdering,
+		ctx, nil, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, filter.Limit, !backwardOrdering,
 	)
 	if err != nil {
 		return
 	}
 
 	// Retrieve the events' contents using their IDs.
-	events, err = d.OutputEvents.SelectEvents(ctx, nil, eIDs, true)
+	events, err = d.OutputEvents.SelectEvents(ctx, nil, eIDs, filter, true)
 	return
 }
 
@@ -619,7 +620,7 @@ func (d *Database) fetchMissingStateEvents(
 ) ([]types.StreamEvent, error) {
 	// Fetch from the events table first so we pick up the stream ID for the
 	// event.
-	events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs, false)
+	events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs, nil, false)
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index b0aeb70f2..71a098177 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -104,8 +104,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 		},
 		filter.Senders, filter.NotSenders,
 		filter.Types, filter.NotTypes,
-		[]string{}, filter.Limit, FilterOrderAsc,
-	)
+		[]string{}, nil, filter.Limit, FilterOrderAsc)
 
 	rows, err := stmt.QueryContext(ctx, params...)
 	if err != nil {
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 464f32e04..ccda005c1 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -220,7 +220,7 @@ func (s *currentRoomStateStatements) SelectCurrentState(
 		},
 		stateFilter.Senders, stateFilter.NotSenders,
 		stateFilter.Types, stateFilter.NotTypes,
-		excludeEventIDs, stateFilter.Limit, FilterOrderNone,
+		excludeEventIDs, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderNone,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("s.prepareWithFilters: %w", err)
diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go
index 54b12ddf8..05edb7b8c 100644
--- a/syncapi/storage/sqlite3/filtering.go
+++ b/syncapi/storage/sqlite3/filtering.go
@@ -26,7 +26,7 @@ const (
 func prepareWithFilters(
 	db *sql.DB, txn *sql.Tx, query string, params []interface{},
 	senders, notsenders, types, nottypes *[]string, excludeEventIDs []string,
-	limit int, order FilterOrder,
+	containsURL *bool, limit int, order FilterOrder,
 ) (*sql.Stmt, []interface{}, error) {
 	offset := len(params)
 	if senders != nil {
@@ -69,6 +69,9 @@ func prepareWithFilters(
 			query += ` AND type NOT = ""`
 		}
 	}
+	if containsURL != nil {
+		query += fmt.Sprintf(" AND contains_url = %v", *containsURL)
+	}
 	if count := len(excludeEventIDs); count > 0 {
 		query += " AND event_id NOT IN " + sqlutil.QueryVariadicOffset(count, offset)
 		for _, v := range excludeEventIDs {
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 9da9d776e..188f7582b 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -168,7 +168,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		s.db, txn, stmtSQL, inputParams,
 		stateFilter.Senders, stateFilter.NotSenders,
 		stateFilter.Types, stateFilter.NotTypes,
-		nil, stateFilter.Limit, FilterOrderAsc,
+		nil, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderAsc,
 	)
 	if err != nil {
 		return nil, nil, fmt.Errorf("s.prepareWithFilters: %w", err)
@@ -277,7 +277,7 @@ func (s *outputRoomEventsStatements) InsertEvent(
 	// Parse content as JSON and search for an "url" key
 	containsURL := false
 	var content map[string]interface{}
-	if json.Unmarshal(event.Content(), &content) != nil {
+	if json.Unmarshal(event.Content(), &content) == nil {
 		// Set containsURL to true if url is present
 		_, containsURL = content["url"]
 	}
@@ -345,7 +345,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents(
 		},
 		eventFilter.Senders, eventFilter.NotSenders,
 		eventFilter.Types, eventFilter.NotTypes,
-		nil, eventFilter.Limit+1, FilterOrderDesc,
+		nil, eventFilter.ContainsURL, eventFilter.Limit+1, FilterOrderDesc,
 	)
 	if err != nil {
 		return nil, false, fmt.Errorf("s.prepareWithFilters: %w", err)
@@ -393,7 +393,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents(
 		},
 		eventFilter.Senders, eventFilter.NotSenders,
 		eventFilter.Types, eventFilter.NotTypes,
-		nil, eventFilter.Limit, FilterOrderAsc,
+		nil, eventFilter.ContainsURL, eventFilter.Limit, FilterOrderAsc,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("s.prepareWithFilters: %w", err)
@@ -419,20 +419,27 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents(
 // selectEvents returns the events for the given event IDs. If an event is
 // missing from the database, it will be omitted.
 func (s *outputRoomEventsStatements) SelectEvents(
-	ctx context.Context, txn *sql.Tx, eventIDs []string, preserveOrder bool,
+	ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool,
 ) ([]types.StreamEvent, error) {
 	iEventIDs := make([]interface{}, len(eventIDs))
 	for i := range eventIDs {
 		iEventIDs[i] = eventIDs[i]
 	}
 	selectSQL := strings.Replace(selectEventsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1)
-	var rows *sql.Rows
-	var err error
-	if txn != nil {
-		rows, err = txn.QueryContext(ctx, selectSQL, iEventIDs...)
-	} else {
-		rows, err = s.db.QueryContext(ctx, selectSQL, iEventIDs...)
+
+	if filter == nil {
+		filter = &gomatrixserverlib.RoomEventFilter{Limit: 20}
 	}
+	stmt, params, err := prepareWithFilters(
+		s.db, txn, selectSQL, iEventIDs,
+		filter.Senders, filter.NotSenders,
+		filter.Types, filter.NotTypes,
+		nil, filter.ContainsURL, filter.Limit, FilterOrderAsc,
+	)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := stmt.QueryContext(ctx, params...)
 	if err != nil {
 		return nil, err
 	}
@@ -527,7 +534,7 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
 		},
 		filter.Senders, filter.NotSenders,
 		filter.Types, filter.NotTypes,
-		nil, filter.Limit, FilterOrderDesc,
+		nil, filter.ContainsURL, filter.Limit, FilterOrderDesc,
 	)
 
 	rows, err := stmt.QueryContext(ctx, params...)
@@ -563,7 +570,7 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent(
 		},
 		filter.Senders, filter.NotSenders,
 		filter.Types, filter.NotTypes,
-		nil, filter.Limit, FilterOrderAsc,
+		nil, filter.ContainsURL, filter.Limit, FilterOrderAsc,
 	)
 
 	rows, err := stmt.QueryContext(ctx, params...)
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index 4e1634ece..15bb769a2 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -180,7 +180,8 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
 		to := types.TopologyToken{}
 
 		// backpaginate 5 messages starting at the latest position.
-		paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, 5, true)
+		filter := &gomatrixserverlib.RoomEventFilter{Limit: 5}
+		paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true)
 		if err != nil {
 			t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err)
 		}
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index a7df70248..993e2022b 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -59,7 +59,7 @@ type Events interface {
 	SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error)
 	// SelectEarlyEvents returns the earliest events in the given room.
 	SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter) ([]types.StreamEvent, error)
-	SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string, preserveOrder bool) ([]types.StreamEvent, error)
+	SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool) ([]types.StreamEvent, error)
 	UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error
 	// DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely.
 	DeleteEventsForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error)
diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go
index 7a81ffcd2..a143e5ecd 100644
--- a/syncapi/storage/tables/output_room_events_test.go
+++ b/syncapi/storage/tables/output_room_events_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func newOutputRoomEventsTable(t *testing.T, dbType test.DBType) (tables.Events, *sql.DB, func()) {
@@ -61,7 +62,7 @@ func TestOutputRoomEventsTable(t *testing.T) {
 			wantEventIDs := []string{
 				events[2].EventID(), events[0].EventID(), events[3].EventID(), events[1].EventID(),
 			}
-			gotEvents, err := tab.SelectEvents(ctx, txn, wantEventIDs, true)
+			gotEvents, err := tab.SelectEvents(ctx, txn, wantEventIDs, nil, true)
 			if err != nil {
 				return fmt.Errorf("failed to SelectEvents: %s", err)
 			}
@@ -73,6 +74,28 @@ func TestOutputRoomEventsTable(t *testing.T) {
 				return fmt.Errorf("SelectEvents\ngot  %v\n want %v", gotEventIDs, wantEventIDs)
 			}
 
+			// Test that contains_url is correctly populated
+			urlEv := room.CreateEvent(t, alice, "m.text", map[string]interface{}{
+				"body": "test.txt",
+				"url":  "mxc://test.txt",
+			})
+			if _, err = tab.InsertEvent(ctx, txn, urlEv, nil, nil, nil, false); err != nil {
+				return fmt.Errorf("failed to InsertEvent: %s", err)
+			}
+			wantEventID := []string{urlEv.EventID()}
+			t := true
+			gotEvents, err = tab.SelectEvents(ctx, txn, wantEventID, &gomatrixserverlib.RoomEventFilter{Limit: 1, ContainsURL: &t}, true)
+			if err != nil {
+				return fmt.Errorf("failed to SelectEvents: %s", err)
+			}
+			gotEventIDs = make([]string, len(gotEvents))
+			for i := range gotEvents {
+				gotEventIDs[i] = gotEvents[i].EventID()
+			}
+			if !reflect.DeepEqual(gotEventIDs, wantEventID) {
+				return fmt.Errorf("SelectEvents\ngot  %v\n want %v", gotEventIDs, wantEventID)
+			}
+
 			return nil
 		})
 		if err != nil {
diff --git a/sytest-whitelist b/sytest-whitelist
index a7aea05ed..f63b96f52 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -698,4 +698,5 @@ Ignore user in existing room
 Ignore invite in full sync
 Ignore invite in incremental sync
 A filtered timeline reaches its limit
-A change to displayname should not result in a full state sync
\ No newline at end of file
+A change to displayname should not result in a full state sync
+Can fetch images in room
\ No newline at end of file

From 1140f39993f1d4fb80952bf853bb05df0b42ca20 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 13 Apr 2022 12:35:30 +0100
Subject: [PATCH 004/391] Precompute values for `userIDSet` in sync notifier
 (#2348)

* Precompute values for `userIDSet` in sync notifier

* Mutexes

* Fixes

* Sensible initial value

* Update syncapi/notifier/notifier.go

Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>

* Placate the almighty linter

Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>
---
 syncapi/notifier/notifier.go | 73 +++++++++++++++++++++++++++---------
 1 file changed, 56 insertions(+), 17 deletions(-)

diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index 443744b6f..82834239b 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -36,7 +36,7 @@ import (
 type Notifier struct {
 	lock *sync.RWMutex
 	// A map of RoomID => Set<UserID> : Must only be accessed by the OnNewEvent goroutine
-	roomIDToJoinedUsers map[string]userIDSet
+	roomIDToJoinedUsers map[string]*userIDSet
 	// A map of RoomID => Set<UserID> : Must only be accessed by the OnNewEvent goroutine
 	roomIDToPeekingDevices map[string]peekingDeviceSet
 	// The latest sync position
@@ -54,7 +54,7 @@ type Notifier struct {
 // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase).
 func NewNotifier() *Notifier {
 	return &Notifier{
-		roomIDToJoinedUsers:    make(map[string]userIDSet),
+		roomIDToJoinedUsers:    make(map[string]*userIDSet),
 		roomIDToPeekingDevices: make(map[string]peekingDeviceSet),
 		userDeviceStreams:      make(map[string]map[string]*UserDeviceStream),
 		lock:                   &sync.RWMutex{},
@@ -262,7 +262,7 @@ func (n *Notifier) SharedUsers(userID string) []string {
 func (n *Notifier) _sharedUsers(userID string) []string {
 	n._sharedUserMap[userID] = struct{}{}
 	for roomID, users := range n.roomIDToJoinedUsers {
-		if _, ok := users[userID]; !ok {
+		if ok := users.isIn(userID); !ok {
 			continue
 		}
 		for _, userID := range n._joinedUsers(roomID) {
@@ -282,8 +282,11 @@ func (n *Notifier) IsSharedUser(userA, userB string) bool {
 	defer n.lock.RUnlock()
 	var okA, okB bool
 	for _, users := range n.roomIDToJoinedUsers {
-		_, okA = users[userA]
-		_, okB = users[userB]
+		okA = users.isIn(userA)
+		if !okA {
+			continue
+		}
+		okB = users.isIn(userB)
 		if okA && okB {
 			return true
 		}
@@ -345,11 +348,12 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) {
 	// This is just the bulk form of addJoinedUser
 	for roomID, userIDs := range roomIDToUserIDs {
 		if _, ok := n.roomIDToJoinedUsers[roomID]; !ok {
-			n.roomIDToJoinedUsers[roomID] = make(userIDSet, len(userIDs))
+			n.roomIDToJoinedUsers[roomID] = newUserIDSet(len(userIDs))
 		}
 		for _, userID := range userIDs {
 			n.roomIDToJoinedUsers[roomID].add(userID)
 		}
+		n.roomIDToJoinedUsers[roomID].precompute()
 	}
 }
 
@@ -440,16 +444,18 @@ func (n *Notifier) _fetchUserStreams(userID string) []*UserDeviceStream {
 
 func (n *Notifier) _addJoinedUser(roomID, userID string) {
 	if _, ok := n.roomIDToJoinedUsers[roomID]; !ok {
-		n.roomIDToJoinedUsers[roomID] = make(userIDSet)
+		n.roomIDToJoinedUsers[roomID] = newUserIDSet(8)
 	}
 	n.roomIDToJoinedUsers[roomID].add(userID)
+	n.roomIDToJoinedUsers[roomID].precompute()
 }
 
 func (n *Notifier) _removeJoinedUser(roomID, userID string) {
 	if _, ok := n.roomIDToJoinedUsers[roomID]; !ok {
-		n.roomIDToJoinedUsers[roomID] = make(userIDSet)
+		n.roomIDToJoinedUsers[roomID] = newUserIDSet(8)
 	}
 	n.roomIDToJoinedUsers[roomID].remove(userID)
+	n.roomIDToJoinedUsers[roomID].precompute()
 }
 
 func (n *Notifier) JoinedUsers(roomID string) (userIDs []string) {
@@ -521,19 +527,52 @@ func (n *Notifier) _removeEmptyUserStreams() {
 }
 
 // A string set, mainly existing for improving clarity of structs in this file.
-type userIDSet map[string]struct{}
-
-func (s userIDSet) add(str string) {
-	s[str] = struct{}{}
+type userIDSet struct {
+	sync.Mutex
+	set         map[string]struct{}
+	precomputed []string
 }
 
-func (s userIDSet) remove(str string) {
-	delete(s, str)
+func newUserIDSet(cap int) *userIDSet {
+	return &userIDSet{
+		set:         make(map[string]struct{}, cap),
+		precomputed: nil,
+	}
 }
 
-func (s userIDSet) values() (vals []string) {
-	vals = make([]string, 0, len(s))
-	for str := range s {
+func (s *userIDSet) add(str string) {
+	s.Lock()
+	defer s.Unlock()
+	s.set[str] = struct{}{}
+	s.precomputed = s.precomputed[:0] // invalidate cache
+}
+
+func (s *userIDSet) remove(str string) {
+	s.Lock()
+	defer s.Unlock()
+	delete(s.set, str)
+	s.precomputed = s.precomputed[:0] // invalidate cache
+}
+
+func (s *userIDSet) precompute() {
+	s.Lock()
+	defer s.Unlock()
+	s.precomputed = s.values()
+}
+
+func (s *userIDSet) isIn(str string) bool {
+	s.Lock()
+	defer s.Unlock()
+	_, ok := s.set[str]
+	return ok
+}
+
+func (s *userIDSet) values() (vals []string) {
+	if len(s.precomputed) > 0 {
+		return s.precomputed // only return if not invalidated
+	}
+	vals = make([]string, 0, len(s.set))
+	for str := range s.set {
 		vals = append(vals, str)
 	}
 	return

From 3a5e9a0f284eef0fcb94a22035f9ffd2eb65eedf Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 13 Apr 2022 16:41:22 +0100
Subject: [PATCH 005/391] Use default sync filter if specified filter is not
 found (should fix #2350) (#2351)

---
 syncapi/sync/request.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go
index 09a62e3dd..f04f172d3 100644
--- a/syncapi/sync/request.go
+++ b/syncapi/sync/request.go
@@ -15,6 +15,7 @@
 package sync
 
 import (
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -60,10 +61,10 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
 				util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 				return nil, fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 			}
-			if f, err := syncDB.GetFilter(req.Context(), localpart, filterQuery); err != nil {
+			if f, err := syncDB.GetFilter(req.Context(), localpart, filterQuery); err != nil && err != sql.ErrNoRows {
 				util.GetLogger(req.Context()).WithError(err).Error("syncDB.GetFilter failed")
 				return nil, fmt.Errorf("syncDB.GetFilter: %w", err)
-			} else {
+			} else if f != nil {
 				filter = *f
 			}
 		}

From 3ddbffd59ece5f74d951d6209882d9d954db4bc3 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 14 Apr 2022 14:32:48 +0200
Subject: [PATCH 006/391] Refactor media storage layer, add tests (#2352)

* Refactor mediaapi storage layer

* Verify filetype before trying to create thumbnails

* Add media api storage tests

* Fix returned values
---
 mediaapi/mediaapi.go                          |   2 +-
 mediaapi/routing/upload.go                    |  21 +++
 mediaapi/routing/upload_test.go               |   2 +-
 mediaapi/storage/interface.go                 |   8 ++
 .../postgres/media_repository_table.go        |  33 +++--
 mediaapi/storage/postgres/mediaapi.go         |  46 ++++++
 mediaapi/storage/postgres/prepare.go          |  38 -----
 mediaapi/storage/postgres/sql.go              |  36 -----
 mediaapi/storage/postgres/thumbnail_table.go  |  34 +++--
 .../storage.go => shared/mediaapi.go}         |  93 +++++-------
 .../storage/sqlite3/media_repository_table.go |  63 ++++----
 .../storage/sqlite3/{sql.go => mediaapi.go}   |  35 +++--
 mediaapi/storage/sqlite3/prepare.go           |  38 -----
 mediaapi/storage/sqlite3/storage.go           | 123 ----------------
 mediaapi/storage/sqlite3/thumbnail_table.go   |  63 ++++----
 mediaapi/storage/storage.go                   |   8 +-
 mediaapi/storage/storage_test.go              | 135 ++++++++++++++++++
 mediaapi/storage/storage_wasm.go              |   4 +-
 mediaapi/storage/tables/interface.go          |  46 ++++++
 mediaapi/types/types.go                       |   5 +-
 20 files changed, 417 insertions(+), 416 deletions(-)
 create mode 100644 mediaapi/storage/postgres/mediaapi.go
 delete mode 100644 mediaapi/storage/postgres/prepare.go
 delete mode 100644 mediaapi/storage/postgres/sql.go
 rename mediaapi/storage/{postgres/storage.go => shared/mediaapi.go} (52%)
 rename mediaapi/storage/sqlite3/{sql.go => mediaapi.go} (51%)
 delete mode 100644 mediaapi/storage/sqlite3/prepare.go
 delete mode 100644 mediaapi/storage/sqlite3/storage.go
 create mode 100644 mediaapi/storage/storage_test.go
 create mode 100644 mediaapi/storage/tables/interface.go

diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go
index c010981c0..e5daf480d 100644
--- a/mediaapi/mediaapi.go
+++ b/mediaapi/mediaapi.go
@@ -32,7 +32,7 @@ func AddPublicRoutes(
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
 ) {
-	mediaDB, err := storage.Open(&cfg.Database)
+	mediaDB, err := storage.NewMediaAPIDatasource(&cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to media db")
 	}
diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go
index f762b2ff5..972c52af0 100644
--- a/mediaapi/routing/upload.go
+++ b/mediaapi/routing/upload.go
@@ -22,6 +22,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strings"
 
@@ -311,6 +312,26 @@ func (r *uploadRequest) storeFileAndMetadata(
 	}
 
 	go func() {
+		file, err := os.Open(string(finalPath))
+		if err != nil {
+			r.Logger.WithError(err).Error("unable to open file")
+			return
+		}
+		defer file.Close() // nolint: errcheck
+		// http.DetectContentType only needs 512 bytes
+		buf := make([]byte, 512)
+		_, err = file.Read(buf)
+		if err != nil {
+			r.Logger.WithError(err).Error("unable to read file")
+			return
+		}
+		// Check if we need to generate thumbnails
+		fileType := http.DetectContentType(buf)
+		if !strings.HasPrefix(fileType, "image") {
+			r.Logger.WithField("contentType", fileType).Debugf("uploaded file is not an image or can not be thumbnailed, not generating thumbnails")
+			return
+		}
+
 		busy, err := thumbnailer.GenerateThumbnails(
 			context.Background(), finalPath, thumbnailSizes, r.MediaMetadata,
 			activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger,
diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go
index e81254f35..b2c2f5a44 100644
--- a/mediaapi/routing/upload_test.go
+++ b/mediaapi/routing/upload_test.go
@@ -51,7 +51,7 @@ func Test_uploadRequest_doUpload(t *testing.T) {
 	_ = os.Mkdir(testdataPath, os.ModePerm)
 	defer fileutils.RemoveDir(types.Path(testdataPath), nil)
 
-	db, err := storage.Open(&config.DatabaseOptions{
+	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
 		ConnectionString:       "file::memory:?cache=shared",
 		MaxOpenConnections:     100,
 		MaxIdleConnections:     2,
diff --git a/mediaapi/storage/interface.go b/mediaapi/storage/interface.go
index 843199719..d083be1eb 100644
--- a/mediaapi/storage/interface.go
+++ b/mediaapi/storage/interface.go
@@ -22,9 +22,17 @@ import (
 )
 
 type Database interface {
+	MediaRepository
+	Thumbnails
+}
+
+type MediaRepository interface {
 	StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error
 	GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error)
 	GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error)
+}
+
+type Thumbnails interface {
 	StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error
 	GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error)
 	GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error)
diff --git a/mediaapi/storage/postgres/media_repository_table.go b/mediaapi/storage/postgres/media_repository_table.go
index 1d3264ca9..41cee4878 100644
--- a/mediaapi/storage/postgres/media_repository_table.go
+++ b/mediaapi/storage/postgres/media_repository_table.go
@@ -20,6 +20,8 @@ import (
 	"database/sql"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -69,24 +71,25 @@ type mediaStatements struct {
 	selectMediaByHashStmt *sql.Stmt
 }
 
-func (s *mediaStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(mediaSchema)
+func NewPostgresMediaRepositoryTable(db *sql.DB) (tables.MediaRepository, error) {
+	s := &mediaStatements{}
+	_, err := db.Exec(mediaSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertMediaStmt, insertMediaSQL},
 		{&s.selectMediaStmt, selectMediaSQL},
 		{&s.selectMediaByHashStmt, selectMediaByHashSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *mediaStatements) insertMedia(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
+func (s *mediaStatements) InsertMedia(
+	ctx context.Context, txn *sql.Tx, mediaMetadata *types.MediaMetadata,
 ) error {
-	mediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	_, err := s.insertMediaStmt.ExecContext(
+	mediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertMediaStmt).ExecContext(
 		ctx,
 		mediaMetadata.MediaID,
 		mediaMetadata.Origin,
@@ -100,14 +103,14 @@ func (s *mediaStatements) insertMedia(
 	return err
 }
 
-func (s *mediaStatements) selectMedia(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMedia(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		MediaID: mediaID,
 		Origin:  mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaStmt).QueryRowContext(
 		ctx, mediaMetadata.MediaID, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,
@@ -120,14 +123,14 @@ func (s *mediaStatements) selectMedia(
 	return &mediaMetadata, err
 }
 
-func (s *mediaStatements) selectMediaByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMediaByHash(
+	ctx context.Context, txn *sql.Tx, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		Base64Hash: mediaHash,
 		Origin:     mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaByHashStmt).QueryRowContext(
 		ctx, mediaMetadata.Base64Hash, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,
diff --git a/mediaapi/storage/postgres/mediaapi.go b/mediaapi/storage/postgres/mediaapi.go
new file mode 100644
index 000000000..ea70e575b
--- /dev/null
+++ b/mediaapi/storage/postgres/mediaapi.go
@@ -0,0 +1,46 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	// Import the postgres database driver.
+	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+// NewDatabase opens a postgres database.
+func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, err := sqlutil.Open(dbProperties)
+	if err != nil {
+		return nil, err
+	}
+	mediaRepo, err := NewPostgresMediaRepositoryTable(db)
+	if err != nil {
+		return nil, err
+	}
+	thumbnails, err := NewPostgresThumbnailsTable(db)
+	if err != nil {
+		return nil, err
+	}
+	return &shared.Database{
+		MediaRepository: mediaRepo,
+		Thumbnails:      thumbnails,
+		DB:              db,
+		Writer:          sqlutil.NewExclusiveWriter(),
+	}, nil
+}
diff --git a/mediaapi/storage/postgres/prepare.go b/mediaapi/storage/postgres/prepare.go
deleted file mode 100644
index a2e01884e..000000000
--- a/mediaapi/storage/postgres/prepare.go
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// FIXME: This should be made internal!
-
-package postgres
-
-import (
-	"database/sql"
-)
-
-// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
-type statementList []struct {
-	statement **sql.Stmt
-	sql       string
-}
-
-// prepare the SQL for each statement in the list and assign the result to the prepared statement.
-func (s statementList) prepare(db *sql.DB) (err error) {
-	for _, statement := range s {
-		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
-			return
-		}
-	}
-	return
-}
diff --git a/mediaapi/storage/postgres/sql.go b/mediaapi/storage/postgres/sql.go
deleted file mode 100644
index 181cd15ff..000000000
--- a/mediaapi/storage/postgres/sql.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"database/sql"
-)
-
-type statements struct {
-	media     mediaStatements
-	thumbnail thumbnailStatements
-}
-
-func (s *statements) prepare(db *sql.DB) (err error) {
-	if err = s.media.prepare(db); err != nil {
-		return
-	}
-	if err = s.thumbnail.prepare(db); err != nil {
-		return
-	}
-
-	return
-}
diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go
index 3f28cdbbf..7e07b476e 100644
--- a/mediaapi/storage/postgres/thumbnail_table.go
+++ b/mediaapi/storage/postgres/thumbnail_table.go
@@ -21,6 +21,8 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -63,7 +65,7 @@ SELECT content_type, file_size_bytes, creation_ts FROM mediaapi_thumbnail WHERE
 
 // Note: this selects all thumbnails for a media_origin and media_id
 const selectThumbnailsSQL = `
-SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2
+SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2 ORDER BY creation_ts ASC
 `
 
 type thumbnailStatements struct {
@@ -72,24 +74,25 @@ type thumbnailStatements struct {
 	selectThumbnailsStmt *sql.Stmt
 }
 
-func (s *thumbnailStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(thumbnailSchema)
+func NewPostgresThumbnailsTable(db *sql.DB) (tables.Thumbnails, error) {
+	s := &thumbnailStatements{}
+	_, err := db.Exec(thumbnailSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertThumbnailStmt, insertThumbnailSQL},
 		{&s.selectThumbnailStmt, selectThumbnailSQL},
 		{&s.selectThumbnailsStmt, selectThumbnailsSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *thumbnailStatements) insertThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
+func (s *thumbnailStatements) InsertThumbnail(
+	ctx context.Context, txn *sql.Tx, thumbnailMetadata *types.ThumbnailMetadata,
 ) error {
-	thumbnailMetadata.MediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	_, err := s.insertThumbnailStmt.ExecContext(
+	thumbnailMetadata.MediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertThumbnailStmt).ExecContext(
 		ctx,
 		thumbnailMetadata.MediaMetadata.MediaID,
 		thumbnailMetadata.MediaMetadata.Origin,
@@ -103,8 +106,9 @@ func (s *thumbnailStatements) insertThumbnail(
 	return err
 }
 
-func (s *thumbnailStatements) selectThumbnail(
+func (s *thumbnailStatements) SelectThumbnail(
 	ctx context.Context,
+	txn *sql.Tx,
 	mediaID types.MediaID,
 	mediaOrigin gomatrixserverlib.ServerName,
 	width, height int,
@@ -121,7 +125,7 @@ func (s *thumbnailStatements) selectThumbnail(
 			ResizeMethod: resizeMethod,
 		},
 	}
-	err := s.selectThumbnailStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailStmt).QueryRowContext(
 		ctx,
 		thumbnailMetadata.MediaMetadata.MediaID,
 		thumbnailMetadata.MediaMetadata.Origin,
@@ -136,10 +140,10 @@ func (s *thumbnailStatements) selectThumbnail(
 	return &thumbnailMetadata, err
 }
 
-func (s *thumbnailStatements) selectThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *thumbnailStatements) SelectThumbnails(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
 ) ([]*types.ThumbnailMetadata, error) {
-	rows, err := s.selectThumbnailsStmt.QueryContext(
+	rows, err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailsStmt).QueryContext(
 		ctx, mediaID, mediaOrigin,
 	)
 	if err != nil {
diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/shared/mediaapi.go
similarity index 52%
rename from mediaapi/storage/postgres/storage.go
rename to mediaapi/storage/shared/mediaapi.go
index 61ad468fe..c8d9ad6ab 100644
--- a/mediaapi/storage/postgres/storage.go
+++ b/mediaapi/storage/shared/mediaapi.go
@@ -1,5 +1,4 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+// Copyright 2022 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,54 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package postgres
+package shared
 
 import (
 	"context"
 	"database/sql"
 
-	// Import the postgres database driver.
-	_ "github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// Database is used to store metadata about a repository of media files.
 type Database struct {
-	statements statements
-	db         *sql.DB
-}
-
-// Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
-	var d Database
-	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
-		return nil, err
-	}
-	if err = d.statements.prepare(d.db); err != nil {
-		return nil, err
-	}
-	return &d, nil
+	DB              *sql.DB
+	Writer          sqlutil.Writer
+	MediaRepository tables.MediaRepository
+	Thumbnails      tables.Thumbnails
 }
 
 // StoreMediaMetadata inserts the metadata about the uploaded media into the database.
 // Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreMediaMetadata(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
-) error {
-	return d.statements.media.insertMedia(ctx, mediaMetadata)
+func (d Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.MediaRepository.InsertMedia(ctx, txn, mediaMetadata)
+	})
 }
 
 // GetMediaMetadata returns metadata about media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadata(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin)
+func (d Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) {
+	mediaMetadata, err := d.MediaRepository.SelectMedia(ctx, nil, mediaID, mediaOrigin)
 	if err != nil && err == sql.ErrNoRows {
 		return nil, nil
 	}
@@ -70,10 +53,8 @@ func (d *Database) GetMediaMetadata(
 // GetMediaMetadataByHash returns metadata about media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadataByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMediaByHash(ctx, mediaHash, mediaOrigin)
+func (d Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) {
+	mediaMetadata, err := d.MediaRepository.SelectMediaByHash(ctx, nil, mediaHash, mediaOrigin)
 	if err != nil && err == sql.ErrNoRows {
 		return nil, nil
 	}
@@ -82,40 +63,36 @@ func (d *Database) GetMediaMetadataByHash(
 
 // StoreThumbnail inserts the metadata about the thumbnail into the database.
 // Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
-) error {
-	return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata)
+func (d Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.Thumbnails.InsertThumbnail(ctx, txn, thumbnailMetadata)
+	})
 }
 
 // GetThumbnail returns metadata about a specific thumbnail.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this thumbnail.
-func (d *Database) GetThumbnail(
-	ctx context.Context,
-	mediaID types.MediaID,
-	mediaOrigin gomatrixserverlib.ServerName,
-	width, height int,
-	resizeMethod string,
-) (*types.ThumbnailMetadata, error) {
-	thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail(
-		ctx, mediaID, mediaOrigin, width, height, resizeMethod,
-	)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
+func (d Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) {
+	metadata, err := d.Thumbnails.SelectThumbnail(ctx, nil, mediaID, mediaOrigin, width, height, resizeMethod)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
 	}
-	return thumbnailMetadata, err
+	return metadata, err
 }
 
 // GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there are no thumbnails associated with this media.
-func (d *Database) GetThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) ([]*types.ThumbnailMetadata, error) {
-	thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
+func (d Database) GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) {
+	metadatas, err := d.Thumbnails.SelectThumbnails(ctx, nil, mediaID, mediaOrigin)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
 	}
-	return thumbnails, err
+	return metadatas, err
 }
diff --git a/mediaapi/storage/sqlite3/media_repository_table.go b/mediaapi/storage/sqlite3/media_repository_table.go
index bcef609d8..78431967f 100644
--- a/mediaapi/storage/sqlite3/media_repository_table.go
+++ b/mediaapi/storage/sqlite3/media_repository_table.go
@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -66,57 +67,53 @@ SELECT content_type, file_size_bytes, creation_ts, upload_name, media_id, user_i
 
 type mediaStatements struct {
 	db                    *sql.DB
-	writer                sqlutil.Writer
 	insertMediaStmt       *sql.Stmt
 	selectMediaStmt       *sql.Stmt
 	selectMediaByHashStmt *sql.Stmt
 }
 
-func (s *mediaStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	s.db = db
-	s.writer = writer
-
-	_, err = db.Exec(mediaSchema)
+func NewSQLiteMediaRepositoryTable(db *sql.DB) (tables.MediaRepository, error) {
+	s := &mediaStatements{
+		db: db,
+	}
+	_, err := db.Exec(mediaSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertMediaStmt, insertMediaSQL},
 		{&s.selectMediaStmt, selectMediaSQL},
 		{&s.selectMediaByHashStmt, selectMediaByHashSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *mediaStatements) insertMedia(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
+func (s *mediaStatements) InsertMedia(
+	ctx context.Context, txn *sql.Tx, mediaMetadata *types.MediaMetadata,
 ) error {
-	mediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		stmt := sqlutil.TxStmt(txn, s.insertMediaStmt)
-		_, err := stmt.ExecContext(
-			ctx,
-			mediaMetadata.MediaID,
-			mediaMetadata.Origin,
-			mediaMetadata.ContentType,
-			mediaMetadata.FileSizeBytes,
-			mediaMetadata.CreationTimestamp,
-			mediaMetadata.UploadName,
-			mediaMetadata.Base64Hash,
-			mediaMetadata.UserID,
-		)
-		return err
-	})
+	mediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertMediaStmt).ExecContext(
+		ctx,
+		mediaMetadata.MediaID,
+		mediaMetadata.Origin,
+		mediaMetadata.ContentType,
+		mediaMetadata.FileSizeBytes,
+		mediaMetadata.CreationTimestamp,
+		mediaMetadata.UploadName,
+		mediaMetadata.Base64Hash,
+		mediaMetadata.UserID,
+	)
+	return err
 }
 
-func (s *mediaStatements) selectMedia(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMedia(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		MediaID: mediaID,
 		Origin:  mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaStmt).QueryRowContext(
 		ctx, mediaMetadata.MediaID, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,
@@ -129,14 +126,14 @@ func (s *mediaStatements) selectMedia(
 	return &mediaMetadata, err
 }
 
-func (s *mediaStatements) selectMediaByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
+func (s *mediaStatements) SelectMediaByHash(
+	ctx context.Context, txn *sql.Tx, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
 ) (*types.MediaMetadata, error) {
 	mediaMetadata := types.MediaMetadata{
 		Base64Hash: mediaHash,
 		Origin:     mediaOrigin,
 	}
-	err := s.selectMediaStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectMediaByHashStmt).QueryRowContext(
 		ctx, mediaMetadata.Base64Hash, mediaMetadata.Origin,
 	).Scan(
 		&mediaMetadata.ContentType,
diff --git a/mediaapi/storage/sqlite3/sql.go b/mediaapi/storage/sqlite3/mediaapi.go
similarity index 51%
rename from mediaapi/storage/sqlite3/sql.go
rename to mediaapi/storage/sqlite3/mediaapi.go
index 245bd40cc..abf329367 100644
--- a/mediaapi/storage/sqlite3/sql.go
+++ b/mediaapi/storage/sqlite3/mediaapi.go
@@ -16,23 +16,30 @@
 package sqlite3
 
 import (
-	"database/sql"
-
+	// Import the postgres database driver.
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
-type statements struct {
-	media     mediaStatements
-	thumbnail thumbnailStatements
-}
-
-func (s *statements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	if err = s.media.prepare(db, writer); err != nil {
-		return
+// NewDatabase opens a SQLIte database.
+func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, err := sqlutil.Open(dbProperties)
+	if err != nil {
+		return nil, err
 	}
-	if err = s.thumbnail.prepare(db, writer); err != nil {
-		return
+	mediaRepo, err := NewSQLiteMediaRepositoryTable(db)
+	if err != nil {
+		return nil, err
 	}
-
-	return
+	thumbnails, err := NewSQLiteThumbnailsTable(db)
+	if err != nil {
+		return nil, err
+	}
+	return &shared.Database{
+		MediaRepository: mediaRepo,
+		Thumbnails:      thumbnails,
+		DB:              db,
+		Writer:          sqlutil.NewExclusiveWriter(),
+	}, nil
 }
diff --git a/mediaapi/storage/sqlite3/prepare.go b/mediaapi/storage/sqlite3/prepare.go
deleted file mode 100644
index 8fb3b56f3..000000000
--- a/mediaapi/storage/sqlite3/prepare.go
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// FIXME: This should be made internal!
-
-package sqlite3
-
-import (
-	"database/sql"
-)
-
-// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
-type statementList []struct {
-	statement **sql.Stmt
-	sql       string
-}
-
-// prepare the SQL for each statement in the list and assign the result to the prepared statement.
-func (s statementList) prepare(db *sql.DB) (err error) {
-	for _, statement := range s {
-		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
-			return
-		}
-	}
-	return
-}
diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go
deleted file mode 100644
index fa442173b..000000000
--- a/mediaapi/storage/sqlite3/storage.go
+++ /dev/null
@@ -1,123 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-
-	// Import the postgres database driver.
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/mediaapi/types"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// Database is used to store metadata about a repository of media files.
-type Database struct {
-	statements statements
-	db         *sql.DB
-	writer     sqlutil.Writer
-}
-
-// Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
-	d := Database{
-		writer: sqlutil.NewExclusiveWriter(),
-	}
-	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
-		return nil, err
-	}
-	if err = d.statements.prepare(d.db, d.writer); err != nil {
-		return nil, err
-	}
-	return &d, nil
-}
-
-// StoreMediaMetadata inserts the metadata about the uploaded media into the database.
-// Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreMediaMetadata(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
-) error {
-	return d.statements.media.insertMedia(ctx, mediaMetadata)
-}
-
-// GetMediaMetadata returns metadata about media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadata(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return mediaMetadata, err
-}
-
-// GetMediaMetadataByHash returns metadata about media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadataByHash(
-	ctx context.Context, mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMediaByHash(ctx, mediaHash, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return mediaMetadata, err
-}
-
-// StoreThumbnail inserts the metadata about the thumbnail into the database.
-// Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
-) error {
-	return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata)
-}
-
-// GetThumbnail returns metadata about a specific thumbnail.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this thumbnail.
-func (d *Database) GetThumbnail(
-	ctx context.Context,
-	mediaID types.MediaID,
-	mediaOrigin gomatrixserverlib.ServerName,
-	width, height int,
-	resizeMethod string,
-) (*types.ThumbnailMetadata, error) {
-	thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail(
-		ctx, mediaID, mediaOrigin, width, height, resizeMethod,
-	)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return thumbnailMetadata, err
-}
-
-// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there are no thumbnails associated with this media.
-func (d *Database) GetThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) ([]*types.ThumbnailMetadata, error) {
-	thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return thumbnails, err
-}
diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go
index 06b056b6e..5ff2fece0 100644
--- a/mediaapi/storage/sqlite3/thumbnail_table.go
+++ b/mediaapi/storage/sqlite3/thumbnail_table.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/mediaapi/storage/tables"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -54,55 +55,48 @@ SELECT content_type, file_size_bytes, creation_ts FROM mediaapi_thumbnail WHERE
 
 // Note: this selects all thumbnails for a media_origin and media_id
 const selectThumbnailsSQL = `
-SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2
+SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2 ORDER BY creation_ts ASC
 `
 
 type thumbnailStatements struct {
-	db                   *sql.DB
-	writer               sqlutil.Writer
 	insertThumbnailStmt  *sql.Stmt
 	selectThumbnailStmt  *sql.Stmt
 	selectThumbnailsStmt *sql.Stmt
 }
 
-func (s *thumbnailStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	_, err = db.Exec(thumbnailSchema)
+func NewSQLiteThumbnailsTable(db *sql.DB) (tables.Thumbnails, error) {
+	s := &thumbnailStatements{}
+	_, err := db.Exec(thumbnailSchema)
 	if err != nil {
-		return
+		return nil, err
 	}
-	s.db = db
-	s.writer = writer
 
-	return statementList{
+	return s, sqlutil.StatementList{
 		{&s.insertThumbnailStmt, insertThumbnailSQL},
 		{&s.selectThumbnailStmt, selectThumbnailSQL},
 		{&s.selectThumbnailsStmt, selectThumbnailsSQL},
-	}.prepare(db)
+	}.Prepare(db)
 }
 
-func (s *thumbnailStatements) insertThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
-) error {
-	thumbnailMetadata.MediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
-	return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		stmt := sqlutil.TxStmt(txn, s.insertThumbnailStmt)
-		_, err := stmt.ExecContext(
-			ctx,
-			thumbnailMetadata.MediaMetadata.MediaID,
-			thumbnailMetadata.MediaMetadata.Origin,
-			thumbnailMetadata.MediaMetadata.ContentType,
-			thumbnailMetadata.MediaMetadata.FileSizeBytes,
-			thumbnailMetadata.MediaMetadata.CreationTimestamp,
-			thumbnailMetadata.ThumbnailSize.Width,
-			thumbnailMetadata.ThumbnailSize.Height,
-			thumbnailMetadata.ThumbnailSize.ResizeMethod,
-		)
-		return err
-	})
+func (s *thumbnailStatements) InsertThumbnail(ctx context.Context, txn *sql.Tx, thumbnailMetadata *types.ThumbnailMetadata) error {
+	thumbnailMetadata.MediaMetadata.CreationTimestamp = gomatrixserverlib.AsTimestamp(time.Now())
+	_, err := sqlutil.TxStmtContext(ctx, txn, s.insertThumbnailStmt).ExecContext(
+		ctx,
+		thumbnailMetadata.MediaMetadata.MediaID,
+		thumbnailMetadata.MediaMetadata.Origin,
+		thumbnailMetadata.MediaMetadata.ContentType,
+		thumbnailMetadata.MediaMetadata.FileSizeBytes,
+		thumbnailMetadata.MediaMetadata.CreationTimestamp,
+		thumbnailMetadata.ThumbnailSize.Width,
+		thumbnailMetadata.ThumbnailSize.Height,
+		thumbnailMetadata.ThumbnailSize.ResizeMethod,
+	)
+	return err
 }
 
-func (s *thumbnailStatements) selectThumbnail(
+func (s *thumbnailStatements) SelectThumbnail(
 	ctx context.Context,
+	txn *sql.Tx,
 	mediaID types.MediaID,
 	mediaOrigin gomatrixserverlib.ServerName,
 	width, height int,
@@ -119,7 +113,7 @@ func (s *thumbnailStatements) selectThumbnail(
 			ResizeMethod: resizeMethod,
 		},
 	}
-	err := s.selectThumbnailStmt.QueryRowContext(
+	err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailStmt).QueryRowContext(
 		ctx,
 		thumbnailMetadata.MediaMetadata.MediaID,
 		thumbnailMetadata.MediaMetadata.Origin,
@@ -134,10 +128,11 @@ func (s *thumbnailStatements) selectThumbnail(
 	return &thumbnailMetadata, err
 }
 
-func (s *thumbnailStatements) selectThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+func (s *thumbnailStatements) SelectThumbnails(
+	ctx context.Context, txn *sql.Tx, mediaID types.MediaID,
+	mediaOrigin gomatrixserverlib.ServerName,
 ) ([]*types.ThumbnailMetadata, error) {
-	rows, err := s.selectThumbnailsStmt.QueryContext(
+	rows, err := sqlutil.TxStmtContext(ctx, txn, s.selectThumbnailsStmt).QueryContext(
 		ctx, mediaID, mediaOrigin,
 	)
 	if err != nil {
diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go
index 56059f1c8..baa242e57 100644
--- a/mediaapi/storage/storage.go
+++ b/mediaapi/storage/storage.go
@@ -25,13 +25,13 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-// Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+// NewMediaAPIDatasource opens a database connection.
+func NewMediaAPIDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.Open(dbProperties)
+		return sqlite3.NewDatabase(dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.Open(dbProperties)
+		return postgres.NewDatabase(dbProperties)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/mediaapi/storage/storage_test.go b/mediaapi/storage/storage_test.go
new file mode 100644
index 000000000..8d3403045
--- /dev/null
+++ b/mediaapi/storage/storage_test.go
@@ -0,0 +1,135 @@
+package storage_test
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/dendrite/mediaapi/storage"
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+)
+
+func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	})
+	if err != nil {
+		t.Fatalf("NewSyncServerDatasource returned %s", err)
+	}
+	return db, close
+}
+func TestMediaRepository(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		ctx := context.Background()
+		t.Run("can insert media & query media", func(t *testing.T) {
+			metadata := &types.MediaMetadata{
+				MediaID:       "testing",
+				Origin:        "localhost",
+				ContentType:   "image/png",
+				FileSizeBytes: 10,
+				UploadName:    "upload test",
+				Base64Hash:    "dGVzdGluZw==",
+				UserID:        "@alice:localhost",
+			}
+			if err := db.StoreMediaMetadata(ctx, metadata); err != nil {
+				t.Fatalf("unable to store media metadata: %v", err)
+			}
+			// query by media id
+			gotMetadata, err := db.GetMediaMetadata(ctx, metadata.MediaID, metadata.Origin)
+			if err != nil {
+				t.Fatalf("unable to query media metadata: %v", err)
+			}
+			if !reflect.DeepEqual(metadata, gotMetadata) {
+				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
+			}
+			// query by media hash
+			gotMetadata, err = db.GetMediaMetadataByHash(ctx, metadata.Base64Hash, metadata.Origin)
+			if err != nil {
+				t.Fatalf("unable to query media metadata by hash: %v", err)
+			}
+			if !reflect.DeepEqual(metadata, gotMetadata) {
+				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
+			}
+		})
+	})
+}
+
+func TestThumbnailsStorage(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		ctx := context.Background()
+		t.Run("can insert thumbnails & query media", func(t *testing.T) {
+			thumbnails := []*types.ThumbnailMetadata{
+				{
+					MediaMetadata: &types.MediaMetadata{
+						MediaID:       "testing",
+						Origin:        "localhost",
+						ContentType:   "image/png",
+						FileSizeBytes: 6,
+					},
+					ThumbnailSize: types.ThumbnailSize{
+						Width:        5,
+						Height:       5,
+						ResizeMethod: types.Crop,
+					},
+				},
+				{
+					MediaMetadata: &types.MediaMetadata{
+						MediaID:       "testing",
+						Origin:        "localhost",
+						ContentType:   "image/png",
+						FileSizeBytes: 7,
+					},
+					ThumbnailSize: types.ThumbnailSize{
+						Width:        1,
+						Height:       1,
+						ResizeMethod: types.Scale,
+					},
+				},
+			}
+			for i := range thumbnails {
+				if err := db.StoreThumbnail(ctx, thumbnails[i]); err != nil {
+					t.Fatalf("unable to store thumbnail metadata: %v", err)
+				}
+			}
+			// query by single thumbnail
+			gotMetadata, err := db.GetThumbnail(ctx,
+				thumbnails[0].MediaMetadata.MediaID,
+				thumbnails[0].MediaMetadata.Origin,
+				thumbnails[0].ThumbnailSize.Width, thumbnails[0].ThumbnailSize.Height,
+				thumbnails[0].ThumbnailSize.ResizeMethod,
+			)
+			if err != nil {
+				t.Fatalf("unable to query thumbnail metadata: %v", err)
+			}
+			if !reflect.DeepEqual(thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata) {
+				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
+			}
+			if !reflect.DeepEqual(thumbnails[0].ThumbnailSize, gotMetadata.ThumbnailSize) {
+				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
+			}
+			// query by all thumbnails
+			gotMediadatas, err := db.GetThumbnails(ctx, thumbnails[0].MediaMetadata.MediaID, thumbnails[0].MediaMetadata.Origin)
+			if err != nil {
+				t.Fatalf("unable to query media metadata by hash: %v", err)
+			}
+			if len(gotMediadatas) != len(thumbnails) {
+				t.Fatalf("expected %d stored thumbnail metadata, got %d", len(thumbnails), len(gotMediadatas))
+			}
+			for i := range gotMediadatas {
+				if !reflect.DeepEqual(thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata) {
+					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata)
+				}
+				if !reflect.DeepEqual(thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize) {
+					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize)
+				}
+			}
+		})
+	})
+}
diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go
index a6e997b2a..f67f9d5e1 100644
--- a/mediaapi/storage/storage_wasm.go
+++ b/mediaapi/storage/storage_wasm.go
@@ -22,10 +22,10 @@ import (
 )
 
 // Open opens a postgres database.
-func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewMediaAPIDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.Open(dbProperties)
+		return sqlite3.NewDatabase(dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/mediaapi/storage/tables/interface.go b/mediaapi/storage/tables/interface.go
new file mode 100644
index 000000000..bf63bc6ab
--- /dev/null
+++ b/mediaapi/storage/tables/interface.go
@@ -0,0 +1,46 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Thumbnails interface {
+	InsertThumbnail(ctx context.Context, txn *sql.Tx, thumbnailMetadata *types.ThumbnailMetadata) error
+	SelectThumbnail(
+		ctx context.Context, txn *sql.Tx,
+		mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+		width, height int,
+		resizeMethod string,
+	) (*types.ThumbnailMetadata, error)
+	SelectThumbnails(
+		ctx context.Context, txn *sql.Tx, mediaID types.MediaID,
+		mediaOrigin gomatrixserverlib.ServerName,
+	) ([]*types.ThumbnailMetadata, error)
+}
+
+type MediaRepository interface {
+	InsertMedia(ctx context.Context, txn *sql.Tx, mediaMetadata *types.MediaMetadata) error
+	SelectMedia(ctx context.Context, txn *sql.Tx, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error)
+	SelectMediaByHash(
+		ctx context.Context, txn *sql.Tx,
+		mediaHash types.Base64Hash, mediaOrigin gomatrixserverlib.ServerName,
+	) (*types.MediaMetadata, error)
+}
diff --git a/mediaapi/types/types.go b/mediaapi/types/types.go
index 0ba7010ad..ab28b3410 100644
--- a/mediaapi/types/types.go
+++ b/mediaapi/types/types.go
@@ -45,16 +45,13 @@ type RequestMethod string
 // MatrixUserID is a Matrix user ID string in the form @user:domain e.g. @alice:matrix.org
 type MatrixUserID string
 
-// UnixMs is the milliseconds since the Unix epoch
-type UnixMs int64
-
 // MediaMetadata is metadata associated with a media file
 type MediaMetadata struct {
 	MediaID           MediaID
 	Origin            gomatrixserverlib.ServerName
 	ContentType       ContentType
 	FileSizeBytes     FileSizeBytes
-	CreationTimestamp UnixMs
+	CreationTimestamp gomatrixserverlib.Timestamp
 	UploadName        Filename
 	Base64Hash        Base64Hash
 	UserID            MatrixUserID

From 57e3622b85fd4d80d9826404135f09e91ed47973 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 19 Apr 2022 10:46:45 +0200
Subject: [PATCH 007/391] Implement lazy loading on `/sync` (#2346)

* Initial work on lazyloading

* Partially implement lazy loading on /sync

* Rename methods

* Make missing tests pass

* Preallocate slice, even if it will end up with fewer values

* Let the cache handle the user mapping

* Linter

* Cap cache growth
---
 internal/caching/cache_lazy_load_members.go |  86 +++++++++++++++++
 syncapi/streams/stream_pdu.go               | 100 +++++++++++++++++++-
 syncapi/streams/streams.go                  |   4 +-
 syncapi/syncapi.go                          |   6 +-
 sytest-whitelist                            |  11 ++-
 5 files changed, 200 insertions(+), 7 deletions(-)
 create mode 100644 internal/caching/cache_lazy_load_members.go

diff --git a/internal/caching/cache_lazy_load_members.go b/internal/caching/cache_lazy_load_members.go
new file mode 100644
index 000000000..71a317624
--- /dev/null
+++ b/internal/caching/cache_lazy_load_members.go
@@ -0,0 +1,86 @@
+package caching
+
+import (
+	"fmt"
+	"time"
+
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+const (
+	LazyLoadCacheName           = "lazy_load_members"
+	LazyLoadCacheMaxEntries     = 128
+	LazyLoadCacheMaxUserEntries = 128
+	LazyLoadCacheMutable        = true
+	LazyLoadCacheMaxAge         = time.Minute * 30
+)
+
+type LazyLoadCache struct {
+	// InMemoryLRUCachePartition containing other InMemoryLRUCachePartitions
+	// with the actual cached members
+	userCaches *InMemoryLRUCachePartition
+}
+
+// NewLazyLoadCache creates a new LazyLoadCache.
+func NewLazyLoadCache() (*LazyLoadCache, error) {
+	cache, err := NewInMemoryLRUCachePartition(
+		LazyLoadCacheName,
+		LazyLoadCacheMutable,
+		LazyLoadCacheMaxEntries,
+		LazyLoadCacheMaxAge,
+		true,
+	)
+	if err != nil {
+		return nil, err
+	}
+	go cacheCleaner(cache)
+	return &LazyLoadCache{
+		userCaches: cache,
+	}, nil
+}
+
+func (c *LazyLoadCache) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) {
+	cacheName := fmt.Sprintf("%s/%s", device.UserID, device.ID)
+	userCache, ok := c.userCaches.Get(cacheName)
+	if ok && userCache != nil {
+		if cache, ok := userCache.(*InMemoryLRUCachePartition); ok {
+			return cache, nil
+		}
+	}
+	cache, err := NewInMemoryLRUCachePartition(
+		LazyLoadCacheName,
+		LazyLoadCacheMutable,
+		LazyLoadCacheMaxUserEntries,
+		LazyLoadCacheMaxAge,
+		false,
+	)
+	if err != nil {
+		return nil, err
+	}
+	c.userCaches.Set(cacheName, cache)
+	go cacheCleaner(cache)
+	return cache, nil
+}
+
+func (c *LazyLoadCache) StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) {
+	cache, err := c.lazyLoadCacheForUser(device)
+	if err != nil {
+		return
+	}
+	cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID)
+	cache.Set(cacheKey, eventID)
+}
+
+func (c *LazyLoadCache) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool) {
+	cache, err := c.lazyLoadCacheForUser(device)
+	if err != nil {
+		return "", false
+	}
+
+	cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID)
+	val, ok := cache.Get(cacheKey)
+	if !ok {
+		return "", ok
+	}
+	return val.(string), ok
+}
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index bcaf6ca31..ddc2f55c2 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -6,6 +6,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -26,7 +27,8 @@ type PDUStreamProvider struct {
 
 	tasks   chan func()
 	workers atomic.Int32
-	userAPI userapi.UserInternalAPI
+	// userID+deviceID -> lazy loading cache
+	lazyLoadCache *caching.LazyLoadCache
 }
 
 func (p *PDUStreamProvider) worker() {
@@ -188,7 +190,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 	newPos = from
 	for _, delta := range stateDeltas {
 		var pos types.StreamPosition
-		if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, req.Response); err != nil {
+		if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, &stateFilter, req.Response); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
 			return to
 		}
@@ -209,6 +211,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	r types.Range,
 	delta types.StateDelta,
 	eventFilter *gomatrixserverlib.RoomEventFilter,
+	stateFilter *gomatrixserverlib.StateFilter,
 	res *types.Response,
 ) (types.StreamPosition, error) {
 	if delta.MembershipPos > 0 && delta.Membership == gomatrixserverlib.Leave {
@@ -247,7 +250,8 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	// room that were returned.
 	latestPosition := r.To
 	updateLatestPosition := func(mostRecentEventID string) {
-		if _, pos, err := p.DB.PositionInTopology(ctx, mostRecentEventID); err == nil {
+		var pos types.StreamPosition
+		if _, pos, err = p.DB.PositionInTopology(ctx, mostRecentEventID); err == nil {
 			switch {
 			case r.Backwards && pos > latestPosition:
 				fallthrough
@@ -263,6 +267,19 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID())
 	}
 
+	if stateFilter.LazyLoadMembers {
+		if err != nil {
+			return r.From, err
+		}
+		delta.StateEvents, err = p.lazyLoadMembers(
+			ctx, delta.RoomID, true, limited, stateFilter.IncludeRedundantMembers,
+			device, recentEvents, delta.StateEvents,
+		)
+		if err != nil {
+			return r.From, err
+		}
+	}
+
 	hasMembershipChange := false
 	for _, recentEvent := range recentStreamEvents {
 		if recentEvent.Type() == gomatrixserverlib.MRoomMember && recentEvent.StateKey() != nil {
@@ -402,6 +419,20 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	// "Can sync a room with a message with a transaction id" - which does a complete sync to check.
 	recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents)
 	stateEvents = removeDuplicates(stateEvents, recentEvents)
+
+	if stateFilter.LazyLoadMembers {
+		if err != nil {
+			return nil, err
+		}
+		stateEvents, err = p.lazyLoadMembers(ctx, roomID,
+			false, limited, stateFilter.IncludeRedundantMembers,
+			device, recentEvents, stateEvents,
+		)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	jr = types.NewJoinResponse()
 	jr.Summary.JoinedMemberCount = &joinedCount
 	jr.Summary.InvitedMemberCount = &invitedCount
@@ -412,6 +443,69 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	return jr, nil
 }
 
+func (p *PDUStreamProvider) lazyLoadMembers(
+	ctx context.Context, roomID string,
+	incremental, limited, includeRedundant bool,
+	device *userapi.Device,
+	timelineEvents, stateEvents []*gomatrixserverlib.HeaderedEvent,
+) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	if len(timelineEvents) == 0 {
+		return stateEvents, nil
+	}
+	// Work out which memberships to include
+	timelineUsers := make(map[string]struct{})
+	if !incremental {
+		timelineUsers[device.UserID] = struct{}{}
+	}
+	// Add all users the client doesn't know about yet to a list
+	for _, event := range timelineEvents {
+		// Membership is not yet cached, add it to the list
+		if _, ok := p.lazyLoadCache.IsLazyLoadedUserCached(device, roomID, event.Sender()); !ok {
+			timelineUsers[event.Sender()] = struct{}{}
+		}
+	}
+	// Preallocate with the same amount, even if it will end up with fewer values
+	newStateEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(stateEvents))
+	// Remove existing membership events we don't care about, e.g. users not in the timeline.events
+	for _, event := range stateEvents {
+		if event.Type() == gomatrixserverlib.MRoomMember && event.StateKey() != nil {
+			// If this is a gapped incremental sync, we still want this membership
+			isGappedIncremental := limited && incremental
+			// We want this users membership event, keep it in the list
+			_, ok := timelineUsers[event.Sender()]
+			wantMembership := ok || isGappedIncremental
+			if wantMembership {
+				newStateEvents = append(newStateEvents, event)
+				if !includeRedundant {
+					p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, event.Sender(), event.EventID())
+				}
+				delete(timelineUsers, event.Sender())
+			}
+		} else {
+			newStateEvents = append(newStateEvents, event)
+		}
+	}
+	wantUsers := make([]string, 0, len(timelineUsers))
+	for userID := range timelineUsers {
+		wantUsers = append(wantUsers, userID)
+	}
+	// Query missing membership events
+	memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &gomatrixserverlib.StateFilter{
+		Limit:   100,
+		Senders: &wantUsers,
+		Types:   &[]string{gomatrixserverlib.MRoomMember},
+	})
+	if err != nil {
+		return stateEvents, err
+	}
+	// cache the membership events
+	for _, membership := range memberships {
+		p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, membership.Sender(), membership.EventID())
+	}
+	stateEvents = append(newStateEvents, memberships...)
+	return stateEvents, nil
+}
+
 // addIgnoredUsersToFilter adds ignored users to the eventfilter and
 // the syncreq itself for further use in streams.
 func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, req *types.SyncRequest, eventFilter *gomatrixserverlib.RoomEventFilter) error {
diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go
index c7d06a296..d3195b78f 100644
--- a/syncapi/streams/streams.go
+++ b/syncapi/streams/streams.go
@@ -27,12 +27,12 @@ type Streams struct {
 func NewSyncStreamProviders(
 	d storage.Database, userAPI userapi.UserInternalAPI,
 	rsAPI rsapi.RoomserverInternalAPI, keyAPI keyapi.KeyInternalAPI,
-	eduCache *caching.EDUCache, notifier *notifier.Notifier,
+	eduCache *caching.EDUCache, lazyLoadCache *caching.LazyLoadCache, notifier *notifier.Notifier,
 ) *Streams {
 	streams := &Streams{
 		PDUStreamProvider: &PDUStreamProvider{
 			StreamProvider: StreamProvider{DB: d},
-			userAPI:        userAPI,
+			lazyLoadCache:  lazyLoadCache,
 		},
 		TypingStreamProvider: &TypingStreamProvider{
 			StreamProvider: StreamProvider{DB: d},
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index 384121a8a..2f9165d91 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -57,8 +57,12 @@ func AddPublicRoutes(
 	}
 
 	eduCache := caching.NewTypingCache()
+	lazyLoadCache, err := caching.NewLazyLoadCache()
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to create lazy loading cache")
+	}
 	notifier := notifier.NewNotifier()
-	streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, notifier)
+	streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, lazyLoadCache, notifier)
 	notifier.SetCurrentPosition(streams.Latest(context.Background()))
 	if err = notifier.Load(context.Background(), syncDB); err != nil {
 		logrus.WithError(err).Panicf("failed to load notifier ")
diff --git a/sytest-whitelist b/sytest-whitelist
index f63b96f52..c8dedd59c 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -699,4 +699,13 @@ Ignore invite in full sync
 Ignore invite in incremental sync
 A filtered timeline reaches its limit
 A change to displayname should not result in a full state sync
-Can fetch images in room
\ No newline at end of file
+Can fetch images in room
+The only membership state included in an initial sync is for all the senders in the timeline
+The only membership state included in an incremental sync is for senders in the timeline
+Old members are included in gappy incr LL sync if they start speaking
+We do send redundant membership state across incremental syncs if asked
+Rejecting invite over federation doesn't break incremental /sync
+Gapped incremental syncs include all state changes
+Old leaves are present in gapped incremental syncs
+Leaves are present in non-gapped incremental syncs
+Members from the gap are included in gappy incr LL sync
\ No newline at end of file

From abf71649b0a27ddc564105f94db270696698ecd5 Mon Sep 17 00:00:00 2001
From: fcwoknhenuxdfiyv-nextcloud
 <84577563+fcwoknhenuxdfiyv@users.noreply.github.com>
Date: Tue, 19 Apr 2022 10:46:54 +0200
Subject: [PATCH 008/391] Make sure resp.Username is defined before hashing.
 Fixes #2356 (#2357)

Co-authored-by: Jason Quigley <jason@onecha.net>
---
 clientapi/routing/voip.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go
index 13dca7ac0..c7ddaabcf 100644
--- a/clientapi/routing/voip.go
+++ b/clientapi/routing/voip.go
@@ -52,6 +52,7 @@ func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Client
 
 	if turnConfig.SharedSecret != "" {
 		expiry := time.Now().Add(duration).Unix()
+		resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID)
 		mac := hmac.New(sha1.New, []byte(turnConfig.SharedSecret))
 		_, err := mac.Write([]byte(resp.Username))
 
@@ -60,7 +61,6 @@ func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Client
 			return jsonerror.InternalServerError()
 		}
 
-		resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID)
 		resp.Password = base64.StdEncoding.EncodeToString(mac.Sum(nil))
 	} else if turnConfig.Username != "" && turnConfig.Password != "" {
 		resp.Username = turnConfig.Username

From 7e745665a47058209f7f1fcf51c50ff91c43c7c4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 19 Apr 2022 09:51:02 +0100
Subject: [PATCH 009/391] Change `pushkey_ts` to be seconds (fix #2354) (#2358)

---
 internal/pushgateway/pushgateway.go      | 12 +++++-------
 userapi/api/api.go                       | 20 ++++++++++----------
 userapi/internal/api.go                  |  2 +-
 userapi/storage/postgres/pusher_table.go |  3 +--
 userapi/storage/sqlite3/pusher_table.go  |  3 +--
 userapi/storage/tables/interface.go      |  3 +--
 6 files changed, 19 insertions(+), 24 deletions(-)

diff --git a/internal/pushgateway/pushgateway.go b/internal/pushgateway/pushgateway.go
index 88c326eb2..1817a040b 100644
--- a/internal/pushgateway/pushgateway.go
+++ b/internal/pushgateway/pushgateway.go
@@ -3,8 +3,6 @@ package pushgateway
 import (
 	"context"
 	"encoding/json"
-
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // A Client is how interactions with a Push Gateway is done.
@@ -47,11 +45,11 @@ type Counts struct {
 }
 
 type Device struct {
-	AppID     string                      `json:"app_id"`  // Required
-	Data      map[string]interface{}      `json:"data"`    // Required. UNSPEC: Sytests require this to allow unknown keys.
-	PushKey   string                      `json:"pushkey"` // Required
-	PushKeyTS gomatrixserverlib.Timestamp `json:"pushkey_ts,omitempty"`
-	Tweaks    map[string]interface{}      `json:"tweaks,omitempty"`
+	AppID     string                 `json:"app_id"`  // Required
+	Data      map[string]interface{} `json:"data"`    // Required. UNSPEC: Sytests require this to allow unknown keys.
+	PushKey   string                 `json:"pushkey"` // Required
+	PushKeyTS int64                  `json:"pushkey_ts,omitempty"`
+	Tweaks    map[string]interface{} `json:"tweaks,omitempty"`
 }
 
 type Prio string
diff --git a/userapi/api/api.go b/userapi/api/api.go
index b86774d14..6aa6a6842 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -492,16 +492,16 @@ type PerformPusherDeletionRequest struct {
 
 // Pusher represents a push notification subscriber
 type Pusher struct {
-	SessionID         int64                       `json:"session_id,omitempty"`
-	PushKey           string                      `json:"pushkey"`
-	PushKeyTS         gomatrixserverlib.Timestamp `json:"pushkey_ts,omitempty"`
-	Kind              PusherKind                  `json:"kind"`
-	AppID             string                      `json:"app_id"`
-	AppDisplayName    string                      `json:"app_display_name"`
-	DeviceDisplayName string                      `json:"device_display_name"`
-	ProfileTag        string                      `json:"profile_tag"`
-	Language          string                      `json:"lang"`
-	Data              map[string]interface{}      `json:"data"`
+	SessionID         int64                  `json:"session_id,omitempty"`
+	PushKey           string                 `json:"pushkey"`
+	PushKeyTS         int64                  `json:"pushkey_ts,omitempty"`
+	Kind              PusherKind             `json:"kind"`
+	AppID             string                 `json:"app_id"`
+	AppDisplayName    string                 `json:"app_display_name"`
+	DeviceDisplayName string                 `json:"device_display_name"`
+	ProfileTag        string                 `json:"profile_tag"`
+	Language          string                 `json:"lang"`
+	Data              map[string]interface{} `json:"data"`
 }
 
 type PusherKind string
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 206c6f7de..6a16ea686 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -653,7 +653,7 @@ func (a *UserInternalAPI) PerformPusherSet(ctx context.Context, req *api.Perform
 		return a.DB.RemovePusher(ctx, req.Pusher.AppID, req.Pusher.PushKey, req.Localpart)
 	}
 	if req.Pusher.PushKeyTS == 0 {
-		req.Pusher.PushKeyTS = gomatrixserverlib.AsTimestamp(time.Now())
+		req.Pusher.PushKeyTS = int64(time.Now().Second())
 	}
 	return a.DB.UpsertPusher(ctx, req.Pusher, req.Localpart)
 }
diff --git a/userapi/storage/postgres/pusher_table.go b/userapi/storage/postgres/pusher_table.go
index 670dc916f..2eb379ae4 100644
--- a/userapi/storage/postgres/pusher_table.go
+++ b/userapi/storage/postgres/pusher_table.go
@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
 
@@ -95,7 +94,7 @@ type pushersStatements struct {
 // Returns nil error success.
 func (s *pushersStatements) InsertPusher(
 	ctx context.Context, txn *sql.Tx, session_id int64,
-	pushkey string, pushkeyTS gomatrixserverlib.Timestamp, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string,
+	pushkey string, pushkeyTS int64, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string,
 ) error {
 	_, err := sqlutil.TxStmt(txn, s.insertPusherStmt).ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data)
 	logrus.Debugf("Created pusher %d", session_id)
diff --git a/userapi/storage/sqlite3/pusher_table.go b/userapi/storage/sqlite3/pusher_table.go
index e718792e1..d5bd1617b 100644
--- a/userapi/storage/sqlite3/pusher_table.go
+++ b/userapi/storage/sqlite3/pusher_table.go
@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
 
@@ -95,7 +94,7 @@ type pushersStatements struct {
 // Returns nil error success.
 func (s *pushersStatements) InsertPusher(
 	ctx context.Context, txn *sql.Tx, session_id int64,
-	pushkey string, pushkeyTS gomatrixserverlib.Timestamp, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string,
+	pushkey string, pushkeyTS int64, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string,
 ) error {
 	_, err := s.insertPusherStmt.ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data)
 	logrus.Debugf("Created pusher %d", session_id)
diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go
index 99c907b85..eb0cae314 100644
--- a/userapi/storage/tables/interface.go
+++ b/userapi/storage/tables/interface.go
@@ -21,7 +21,6 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type AccountDataTable interface {
@@ -96,7 +95,7 @@ type ThreePIDTable interface {
 }
 
 type PusherTable interface {
-	InsertPusher(ctx context.Context, txn *sql.Tx, session_id int64, pushkey string, pushkeyTS gomatrixserverlib.Timestamp, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string) error
+	InsertPusher(ctx context.Context, txn *sql.Tx, session_id int64, pushkey string, pushkeyTS int64, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string) error
 	SelectPushers(ctx context.Context, txn *sql.Tx, localpart string) ([]api.Pusher, error)
 	DeletePusher(ctx context.Context, txn *sql.Tx, appid, pushkey, localpart string) error
 	DeletePushers(ctx context.Context, txn *sql.Tx, appid, pushkey string) error

From 711e377b9cb066c42c58f31b155aca194ae9928a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 19 Apr 2022 10:34:33 +0100
Subject: [PATCH 010/391] Update `go-sqlite3-js` to matrix-org/go-sqlite3-js#2
 (SQLite 3.36.0)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 070f2b5b9..e3ac6a171 100644
--- a/go.mod
+++ b/go.mod
@@ -36,7 +36,7 @@ require (
 	github.com/libp2p/go-libp2p-record v0.1.3
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
-	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
+	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
diff --git a/go.sum b/go.sum
index 464f774be..e8b0a216b 100644
--- a/go.sum
+++ b/go.sum
@@ -1107,8 +1107,8 @@ github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBE
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4=
-github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d h1:mGhPVaTht5NViFN/UpdrIlRApmH2FWcVaKUH5MdBKiY=
-github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
+github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA=
+github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=

From 85b1631ecfbeaa382c4bd17e7e310e668f5400dd Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 19 Apr 2022 10:48:32 +0100
Subject: [PATCH 011/391] Add newly passing test to list

---
 sytest-whitelist | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sytest-whitelist b/sytest-whitelist
index c8dedd59c..aef18c512 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -708,4 +708,5 @@ Rejecting invite over federation doesn't break incremental /sync
 Gapped incremental syncs include all state changes
 Old leaves are present in gapped incremental syncs
 Leaves are present in non-gapped incremental syncs
-Members from the gap are included in gappy incr LL sync
\ No newline at end of file
+Members from the gap are included in gappy incr LL sync
+Presence can be set from sync
\ No newline at end of file

From 073972646fd38b6987c2b62f8db5500c2f87d2d0 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 19 Apr 2022 13:57:02 +0100
Subject: [PATCH 012/391] Use unix not second

---
 userapi/internal/api.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 6a16ea686..d1c12f05f 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -653,7 +653,7 @@ func (a *UserInternalAPI) PerformPusherSet(ctx context.Context, req *api.Perform
 		return a.DB.RemovePusher(ctx, req.Pusher.AppID, req.Pusher.PushKey, req.Localpart)
 	}
 	if req.Pusher.PushKeyTS == 0 {
-		req.Pusher.PushKeyTS = int64(time.Now().Second())
+		req.Pusher.PushKeyTS = int64(time.Now().Unix())
 	}
 	return a.DB.UpsertPusher(ctx, req.Pusher, req.Localpart)
 }

From a9f0a390c6e6efed9c09c72839960f57b5f3e5f2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 20 Apr 2022 15:13:04 +0100
Subject: [PATCH 013/391] Update to NATS Server v2.8.0 and nats.go v1.14.0
 (#2359)

---
 go.mod |  6 +++---
 go.sum | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index e3ac6a171..a9774c2c8 100644
--- a/go.mod
+++ b/go.mod
@@ -1,8 +1,8 @@
 module github.com/matrix-org/dendrite
 
-replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e
+replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9
 
-replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c
+replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
 
 require (
 	github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0
@@ -43,7 +43,7 @@ require (
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
 	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
-	github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d
+	github.com/nats-io/nats.go v1.14.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
diff --git a/go.sum b/go.sum
index e8b0a216b..ee4096fdc 100644
--- a/go.sum
+++ b/go.sum
@@ -1266,8 +1266,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 h1:vU9tpM3apjYlLLeY23zRWJ9Zktr5jp+mloR942LEOpY=
-github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
+github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a h1:lem6QCvxR0Y28gth9P+wV2K/zYUUAkJ+55U8cpS0p5I=
+github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
 github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -1276,10 +1276,10 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e h1:5tEHLzvDeS6IeqO2o9FFhsE3V2erYj8FlMt2J91wzsk=
-github.com/neilalexander/nats-server/v2 v2.7.5-0.20220311134712-e2e4a244f30e/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc=
-github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c h1:G2qsv7D0rY94HAu8pXmElMluuMHQ85waxIDQBhIzV2Q=
-github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9 h1:VGU5HYAwy8LRbSkrT+kCHvujVmwK8Aa/vc1O+eReTbM=
+github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9/go.mod h1:5vic7C58BFEVltiZhs7Kq81q2WcEPhJPsmNv1FOrdv0=
+github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y=
+github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=

From bb987cd64b118044a4f3351c377516813514ee19 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 20 Apr 2022 16:06:46 +0100
Subject: [PATCH 014/391] Lazy loading fixes (#2362)

* Return some more usefully wrapped errors when doing sync

* Remove unnecessary error check

* Couple of guards around `sql.ErrNoRows`

* Nolint
---
 syncapi/streams/stream_pdu.go | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index ddc2f55c2..0f11d55f6 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -3,6 +3,7 @@ package streams
 import (
 	"context"
 	"database/sql"
+	"fmt"
 	"sync"
 	"time"
 
@@ -205,6 +206,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 	return newPos
 }
 
+// nolint:gocyclo
 func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	ctx context.Context,
 	device *userapi.Device,
@@ -228,13 +230,16 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		eventFilter, true, true,
 	)
 	if err != nil {
-		return r.From, err
+		if err == sql.ErrNoRows {
+			return r.To, nil
+		}
+		return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err)
 	}
 	recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents)
 	delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back
 	prevBatch, err := p.DB.GetBackwardTopologyPos(ctx, recentStreamEvents)
 	if err != nil {
-		return r.From, err
+		return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err)
 	}
 
 	// If we didn't return any events at all then don't bother doing anything else.
@@ -268,15 +273,12 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	}
 
 	if stateFilter.LazyLoadMembers {
-		if err != nil {
-			return r.From, err
-		}
 		delta.StateEvents, err = p.lazyLoadMembers(
 			ctx, delta.RoomID, true, limited, stateFilter.IncludeRedundantMembers,
 			device, recentEvents, delta.StateEvents,
 		)
-		if err != nil {
-			return r.From, err
+		if err != nil && err != sql.ErrNoRows {
+			return r.From, fmt.Errorf("p.lazyLoadMembers: %w", err)
 		}
 	}
 

From 54e7ea41c688a0bcc89150c99045b2dcdf8d0b12 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 20 Apr 2022 16:51:37 +0100
Subject: [PATCH 015/391] Eliminate more SQL no row errors in sync API (#2363)

* Handle `sql.ErrNoRows` in main `/sync` codepaths

* Catch more
---
 syncapi/storage/shared/syncserver.go | 34 ++++++++++++++++++++++++++--
 syncapi/streams/stream_pdu.go        |  7 ++++--
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 91eba44e1..2143fd672 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -688,6 +688,9 @@ func (d *Database) GetStateDeltas(
 	// user has ever interacted with — joined to, kicked/banned from, left.
 	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
 		return nil, nil, err
 	}
 
@@ -705,17 +708,23 @@ func (d *Database) GetStateDeltas(
 	// get all the state events ever (i.e. for all available rooms) between these two positions
 	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
 		return nil, nil, err
 	}
 	state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
 		return nil, nil, err
 	}
 
 	// find out which rooms this user is peeking, if any.
 	// We do this before joins so any peeks get overwritten
 	peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r)
-	if err != nil {
+	if err != nil && err != sql.ErrNoRows {
 		return nil, nil, err
 	}
 
@@ -726,6 +735,9 @@ func (d *Database) GetStateDeltas(
 			var s []types.StreamEvent
 			s, err = d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter)
 			if err != nil {
+				if err == sql.ErrNoRows {
+					continue
+				}
 				return nil, nil, err
 			}
 			state[peek.RoomID] = s
@@ -753,6 +765,9 @@ func (d *Database) GetStateDeltas(
 					var s []types.StreamEvent
 					s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter)
 					if err != nil {
+						if err == sql.ErrNoRows {
+							continue
+						}
 						return nil, nil, err
 					}
 					state[roomID] = s
@@ -803,6 +818,9 @@ func (d *Database) GetStateDeltasForFullStateSync(
 	// user has ever interacted with — joined to, kicked/banned from, left.
 	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
 		return nil, nil, err
 	}
 
@@ -819,7 +837,7 @@ func (d *Database) GetStateDeltasForFullStateSync(
 	deltas := make(map[string]types.StateDelta)
 
 	peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r)
-	if err != nil {
+	if err != nil && err != sql.ErrNoRows {
 		return nil, nil, err
 	}
 
@@ -828,6 +846,9 @@ func (d *Database) GetStateDeltasForFullStateSync(
 		if !peek.Deleted {
 			s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter)
 			if stateErr != nil {
+				if stateErr == sql.ErrNoRows {
+					continue
+				}
 				return nil, nil, stateErr
 			}
 			deltas[peek.RoomID] = types.StateDelta{
@@ -841,10 +862,16 @@ func (d *Database) GetStateDeltasForFullStateSync(
 	// Get all the state events ever between these two positions
 	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
 		return nil, nil, err
 	}
 	state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
 		return nil, nil, err
 	}
 
@@ -869,6 +896,9 @@ func (d *Database) GetStateDeltasForFullStateSync(
 	for _, joinedRoomID := range joinedRoomIDs {
 		s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter)
 		if stateErr != nil {
+			if stateErr == sql.ErrNoRows {
+				continue
+			}
 			return nil, nil, stateErr
 		}
 		deltas[joinedRoomID] = types.StateDelta{
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 0f11d55f6..df5fb8e08 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -341,12 +341,16 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	wantFullState bool,
 	device *userapi.Device,
 ) (jr *types.JoinResponse, err error) {
+	jr = types.NewJoinResponse()
 	// TODO: When filters are added, we may need to call this multiple times to get enough events.
 	//       See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316
 	recentStreamEvents, limited, err := p.DB.RecentEvents(
 		ctx, roomID, r, eventFilter, true, true,
 	)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return jr, nil
+		}
 		return
 	}
 
@@ -430,12 +434,11 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 			false, limited, stateFilter.IncludeRedundantMembers,
 			device, recentEvents, stateEvents,
 		)
-		if err != nil {
+		if err != nil && err != sql.ErrNoRows {
 			return nil, err
 		}
 	}
 
-	jr = types.NewJoinResponse()
 	jr.Summary.JoinedMemberCount = &joinedCount
 	jr.Summary.InvitedMemberCount = &invitedCount
 	jr.Timeline.PrevBatch = prevBatch

From 2258387d393426922344213948c6d814de53f465 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 20 Apr 2022 16:55:24 +0100
Subject: [PATCH 016/391] Update test list

---
 sytest-whitelist | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sytest-whitelist b/sytest-whitelist
index aef18c512..979f12bf6 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -312,10 +312,10 @@ Inbound federation can return events
 Inbound federation can return missing events for world_readable visibility
 Inbound federation can return missing events for invite visibility
 Inbound federation can get public room list
-POST /rooms/:room_id/redact/:event_id as power user redacts message
-POST /rooms/:room_id/redact/:event_id as original message sender redacts message
-POST /rooms/:room_id/redact/:event_id as random user does not redact message
-POST /redact disallows redaction of event in different room
+PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message
+PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message
+PUT /rooms/:room_id/redact/:event_id/:txn_id as random user does not redact message
+PUT /redact disallows redaction of event in different room
 An event which redacts itself should be ignored
 A pair of events which redact each other should be ignored
 Redaction of a redaction redacts the redaction reason

From 144c060fa7e413d938200896b345b85ab975bad8 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 21 Apr 2022 13:31:43 +0100
Subject: [PATCH 017/391] Remove libp2p demos (#2337)

* Remove libp2p demos

* Put it back
---
 cmd/dendrite-demo-libp2p/main.go         | 230 ------------
 cmd/dendrite-demo-libp2p/mdnslistener.go |  62 ----
 cmd/dendrite-demo-libp2p/p2pdendrite.go  | 126 -------
 cmd/dendrite-demo-libp2p/publicrooms.go  | 153 --------
 cmd/dendritejs/jsServer.go               | 101 ------
 cmd/dendritejs/keyfetcher.go             |  87 -----
 cmd/dendritejs/main.go                   | 270 --------------
 cmd/dendritejs/main_noop.go              |  24 --
 cmd/dendritejs/publicrooms.go            | 155 --------
 go.mod                                   |  10 +-
 go.sum                                   | 442 -----------------------
 11 files changed, 1 insertion(+), 1659 deletions(-)
 delete mode 100644 cmd/dendrite-demo-libp2p/main.go
 delete mode 100644 cmd/dendrite-demo-libp2p/mdnslistener.go
 delete mode 100644 cmd/dendrite-demo-libp2p/p2pdendrite.go
 delete mode 100644 cmd/dendrite-demo-libp2p/publicrooms.go
 delete mode 100644 cmd/dendritejs/jsServer.go
 delete mode 100644 cmd/dendritejs/keyfetcher.go
 delete mode 100644 cmd/dendritejs/main.go
 delete mode 100644 cmd/dendritejs/main_noop.go
 delete mode 100644 cmd/dendritejs/publicrooms.go

diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go
deleted file mode 100644
index 26c8eb85f..000000000
--- a/cmd/dendrite-demo-libp2p/main.go
+++ /dev/null
@@ -1,230 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"crypto/ed25519"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/gorilla/mux"
-	gostream "github.com/libp2p/go-libp2p-gostream"
-	p2phttp "github.com/libp2p/go-libp2p-http"
-	p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/mscs"
-	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/sirupsen/logrus"
-
-	_ "github.com/mattn/go-sqlite3"
-)
-
-func createKeyDB(
-	base *P2PDendrite,
-	db *gomatrixserverlib.KeyRing,
-) {
-	mdns := mDNSListener{
-		host:  base.LibP2P,
-		keydb: db,
-	}
-	serv, err := p2pdisc.NewMdnsService(
-		base.LibP2PContext,
-		base.LibP2P,
-		time.Second*10,
-		"_matrix-dendrite-p2p._tcp",
-	)
-	if err != nil {
-		panic(err)
-	}
-	serv.RegisterNotifee(&mdns)
-}
-
-func createFederationClient(
-	base *P2PDendrite,
-) *gomatrixserverlib.FederationClient {
-	fmt.Println("Running in libp2p federation mode")
-	fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!")
-	tr := &http.Transport{}
-	tr.RegisterProtocol(
-		"matrix",
-		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
-	)
-	return gomatrixserverlib.NewFederationClient(
-		base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID,
-		base.Base.Cfg.Global.PrivateKey,
-		gomatrixserverlib.WithTransport(tr),
-	)
-}
-
-func createClient(
-	base *P2PDendrite,
-) *gomatrixserverlib.Client {
-	tr := &http.Transport{}
-	tr.RegisterProtocol(
-		"matrix",
-		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
-	)
-	return gomatrixserverlib.NewClient(
-		gomatrixserverlib.WithTransport(tr),
-	)
-}
-
-func main() {
-	instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance")
-	instancePort := flag.Int("port", 8080, "the port that the client API will listen on")
-	flag.Parse()
-
-	filename := fmt.Sprintf("%s-private.key", *instanceName)
-	_, err := os.Stat(filename)
-	var privKey ed25519.PrivateKey
-	if os.IsNotExist(err) {
-		_, privKey, _ = ed25519.GenerateKey(nil)
-		if err = ioutil.WriteFile(filename, privKey, 0600); err != nil {
-			fmt.Printf("Couldn't write private key to file '%s': %s\n", filename, err)
-		}
-	} else {
-		privKey, err = ioutil.ReadFile(filename)
-		if err != nil {
-			fmt.Printf("Couldn't read private key from file '%s': %s\n", filename, err)
-			_, privKey, _ = ed25519.GenerateKey(nil)
-		}
-	}
-
-	cfg := config.Dendrite{}
-	cfg.Defaults(true)
-	cfg.Global.ServerName = "p2p"
-	cfg.Global.PrivateKey = privKey
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
-	cfg.FederationAPI.FederationMaxRetries = 6
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
-	cfg.MSCs.MSCs = []string{"msc2836"}
-	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
-	if err = cfg.Derive(); err != nil {
-		panic(err)
-	}
-
-	base := NewP2PDendrite(&cfg, "Monolith")
-	defer base.Base.Close() // nolint: errcheck
-
-	accountDB := base.Base.CreateAccountsDB()
-	federation := createFederationClient(base)
-	keyAPI := keyserver.NewInternalAPI(&base.Base, &base.Base.Cfg.KeyServer, federation)
-
-	rsAPI := roomserver.NewInternalAPI(
-		&base.Base,
-	)
-
-	userAPI := userapi.NewInternalAPI(&base.Base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.Base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
-
-	asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
-	rsAPI.SetAppserviceAPI(asAPI)
-	fsAPI := federationapi.NewInternalAPI(
-		&base.Base, federation, rsAPI, base.Base.Caches, nil, true,
-	)
-	keyRing := fsAPI.KeyRing()
-	rsAPI.SetFederationAPI(fsAPI, keyRing)
-	provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI)
-	err = provider.Start()
-	if err != nil {
-		panic("failed to create new public rooms provider: " + err.Error())
-	}
-
-	createKeyDB(
-		base, keyRing,
-	)
-
-	monolith := setup.Monolith{
-		Config:    base.Base.Cfg,
-		AccountDB: accountDB,
-		Client:    createClient(base),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI:          asAPI,
-		FederationAPI:          fsAPI,
-		RoomserverAPI:          rsAPI,
-		UserAPI:                userAPI,
-		KeyAPI:                 keyAPI,
-		ExtPublicRoomsProvider: provider,
-	}
-	monolith.AddAllPublicRoutes(
-		base.Base.ProcessContext,
-		base.Base.PublicClientAPIMux,
-		base.Base.PublicFederationAPIMux,
-		base.Base.PublicKeyAPIMux,
-		base.Base.PublicWellKnownAPIMux,
-		base.Base.PublicMediaAPIMux,
-		base.Base.SynapseAdminMux,
-	)
-	if err := mscs.Enable(&base.Base, &monolith); err != nil {
-		logrus.WithError(err).Fatalf("Failed to enable MSCs")
-	}
-
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.Base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux)
-	embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
-
-	libp2pRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	libp2pRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Base.PublicFederationAPIMux)
-	libp2pRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(base.Base.PublicKeyAPIMux)
-	libp2pRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux)
-
-	// Expose the matrix APIs directly rather than putting them under a /api path.
-	go func() {
-		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
-		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
-	}()
-	// Expose the matrix APIs also via libp2p
-	if base.LibP2P != nil {
-		go func() {
-			logrus.Info("Listening on libp2p host ID ", base.LibP2P.ID())
-			listener, err := gostream.Listen(base.LibP2P, "/matrix")
-			if err != nil {
-				panic(err)
-			}
-			defer func() {
-				logrus.Fatal(listener.Close())
-			}()
-			logrus.Fatal(http.Serve(listener, libp2pRouter))
-		}()
-	}
-
-	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.Base.WaitForShutdown()
-}
diff --git a/cmd/dendrite-demo-libp2p/mdnslistener.go b/cmd/dendrite-demo-libp2p/mdnslistener.go
deleted file mode 100644
index c6105e52c..000000000
--- a/cmd/dendrite-demo-libp2p/mdnslistener.go
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"math"
-
-	"github.com/libp2p/go-libp2p-core/host"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-type mDNSListener struct {
-	keydb *gomatrixserverlib.KeyRing
-	host  host.Host
-}
-
-func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) {
-	if err := n.host.Connect(context.Background(), p); err != nil {
-		fmt.Println("Error adding peer", p.ID.String(), "via mDNS:", err)
-	}
-	if pubkey, err := p.ID.ExtractPublicKey(); err == nil {
-		raw, _ := pubkey.Raw()
-		if err := n.keydb.KeyDatabase.StoreKeys(
-			context.Background(),
-			map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{
-				{
-					ServerName: gomatrixserverlib.ServerName(p.ID.String()),
-					KeyID:      "ed25519:p2pdemo",
-				}: {
-					VerifyKey: gomatrixserverlib.VerifyKey{
-						Key: gomatrixserverlib.Base64Bytes(raw),
-					},
-					ValidUntilTS: math.MaxUint64 >> 1,
-					ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
-				},
-			},
-		); err != nil {
-			fmt.Println("Failed to store keys:", err)
-		}
-	}
-	fmt.Println("Discovered", len(n.host.Peerstore().Peers())-1, "other libp2p peer(s):")
-	for _, peer := range n.host.Peerstore().Peers() {
-		if peer != n.host.ID() {
-			fmt.Println("-", peer)
-		}
-	}
-}
diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go
deleted file mode 100644
index ba1868b27..000000000
--- a/cmd/dendrite-demo-libp2p/p2pdendrite.go
+++ /dev/null
@@ -1,126 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"fmt"
-
-	"errors"
-
-	pstore "github.com/libp2p/go-libp2p-core/peerstore"
-	record "github.com/libp2p/go-libp2p-record"
-
-	"github.com/libp2p/go-libp2p"
-	circuit "github.com/libp2p/go-libp2p-circuit"
-	crypto "github.com/libp2p/go-libp2p-core/crypto"
-	routing "github.com/libp2p/go-libp2p-core/routing"
-
-	host "github.com/libp2p/go-libp2p-core/host"
-	dht "github.com/libp2p/go-libp2p-kad-dht"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-// P2PDendrite is a Peer-to-Peer variant of BaseDendrite.
-type P2PDendrite struct {
-	Base base.BaseDendrite
-
-	// Store our libp2p object so that we can make outgoing connections from it
-	// later
-	LibP2P        host.Host
-	LibP2PContext context.Context
-	LibP2PCancel  context.CancelFunc
-	LibP2PDHT     *dht.IpfsDHT
-	LibP2PPubsub  *pubsub.PubSub
-}
-
-// NewP2PDendrite creates a new instance to be used by a component.
-// The componentName is used for logging purposes, and should be a friendly name
-// of the component running, e.g. SyncAPI.
-func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
-	baseDendrite := base.NewBaseDendrite(cfg, componentName)
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Global.PrivateKey[:])
-	if err != nil {
-		panic(err)
-	}
-
-	//defaultIP6ListenAddr, _ := multiaddr.NewMultiaddr("/ip6/::/tcp/0")
-	var libp2pdht *dht.IpfsDHT
-	libp2p, err := libp2p.New(ctx,
-		libp2p.Identity(privKey),
-		libp2p.DefaultListenAddrs,
-		//libp2p.ListenAddrs(defaultIP6ListenAddr),
-		libp2p.DefaultTransports,
-		libp2p.Routing(func(h host.Host) (r routing.PeerRouting, err error) {
-			libp2pdht, err = dht.New(ctx, h)
-			if err != nil {
-				return nil, err
-			}
-			libp2pdht.Validator = libP2PValidator{}
-			r = libp2pdht
-			return
-		}),
-		libp2p.EnableAutoRelay(),
-		libp2p.EnableRelay(circuit.OptHop),
-	)
-	if err != nil {
-		panic(err)
-	}
-
-	libp2ppubsub, err := pubsub.NewFloodSub(context.Background(), libp2p, []pubsub.Option{
-		pubsub.WithMessageSigning(true),
-	}...)
-	if err != nil {
-		panic(err)
-	}
-
-	fmt.Println("Our public key:", privKey.GetPublic())
-	fmt.Println("Our node ID:", libp2p.ID())
-	fmt.Println("Our addresses:", libp2p.Addrs())
-
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
-
-	return &P2PDendrite{
-		Base:          *baseDendrite,
-		LibP2P:        libp2p,
-		LibP2PContext: ctx,
-		LibP2PCancel:  cancel,
-		LibP2PDHT:     libp2pdht,
-		LibP2PPubsub:  libp2ppubsub,
-	}
-}
-
-type libP2PValidator struct {
-	KeyBook pstore.KeyBook
-}
-
-func (v libP2PValidator) Validate(key string, value []byte) error {
-	ns, _, err := record.SplitKey(key)
-	if err != nil || ns != "matrix" {
-		return errors.New("not Matrix path")
-	}
-	return nil
-}
-
-func (v libP2PValidator) Select(k string, vals [][]byte) (int, error) {
-	return 0, nil
-}
diff --git a/cmd/dendrite-demo-libp2p/publicrooms.go b/cmd/dendrite-demo-libp2p/publicrooms.go
deleted file mode 100644
index 96e8ab5e1..000000000
--- a/cmd/dendrite-demo-libp2p/publicrooms.go
+++ /dev/null
@@ -1,153 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-const MaintenanceInterval = time.Second * 10
-
-type discoveredRoom struct {
-	time time.Time
-	room gomatrixserverlib.PublicRoom
-}
-
-type publicRoomsProvider struct {
-	pubsub           *pubsub.PubSub
-	topic            *pubsub.Topic
-	subscription     *pubsub.Subscription
-	foundRooms       map[string]discoveredRoom // additional rooms we have learned about from the DHT
-	foundRoomsMutex  sync.RWMutex              // protects foundRooms
-	maintenanceTimer *time.Timer               //
-	roomsAdvertised  atomic.Value              // stores int
-	rsAPI            roomserverAPI.RoomserverInternalAPI
-}
-
-func newPublicRoomsProvider(ps *pubsub.PubSub, rsAPI roomserverAPI.RoomserverInternalAPI) *publicRoomsProvider {
-	return &publicRoomsProvider{
-		foundRooms: make(map[string]discoveredRoom),
-		pubsub:     ps,
-		rsAPI:      rsAPI,
-	}
-}
-
-func (p *publicRoomsProvider) Start() error {
-	if topic, err := p.pubsub.Join("/matrix/publicRooms"); err != nil {
-		return err
-	} else if sub, err := topic.Subscribe(); err == nil {
-		p.topic = topic
-		p.subscription = sub
-		go p.MaintenanceTimer()
-		go p.FindRooms()
-		p.roomsAdvertised.Store(0)
-	} else {
-		return err
-	}
-	return nil
-}
-
-func (p *publicRoomsProvider) MaintenanceTimer() {
-	if p.maintenanceTimer != nil && !p.maintenanceTimer.Stop() {
-		<-p.maintenanceTimer.C
-	}
-	p.Interval()
-}
-
-func (p *publicRoomsProvider) Interval() {
-	p.foundRoomsMutex.Lock()
-	for k, v := range p.foundRooms {
-		if time.Since(v.time) > time.Minute {
-			delete(p.foundRooms, k)
-		}
-	}
-	p.foundRoomsMutex.Unlock()
-	if err := p.AdvertiseRooms(); err != nil {
-		fmt.Println("Failed to advertise room in DHT:", err)
-	}
-	p.foundRoomsMutex.RLock()
-	defer p.foundRoomsMutex.RUnlock()
-	fmt.Println("Found", len(p.foundRooms), "room(s), advertised", p.roomsAdvertised.Load(), "room(s)")
-	p.maintenanceTimer = time.AfterFunc(MaintenanceInterval, p.Interval)
-}
-
-func (p *publicRoomsProvider) AdvertiseRooms() error {
-	ctx := context.Background()
-	var queryRes roomserverAPI.QueryPublishedRoomsResponse
-	// Query published rooms on our server. This will not invoke clientapi.ExtraPublicRoomsProvider
-	err := p.rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
-		return err
-	}
-	ourRooms, err := roomserverAPI.PopulatePublicRooms(ctx, queryRes.RoomIDs, p.rsAPI)
-	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed")
-		return err
-	}
-	advertised := 0
-	for _, room := range ourRooms {
-		if j, err := json.Marshal(room); err == nil {
-			if err := p.topic.Publish(context.TODO(), j); err != nil {
-				fmt.Println("Failed to publish public room:", err)
-			} else {
-				advertised++
-			}
-		}
-	}
-
-	p.roomsAdvertised.Store(advertised)
-	return nil
-}
-
-func (p *publicRoomsProvider) FindRooms() {
-	for {
-		msg, err := p.subscription.Next(context.Background())
-		if err != nil {
-			continue
-		}
-		received := discoveredRoom{
-			time: time.Now(),
-		}
-		if err := json.Unmarshal(msg.Data, &received.room); err != nil {
-			fmt.Println("Unmarshal error:", err)
-			continue
-		}
-		fmt.Printf("received %+v \n", received)
-		p.foundRoomsMutex.Lock()
-		p.foundRooms[received.room.RoomID] = received
-		p.foundRoomsMutex.Unlock()
-	}
-}
-
-func (p *publicRoomsProvider) Rooms() (rooms []gomatrixserverlib.PublicRoom) {
-	p.foundRoomsMutex.RLock()
-	defer p.foundRoomsMutex.RUnlock()
-	for _, dr := range p.foundRooms {
-		rooms = append(rooms, dr.room)
-	}
-	return
-}
diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go
deleted file mode 100644
index 4298c2ae9..000000000
--- a/cmd/dendritejs/jsServer.go
+++ /dev/null
@@ -1,101 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"syscall/js"
-)
-
-// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
-type JSServer struct {
-	// The router which will service requests
-	Mux http.Handler
-}
-
-// OnRequestFromJS is the function that JS will invoke when there is a new request.
-// The JS function signature is:
-//   function(reqString: string): Promise<{result: string, error: string}>
-// Usage is like:
-//   const res = await global._go_js_server.fetch(reqString);
-//   if (res.error) {
-//     // handle error: this is a 'network' error, not a non-2xx error.
-//   }
-//   const rawHttpResponse = res.result;
-func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
-	// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
-	// if this request blocks at all e.g for /sync calls
-	httpStr := args[0].String()
-	promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} {
-		// The initial callback code for new Promise() is also called on the critical path, which is why
-		// we need to put this in an immediately invoked goroutine.
-		go func() {
-			resolve := pargs[0]
-			resStr, err := h.handle(httpStr)
-			errStr := ""
-			if err != nil {
-				errStr = err.Error()
-			}
-			resolve.Invoke(map[string]interface{}{
-				"result": resStr,
-				"error":  errStr,
-			})
-		}()
-		return nil
-	}))
-	return promise
-}
-
-// handle invokes the http.ServeMux for this request and returns the raw HTTP response.
-func (h *JSServer) handle(httpStr string) (resStr string, err error) {
-	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr)))
-	if err != nil {
-		return
-	}
-	w := httptest.NewRecorder()
-
-	h.Mux.ServeHTTP(w, req)
-
-	res := w.Result()
-	var resBuffer strings.Builder
-	err = res.Write(&resBuffer)
-	return resBuffer.String(), err
-}
-
-// ListenAndServe registers a variable in JS-land with the given namespace. This variable is
-// a function which JS-land can call to 'send' HTTP requests. The function is attached to
-// a global object called "_go_js_server". See OnRequestFromJS for more info.
-func (h *JSServer) ListenAndServe(namespace string) {
-	globalName := "_go_js_server"
-	// register a hook in JS-land for it to invoke stuff
-	server := js.Global().Get(globalName)
-	if !server.Truthy() {
-		server = js.Global().Get("Object").New()
-		js.Global().Set(globalName, server)
-	}
-
-	server.Set(namespace, js.FuncOf(h.OnRequestFromJS))
-
-	fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace)
-	// Block forever to mimic http.ListenAndServe
-	select {}
-}
diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go
deleted file mode 100644
index cdf937649..000000000
--- a/cmd/dendritejs/keyfetcher.go
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-const libp2pMatrixKeyID = "ed25519:libp2p-dendrite"
-
-type libp2pKeyFetcher struct {
-}
-
-// FetchKeys looks up a batch of public keys.
-// Takes a map from (server name, key ID) pairs to timestamp.
-// The timestamp is when the keys need to be vaild up to.
-// Returns a map from (server name, key ID) pairs to server key objects for
-// that server name containing that key ID
-// The result may have fewer (server name, key ID) pairs than were in the request.
-// The result may have more (server name, key ID) pairs than were in the request.
-// Returns an error if there was a problem fetching the keys.
-func (f *libp2pKeyFetcher) FetchKeys(
-	ctx context.Context,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
-) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
-	res := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
-	for req := range requests {
-		if req.KeyID != libp2pMatrixKeyID {
-			return nil, fmt.Errorf("FetchKeys: cannot fetch key with ID %s, should be %s", req.KeyID, libp2pMatrixKeyID)
-		}
-
-		// The server name is a libp2p peer ID
-		peerIDStr := string(req.ServerName)
-		peerID, err := peer.Decode(peerIDStr)
-		if err != nil {
-			return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %w", peerIDStr, err)
-		}
-		pubKey, err := peerID.ExtractPublicKey()
-		if err != nil {
-			return nil, fmt.Errorf("Failed to extract public key from peer ID: %w", err)
-		}
-		pubKeyBytes, err := pubKey.Raw()
-		if err != nil {
-			return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err)
-		}
-		b64Key := gomatrixserverlib.Base64Bytes(pubKeyBytes)
-		res[req] = gomatrixserverlib.PublicKeyLookupResult{
-			VerifyKey: gomatrixserverlib.VerifyKey{
-				Key: b64Key,
-			},
-			ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
-			ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(24 * time.Hour * 365)),
-		}
-	}
-	return res, nil
-}
-
-// FetcherName returns the name of this fetcher, which can then be used for
-// logging errors etc.
-func (f *libp2pKeyFetcher) FetcherName() string {
-	return "libp2pKeyFetcher"
-}
-
-// no-op function for storing keys - we don't do any work to fetch them so don't bother storing.
-func (f *libp2pKeyFetcher) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error {
-	return nil
-}
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
deleted file mode 100644
index 05e0f0ad9..000000000
--- a/cmd/dendritejs/main.go
+++ /dev/null
@@ -1,270 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"crypto/ed25519"
-	"fmt"
-	"syscall/js"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi"
-	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
-
-	"github.com/matrix-org/gomatrixserverlib"
-
-	"github.com/sirupsen/logrus"
-
-	_ "github.com/matrix-org/go-sqlite3-js"
-)
-
-var GitCommit string
-
-func init() {
-	fmt.Printf("[%s] dendrite.js starting...\n", GitCommit)
-}
-
-const keyNameEd25519 = "_go_ed25519_key"
-
-func readKeyFromLocalStorage() (key ed25519.PrivateKey, err error) {
-	localforage := js.Global().Get("localforage")
-	if !localforage.Truthy() {
-		err = fmt.Errorf("readKeyFromLocalStorage: no localforage")
-		return
-	}
-	// https://localforage.github.io/localForage/
-	item, ok := await(localforage.Call("getItem", keyNameEd25519))
-	if !ok || !item.Truthy() {
-		err = fmt.Errorf("readKeyFromLocalStorage: no key in localforage")
-		return
-	}
-	fmt.Println("Found key in localforage")
-	// extract []byte and make an ed25519 key
-	seed := make([]byte, 32, 32)
-	js.CopyBytesToGo(seed, item)
-
-	return ed25519.NewKeyFromSeed(seed), nil
-}
-
-func writeKeyToLocalStorage(key ed25519.PrivateKey) error {
-	localforage := js.Global().Get("localforage")
-	if !localforage.Truthy() {
-		return fmt.Errorf("writeKeyToLocalStorage: no localforage")
-	}
-
-	// make a Uint8Array from the key's seed
-	seed := key.Seed()
-	jsSeed := js.Global().Get("Uint8Array").New(len(seed))
-	js.CopyBytesToJS(jsSeed, seed)
-	// write it
-	localforage.Call("setItem", keyNameEd25519, jsSeed)
-	return nil
-}
-
-// taken from https://go-review.googlesource.com/c/go/+/150917
-
-// await waits until the promise v has been resolved or rejected and returns the promise's result value.
-// The boolean value ok is true if the promise has been resolved, false if it has been rejected.
-// If v is not a promise, v itself is returned as the value and ok is true.
-func await(v js.Value) (result js.Value, ok bool) {
-	if v.Type() != js.TypeObject || v.Get("then").Type() != js.TypeFunction {
-		return v, true
-	}
-	done := make(chan struct{})
-	onResolve := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
-		result = args[0]
-		ok = true
-		close(done)
-		return nil
-	})
-	defer onResolve.Release()
-	onReject := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
-		result = args[0]
-		ok = false
-		close(done)
-		return nil
-	})
-	defer onReject.Release()
-	v.Call("then", onResolve, onReject)
-	<-done
-	return
-}
-
-func generateKey() ed25519.PrivateKey {
-	// attempt to look for a seed in JS-land and if it exists use it.
-	priv, err := readKeyFromLocalStorage()
-	if err == nil {
-		fmt.Println("Read key from localStorage")
-		return priv
-	}
-	// generate a new key
-	fmt.Println(err, " : Generating new ed25519 key")
-	_, priv, err = ed25519.GenerateKey(nil)
-	if err != nil {
-		logrus.Fatalf("Failed to generate ed25519 key: %s", err)
-	}
-	if err := writeKeyToLocalStorage(priv); err != nil {
-		fmt.Println("failed to write key to localStorage: ", err)
-		// non-fatal, we'll just have amnesia for a while
-	}
-	return priv
-}
-
-func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.FederationClient {
-	fmt.Println("Running in js-libp2p federation mode")
-	fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!")
-	tr := go_http_js_libp2p.NewP2pTransport(node)
-
-	fed := gomatrixserverlib.NewFederationClient(
-		cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey,
-		gomatrixserverlib.WithTransport(tr),
-	)
-
-	return fed
-}
-
-func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client {
-	tr := go_http_js_libp2p.NewP2pTransport(node)
-	return gomatrixserverlib.NewClient(
-		gomatrixserverlib.WithTransport(tr),
-	)
-}
-
-func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) {
-	hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/"
-	node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}, "p2p")
-	serverName = node.Id
-	fmt.Println("p2p assigned ServerName: ", serverName)
-	return
-}
-
-func main() {
-	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
-	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
-	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
-	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
-	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
-	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
-	cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db"
-	cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db"
-	cfg.Global.JetStream.StoragePath = "file:/idb/dendritejs/"
-	cfg.Global.TrustedIDServers = []string{
-		"matrix.org", "vector.im",
-	}
-	cfg.Global.KeyID = libp2pMatrixKeyID
-	cfg.Global.PrivateKey = generateKey()
-
-	serverName, node := createP2PNode(cfg.Global.PrivateKey)
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(serverName)
-
-	if err := cfg.Derive(); err != nil {
-		logrus.Fatalf("Failed to derive values from config: %s", err)
-	}
-	base := setup.NewBaseDendrite(cfg, "Monolith")
-	defer base.Close() // nolint: errcheck
-
-	accountDB := base.CreateAccountsDB()
-	federation := createFederationClient(cfg, node)
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
-	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
-	keyAPI.SetUserAPI(userAPI)
-
-	fetcher := &libp2pKeyFetcher{}
-	keyRing := gomatrixserverlib.KeyRing{
-		KeyFetchers: []gomatrixserverlib.KeyFetcher{
-			fetcher,
-		},
-		KeyDatabase: fetcher,
-	}
-
-	rsAPI := roomserver.NewInternalAPI(base)
-	asQuery := appservice.NewInternalAPI(
-		base, userAPI, rsAPI,
-	)
-	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true)
-	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
-	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation)
-
-	psAPI := pushserver.NewInternalAPI(base)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		AccountDB: accountDB,
-		Client:    createClient(node),
-		FedClient: federation,
-		KeyRing:   &keyRing,
-
-		AppserviceAPI:       asQuery,
-		FederationSenderAPI: fedSenderAPI,
-		RoomserverAPI:       rsAPI,
-		UserAPI:             userAPI,
-		KeyAPI:              keyAPI,
-		PushserverAPI:       psAPI,
-		//ServerKeyAPI:        serverKeyAPI,
-		ExtPublicRoomsProvider: p2pPublicRoomProvider,
-	}
-	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-	)
-
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	libp2pRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	libp2pRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	libp2pRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	// Expose the matrix APIs via libp2p-js - for federation traffic
-	if node != nil {
-		go func() {
-			logrus.Info("Listening on libp2p-js host ID ", node.Id)
-			s := JSServer{
-				Mux: libp2pRouter,
-			}
-			s.ListenAndServe("p2p")
-		}()
-	}
-
-	// Expose the matrix APIs via fetch - for local traffic
-	go func() {
-		logrus.Info("Listening for service-worker fetch traffic")
-		s := JSServer{
-			Mux: httpRouter,
-		}
-		s.ListenAndServe("fetch")
-	}()
-
-	// We want to block forever to let the fetch and libp2p handler serve the APIs
-	select {}
-}
diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go
deleted file mode 100644
index 0cc7e47e5..000000000
--- a/cmd/dendritejs/main_noop.go
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build !wasm
-// +build !wasm
-
-package main
-
-import "fmt"
-
-func main() {
-	fmt.Println("dendritejs: no-op when not compiling for WebAssembly")
-}
diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go
deleted file mode 100644
index 2e3339a45..000000000
--- a/cmd/dendritejs/publicrooms.go
+++ /dev/null
@@ -1,155 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build wasm
-// +build wasm
-
-package main
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/dendrite/federationapi/api"
-	go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-type libp2pPublicRoomsProvider struct {
-	node      *go_http_js_libp2p.P2pLocalNode
-	providers []go_http_js_libp2p.PeerInfo
-	fedSender api.FederationInternalAPI
-	fedClient *gomatrixserverlib.FederationClient
-}
-
-func NewLibP2PPublicRoomsProvider(
-	node *go_http_js_libp2p.P2pLocalNode, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient,
-) *libp2pPublicRoomsProvider {
-	p := &libp2pPublicRoomsProvider{
-		node:      node,
-		fedSender: fedSender,
-		fedClient: fedClient,
-	}
-	node.RegisterFoundProviders(p.foundProviders)
-	return p
-}
-
-func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) {
-	// work out the diff then poke for new ones
-	seen := make(map[string]bool, len(p.providers))
-	for _, pr := range p.providers {
-		seen[pr.Id] = true
-	}
-	var newPeers []gomatrixserverlib.ServerName
-	for _, pi := range peerInfos {
-		if !seen[pi.Id] {
-			newPeers = append(newPeers, gomatrixserverlib.ServerName(pi.Id))
-		}
-	}
-	if len(newPeers) > 0 {
-		var res api.PerformServersAliveResponse
-		// ignore errors, we don't care.
-		p.fedSender.PerformServersAlive(context.Background(), &api.PerformServersAliveRequest{
-			Servers: newPeers,
-		}, &res)
-	}
-
-	p.providers = peerInfos
-}
-
-func (p *libp2pPublicRoomsProvider) Rooms() []gomatrixserverlib.PublicRoom {
-	return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, p.homeservers())
-}
-
-func (p *libp2pPublicRoomsProvider) homeservers() []string {
-	result := make([]string, len(p.providers))
-	for i := range p.providers {
-		result[i] = p.providers[i].Id
-	}
-	return result
-}
-
-// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
-// Returns a list of public rooms.
-func bulkFetchPublicRoomsFromServers(
-	ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string,
-) (publicRooms []gomatrixserverlib.PublicRoom) {
-	limit := 200
-	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
-	// goroutines send rooms to this channel
-	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
-	// signalling channel to tell goroutines to stop sending rooms and quit
-	done := make(chan bool)
-	// signalling to say when we can close the room channel
-	var wg sync.WaitGroup
-	wg.Add(len(homeservers))
-	// concurrently query for public rooms
-	for _, hs := range homeservers {
-		go func(homeserverDomain string) {
-			defer wg.Done()
-			util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
-			fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "")
-			if err != nil {
-				util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
-					"bulkFetchPublicRoomsFromServers: failed to query hs",
-				)
-				return
-			}
-			for _, room := range fres.Chunk {
-				// atomically send a room or stop
-				select {
-				case roomCh <- room:
-				case <-done:
-					util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
-					return
-				}
-			}
-		}(hs)
-	}
-
-	// Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request.
-	// This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be
-	// closed.
-	go func() {
-		wg.Wait()
-		util.GetLogger(ctx).Info("Cleaning up resources")
-		close(roomCh)
-	}()
-
-	// fan-in results with timeout. We stop when we reach the limit.
-FanIn:
-	for len(publicRooms) < int(limit) || limit == 0 {
-		// add a room or timeout
-		select {
-		case room, ok := <-roomCh:
-			if !ok {
-				util.GetLogger(ctx).Info("All homeservers have been queried, returning results.")
-				break FanIn
-			}
-			publicRooms = append(publicRooms, room)
-		case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got.
-			util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early")
-			break FanIn
-		case <-ctx.Done(): // the client hung up on us, let's stop.
-			util.GetLogger(ctx).Info("Client hung up, returning early")
-			break FanIn
-		}
-	}
-	// tell goroutines to stop
-	close(done)
-
-	return publicRooms
-}
diff --git a/go.mod b/go.mod
index a9774c2c8..ba222ed8f 100644
--- a/go.mod
+++ b/go.mod
@@ -26,22 +26,14 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
 	github.com/lib/pq v1.10.5
-	github.com/libp2p/go-libp2p v0.13.0
-	github.com/libp2p/go-libp2p-circuit v0.4.0
-	github.com/libp2p/go-libp2p-core v0.8.3
-	github.com/libp2p/go-libp2p-gostream v0.3.1
-	github.com/libp2p/go-libp2p-http v0.2.0
-	github.com/libp2p/go-libp2p-kad-dht v0.11.1
-	github.com/libp2p/go-libp2p-pubsub v0.4.1
-	github.com/libp2p/go-libp2p-record v0.1.3
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
-	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
+	github.com/miekg/dns v1.1.31 // indirect
 	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
 	github.com/nats-io/nats.go v1.14.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
diff --git a/go.sum b/go.sum
index ee4096fdc..8bb306a82 100644
--- a/go.sum
+++ b/go.sum
@@ -47,7 +47,6 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
-github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0 h1:QUqcb7BOcBU2p7Nax7pESOb8hrZYtI0Ts6j4v4mvcQo=
 github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
@@ -80,7 +79,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
-github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc=
 github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@@ -122,7 +120,6 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
-github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw=
@@ -153,8 +150,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
-github.com/benbjohnson/clock v1.0.2/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -174,19 +169,6 @@ github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2w
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
-github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
-github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
-github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
-github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw=
-github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
-github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
-github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
-github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
-github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
-github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
-github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
@@ -340,7 +322,6 @@ github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka
 github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -363,21 +344,13 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
 github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
-github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0=
-github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
-github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
-github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
-github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU=
-github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
@@ -426,8 +399,6 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
-github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -532,7 +503,6 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -546,7 +516,6 @@ github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3K
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
@@ -593,9 +562,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
-github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
-github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -646,8 +612,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
-github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -656,15 +620,12 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
-github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -686,9 +647,6 @@ github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCe
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
-github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
-github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
-github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
@@ -699,49 +657,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
-github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
-github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
-github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
-github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
-github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
-github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
-github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
-github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
-github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
-github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
-github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
-github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
-github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
-github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
-github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg=
-github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs=
-github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
-github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
-github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
-github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s=
-github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk=
-github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE=
-github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk=
-github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
-github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
-github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
-github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
-github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
-github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
-github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
-github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
-github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs=
-github.com/ipfs/go-ipns v0.0.2/go.mod h1:WChil4e0/m9cIINWLxZe1Jtf77oz5L05rO2ei/uKJ5U=
-github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
-github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
-github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
-github.com/ipfs/go-log v1.0.4 h1:6nLQdX4W8P9yZZFH7mO+X/PzjN8Laozm/lMJ6esdgzY=
-github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs=
-github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
-github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
-github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw=
-github.com/ipfs/go-log/v2 v2.1.1 h1:G4TtqN+V9y9HY9TA6BwbCVyyBZ2B9MbCjR2MtGx8FR0=
-github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM=
 github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
 github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
 github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
@@ -749,23 +664,7 @@ github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0Gqw
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw=
-github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
-github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
-github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
-github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
-github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
-github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
-github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
-github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
-github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk=
-github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk=
-github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
-github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
-github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
-github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
@@ -773,7 +672,6 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -823,7 +721,6 @@ github.com/julienschmidt/httprouter v1.1.1-0.20151013225520-77a895ad01eb/go.mod
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
-github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
 github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc=
 github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=
 github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=
@@ -834,7 +731,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@@ -846,8 +742,6 @@ github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ=
-github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -866,215 +760,6 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
 github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
-github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU=
-github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E=
-github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
-github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs=
-github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
-github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=
-github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=
-github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc=
-github.com/libp2p/go-conn-security-multistream v0.2.0 h1:uNiDjS58vrvJTg9jO6bySd1rMKejieG7v45ekqHbZ1M=
-github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU=
-github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4=
-github.com/libp2p/go-eventbus v0.2.1 h1:VanAdErQnpTioN2TowqNcOijf6YwhuODe4pPKSDpxGc=
-github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8=
-github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8=
-github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
-github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM=
-github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
-github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZkfEI5sT54=
-github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k=
-github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw=
-github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o=
-github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0=
-github.com/libp2p/go-libp2p v0.13.0 h1:tDdrXARSghmusdm0nf1U/4M8aj8Rr0V2IzQOXmbzQ3s=
-github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo=
-github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 h1:BM7aaOF7RpmNn9+9g6uTjGJ0cTzWr5j9i9IKeun2M8U=
-github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo=
-github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE=
-github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI=
-github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI=
-github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A=
-github.com/libp2p/go-libp2p-autonat v0.4.0 h1:3y8XQbpr+ssX8QfZUHekjHCYK64sj6/4hnf/awD4+Ug=
-github.com/libp2p/go-libp2p-autonat v0.4.0/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk=
-github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
-github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU=
-github.com/libp2p/go-libp2p-blankhost v0.2.0 h1:3EsGAi0CBGcZ33GwRuXEYJLLPoVWyXJ1bcJzAJjINkk=
-github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ=
-github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU=
-github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo=
-github.com/libp2p/go-libp2p-circuit v0.4.0 h1:eqQ3sEYkGTtybWgr6JLqJY6QLtPWRErvFjFDfAOO1wc=
-github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA=
-github.com/libp2p/go-libp2p-connmgr v0.2.4 h1:TMS0vc0TCBomtQJyWr7fYxcVYYhx+q/2gF++G5Jkl/w=
-github.com/libp2p/go-libp2p-connmgr v0.2.4/go.mod h1:YV0b/RIm8NGPnnNWM7hG9Q38OeQiQfKhHCCs1++ufn0=
-github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco=
-github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I=
-github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI=
-github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0=
-github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
-github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA=
-github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw=
-github.com/libp2p/go-libp2p-core v0.3.1/go.mod h1:thvWy0hvaSBhnVBaW37BvzgVV68OUhgJJLAa6almrII=
-github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0=
-github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0=
-github.com/libp2p/go-libp2p-core v0.5.1/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y=
-github.com/libp2p/go-libp2p-core v0.5.3/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y=
-github.com/libp2p/go-libp2p-core v0.5.4/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y=
-github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqeHCopzbYKZdRjM=
-github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo=
-github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo=
-github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo=
-github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8=
-github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8=
-github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8=
-github.com/libp2p/go-libp2p-core v0.8.3 h1:BZTReEF6o8g/n4DwxTyeFannOeae35Xy0TD+mES3CNE=
-github.com/libp2p/go-libp2p-core v0.8.3/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8=
-github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI=
-github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg=
-github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw=
-github.com/libp2p/go-libp2p-discovery v0.5.0 h1:Qfl+e5+lfDgwdrXdu4YNCWyEo3fWuP+WgN9mN0iWviQ=
-github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug=
-github.com/libp2p/go-libp2p-gostream v0.3.0/go.mod h1:pLBQu8db7vBMNINGsAwLL/ZCE8wng5V1FThoaE5rNjc=
-github.com/libp2p/go-libp2p-gostream v0.3.1 h1:XlwohsPn6uopGluEWs1Csv1QCEjrTXf2ZQagzZ5paAg=
-github.com/libp2p/go-libp2p-gostream v0.3.1/go.mod h1:1V3b+u4Zhaq407UUY9JLCpboaeufAeVQbnvAt12LRsI=
-github.com/libp2p/go-libp2p-http v0.2.0 h1:GYeVd+RZzkRa8XFLITqOpcrIQG6KbFLPJqII6HHBHzY=
-github.com/libp2p/go-libp2p-http v0.2.0/go.mod h1:GlNKFqDZHe25LVy2CvnZKx75/jLtMaD3VxZV6N39X7E=
-github.com/libp2p/go-libp2p-kad-dht v0.11.1 h1:FsriVQhOUZpCotWIjyFSjEDNJmUzuMma/RyyTDZanwc=
-github.com/libp2p/go-libp2p-kad-dht v0.11.1/go.mod h1:5ojtR2acDPqh/jXf5orWy8YGb8bHQDS+qeDcoscL/PI=
-github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70=
-github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk=
-github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8=
-github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90=
-github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo=
-github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE=
-github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo=
-github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek=
-github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs=
-github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw=
-github.com/libp2p/go-libp2p-mplex v0.4.1 h1:/pyhkP1nLwjG3OM+VuaNJkQT/Pqq73WzB3aDN3Fx1sc=
-github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g=
-github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE=
-github.com/libp2p/go-libp2p-nat v0.0.6 h1:wMWis3kYynCbHoyKLPBEMu4YRLltbm8Mk08HGSfvTkU=
-github.com/libp2p/go-libp2p-nat v0.0.6/go.mod h1:iV59LVhB3IkFvS6S6sauVTSOrNEANnINbI/fkaLimiw=
-github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ=
-github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
-github.com/libp2p/go-libp2p-noise v0.1.1 h1:vqYQWvnIcHpIoWJKC7Al4D6Hgj0H012TuXRhPwSMGpQ=
-github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM=
-github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
-github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
-github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
-github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs=
-github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ=
-github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
-github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
-github.com/libp2p/go-libp2p-peerstore v0.2.6 h1:2ACefBX23iMdJU9Ke+dcXt3w86MIryes9v7In4+Qq3U=
-github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s=
-github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k=
-github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA=
-github.com/libp2p/go-libp2p-pubsub v0.4.1 h1:j4umIg5nyus+sqNfU+FWvb9aeYFQH/A+nDFhWj+8yy8=
-github.com/libp2p/go-libp2p-pubsub v0.4.1/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ=
-github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGdsU/9W//C8dqjQDk=
-github.com/libp2p/go-libp2p-record v0.1.3 h1:R27hoScIhQf/A8XJZ8lYpnqh9LatJ5YbHs28kCIfql0=
-github.com/libp2p/go-libp2p-record v0.1.3/go.mod h1:yNUff/adKIfPnYQXgp6FQmNu3gLJ6EMg7+/vv2+9pY4=
-github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw=
-github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
-github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
-github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
-github.com/libp2p/go-libp2p-secio v0.2.2/go.mod h1:wP3bS+m5AUnFA+OFO7Er03uO1mncHG0uVwGrwvjYlNY=
-github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4=
-github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU=
-github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM=
-github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM=
-github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk=
-github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk=
-github.com/libp2p/go-libp2p-swarm v0.4.0 h1:hahq/ijRoeH6dgROOM8x7SeaKK5VgjjIr96vdrT+NUA=
-github.com/libp2p/go-libp2p-swarm v0.4.0/go.mod h1:XVFcO52VoLoo0eitSxNQWYq4D6sydGOweTOAjJNraCw=
-github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
-github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
-github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
-github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0=
-github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0=
-github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc=
-github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g=
-github.com/libp2p/go-libp2p-testing v0.4.0 h1:PrwHRi0IGqOwVQWR3xzgigSlhlLfxgfXgkHxr77EghQ=
-github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0=
-github.com/libp2p/go-libp2p-tls v0.1.3 h1:twKMhMu44jQO+HgQK9X8NHO5HkeJu2QbhLzLJpa8oNM=
-github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M=
-github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA=
-github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns=
-github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o=
-github.com/libp2p/go-libp2p-transport-upgrader v0.4.0 h1:xwj4h3hJdBrxqMOyMUjwscjoVst0AASTsKtZiTChoHI=
-github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s=
-github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8=
-github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw=
-github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ48OpsfmQVTErwA=
-github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhLEn0XhIoZ5viCwU=
-github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4=
-github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30=
-github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po=
-github.com/libp2p/go-libp2p-yamux v0.5.1 h1:sX4WQPHMhRxJE5UZTfjEuBvlQWXB5Bo3A2JK9ZJ9EM0=
-github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4=
-github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q=
-github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M=
-github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU=
-github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0=
-github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU=
-github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk=
-github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk=
-github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ=
-github.com/libp2p/go-mplex v0.3.0 h1:U1T+vmCYJaEoDJPV1aq31N56hS+lJgb397GsylNSgrU=
-github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ=
-github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
-github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
-github.com/libp2p/go-msgio v0.0.6 h1:lQ7Uc0kS1wb1EfRxO2Eir/RJoHkHn7t6o+EiwsYIKJA=
-github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA=
-github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo=
-github.com/libp2p/go-nat v0.0.5 h1:qxnwkco8RLKqVh1NmjQ+tJ8p8khNLFxuElYG/TwqW4Q=
-github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU=
-github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk=
-github.com/libp2p/go-netroute v0.1.3 h1:1ngWRx61us/EpaKkdqkMjKk/ufr/JlIFYQAxV2XX8Ig=
-github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk=
-github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0=
-github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
-github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
-github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
-github.com/libp2p/go-openssl v0.0.7 h1:eCAzdLejcNVBzP/iZM9vqHnQm+XyCEbSSIheIPRGNsw=
-github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
-github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
-github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU=
-github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ=
-github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
-github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM=
-github.com/libp2p/go-reuseport-transport v0.0.4 h1:OZGz0RB620QDGpv300n1zaOcKGGAoGVf8h9txtt/1uM=
-github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw=
-github.com/libp2p/go-sockaddr v0.0.2 h1:tCuXfpA9rq7llM/v834RKc/Xvovy/AqM9kHvTV/jY/Q=
-github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
-github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
-github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc=
-github.com/libp2p/go-stream-muxer-multistream v0.3.0 h1:TqnSHPJEIqDEO7h1wZZ0p3DXdvDSiLHQidKKUGZtiOY=
-github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA=
-github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
-github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY=
-github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0=
-github.com/libp2p/go-tcp-transport v0.2.1 h1:ExZiVQV+h+qL16fzCWtd1HSzPsqWottJ8KXwWaVi8Ns=
-github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M=
-github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM=
-github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk=
-github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk=
-github.com/libp2p/go-ws-transport v0.4.0 h1:9tvtQ9xbws6cA5LvqdE6Ne3vcmGB4f1z9SByggk4s0k=
-github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA=
-github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
-github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
-github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
-github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
-github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
-github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
-github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI=
-github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
-github.com/libp2p/go-yamux/v2 v2.0.0 h1:vSGhAy5u6iHBq11ZDcyHH4Blcf9xlBhT4WQDoOE90LU=
-github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U=
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
 github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A=
 github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
@@ -1085,7 +770,6 @@ github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -1105,8 +789,6 @@ github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zw
 github.com/masterzen/xmlpath v0.0.0-20140218185901-13f4951698ad/go.mod h1:A0zPC53iKKKcXYxr4ROjpQRQ5FgJXtelNdSmHHuq/tY=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
-github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4=
-github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
@@ -1121,14 +803,12 @@ github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335M
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-colorable v0.0.6/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -1150,26 +830,15 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
 github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
-github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
-github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
 github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
-github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
-github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
-github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
-github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
-github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -1202,65 +871,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
-github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
-github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
-github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
-github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
-github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
-github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
-github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
-github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
-github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
-github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
-github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
-github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
-github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
-github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
-github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
-github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
-github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE=
-github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y=
-github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI=
-github.com/multiformats/go-multiaddr v0.3.1 h1:1bxa+W7j9wZKTZREySx1vPMs2TqrYWjVZ7zE6/XLG1I=
-github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc=
-github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
-github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
-github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA=
-github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
-github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
-github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=
-github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo=
-github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU=
-github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ=
-github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ=
-github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y=
-github.com/multiformats/go-multiaddr-net v0.1.3/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA=
-github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA=
-github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA=
-github.com/multiformats/go-multiaddr-net v0.2.0 h1:MSXRGN0mFymt6B1yo/6BPnIRpLPEnKgQNvVfCX5VDJk=
-github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA=
-github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
-github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
-github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
-github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
-github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
-github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
-github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
-github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
-github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
-github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
-github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
-github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
-github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38=
-github.com/multiformats/go-multistream v0.2.0 h1:6AuNmQVKUkRnddw2YiDjt5Elit40SFxMJkVnhmETXtU=
-github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k=
-github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
-github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1296,7 +908,6 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
 github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -1310,7 +921,6 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
@@ -1349,7 +959,6 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
@@ -1475,17 +1084,11 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
-github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
-github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
-github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -1503,7 +1106,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -1524,7 +1126,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -1572,21 +1173,9 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
-github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
-github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
-github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=
-github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
-github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD21WxrmfeIYCFPuVPRCY2XZTWzTNHGw30=
-github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
-github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds=
-github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
-github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow=
-github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
@@ -1623,12 +1212,10 @@ go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvS
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
@@ -1653,28 +1240,17 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
 go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
 go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
-go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
-go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
-go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
-golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -1683,22 +1259,16 @@ golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -1773,7 +1343,6 @@ golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -1858,7 +1427,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1871,19 +1439,15 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -2015,7 +1579,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -2038,8 +1601,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -2204,7 +1765,6 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
@@ -2269,8 +1829,6 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
-gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk=

From 0eb5bd1e13b974a6a1f9e4548eab979cd6fd7528 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 21 Apr 2022 17:17:52 +0100
Subject: [PATCH 018/391] Update AWSY test groups (#2365)

* Update AWSY test groups

* Better names
---
 are-we-synapse-yet.list | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list
index 4281171ab..c776a7400 100644
--- a/are-we-synapse-yet.list
+++ b/are-we-synapse-yet.list
@@ -212,11 +212,12 @@ plv Users cannot set kick powerlevel higher than their own (2 subtests)
 plv Users cannot set redact powerlevel higher than their own (2 subtests)
 v1s Check that event streams started after a client joined a room work (SYT-1)
 v1s Event stream catches up fully after many messages
-xxx POST /rooms/:room_id/redact/:event_id as power user redacts message
-xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message
-xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message
-xxx POST /redact disallows redaction of event in different room
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as random user does not redact message
+xxx PUT /redact disallows redaction of event in different room
 xxx Redaction of a redaction redacts the redaction reason
+xxx PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
 v1s A departed room is still included in /initialSync (SPEC-216)
 v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216)
 rst Can get rooms/{roomId}/state for a departed room (SPEC-216)
@@ -921,3 +922,18 @@ msc We can't peek into rooms with invited history_visibility
 msc We can't peek into rooms with joined history_visibility
 msc Local users can peek by room alias
 msc Peeked rooms only turn up in the sync for the device who peeked them
+ban 'ban' event respects room powerlevel (2 subtests)
+inv Test that we can be reinvited to a room we created (11 subtests)
+fiv Rejecting invite over federation doesn't break incremental /sync
+pre Presence can be set from sync
+fst /state returns M_NOT_FOUND for an outlier
+fst /state_ids returns M_NOT_FOUND for an outlier
+fst /state returns M_NOT_FOUND for a rejected message event
+fst /state_ids returns M_NOT_FOUND for a rejected message event
+fst /state returns M_NOT_FOUND for a rejected state event
+fst /state_ids returns M_NOT_FOUND for a rejected state event
+fst Room state after a rejected message event is the same as before
+fst Room state after a rejected state event is the same as before
+fpb Federation publicRoom Name/topic keys are correct
+fed New federated private chats get full presence information (SYN-115) (10 subtests)
+dvk Rejects invalid device keys
\ No newline at end of file

From c07f347f0054515a41834e171759569dc513187e Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 22 Apr 2022 11:38:29 +0200
Subject: [PATCH 019/391] Reuse the existing lazyload cache on /context and
 /messages (#2367)

---
 syncapi/routing/context.go  | 23 ++++++++++++++++++-----
 syncapi/routing/messages.go |  9 ++++++++-
 syncapi/routing/routing.go  |  5 ++++-
 syncapi/syncapi.go          |  2 +-
 4 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index aaa0c61bf..17215b669 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -22,6 +22,7 @@ import (
 	"strconv"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/caching"
 	roomserver "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -44,6 +45,7 @@ func Context(
 	rsAPI roomserver.RoomserverInternalAPI,
 	syncDB storage.Database,
 	roomID, eventID string,
+	lazyLoadCache *caching.LazyLoadCache,
 ) util.JSONResponse {
 	filter, err := parseRoomEventFilter(req)
 	if err != nil {
@@ -129,7 +131,7 @@ func Context(
 
 	eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll)
 	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll)
-	newState := applyLazyLoadMembers(filter, eventsAfterClient, eventsBeforeClient, state)
+	newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache)
 
 	response := ContextRespsonse{
 		Event:        gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll),
@@ -148,15 +150,25 @@ func Context(
 	}
 }
 
-func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, state []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
+func applyLazyLoadMembers(
+	device *userapi.Device,
+	filter *gomatrixserverlib.RoomEventFilter,
+	eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent,
+	state []*gomatrixserverlib.HeaderedEvent,
+	lazyLoadCache *caching.LazyLoadCache,
+) []*gomatrixserverlib.HeaderedEvent {
 	if filter == nil || !filter.LazyLoadMembers {
 		return state
 	}
 	allEvents := append(eventsBefore, eventsAfter...)
-	x := make(map[string]bool)
+	x := make(map[string]struct{})
 	// get members who actually send an event
 	for _, e := range allEvents {
-		x[e.Sender] = true
+		// Don't add membership events the client should already know about
+		if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, e.RoomID, e.Sender); cached {
+			continue
+		}
+		x[e.Sender] = struct{}{}
 	}
 
 	newState := []*gomatrixserverlib.HeaderedEvent{}
@@ -166,8 +178,9 @@ func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter
 			newState = append(newState, event)
 		} else {
 			// did the user send an event?
-			if x[event.Sender()] {
+			if _, ok := x[event.Sender()]; ok {
 				membershipEvents = append(membershipEvents, event)
+				lazyLoadCache.StoreLazyLoadedUser(device, event.RoomID(), event.Sender(), event.EventID())
 			}
 		}
 	}
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 519aeff68..f34901bf2 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -21,6 +21,7 @@ import (
 	"sort"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage"
@@ -64,6 +65,7 @@ func OnIncomingMessagesRequest(
 	rsAPI api.RoomserverInternalAPI,
 	cfg *config.SyncAPI,
 	srp *sync.RequestPool,
+	lazyLoadCache *caching.LazyLoadCache,
 ) util.JSONResponse {
 	var err error
 
@@ -200,6 +202,10 @@ func OnIncomingMessagesRequest(
 	if filter.LazyLoadMembers {
 		membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent)
 		for _, evt := range clientEvents {
+			// Don't add membership events the client should already know about
+			if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached {
+				continue
+			}
 			membership, err := db.GetStateEvent(req.Context(), roomID, gomatrixserverlib.MRoomMember, evt.Sender)
 			if err != nil {
 				util.GetLogger(req.Context()).WithError(err).Error("failed to get membership event for user")
@@ -207,10 +213,11 @@ func OnIncomingMessagesRequest(
 			}
 			if membership != nil {
 				membershipToUser[evt.Sender] = membership
+				lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID())
 			}
 		}
 		for _, evt := range membershipToUser {
-			state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatAll))
+			state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatSync))
 		}
 	}
 
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index be366ba10..4102cf073 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -18,6 +18,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -38,6 +39,7 @@ func Setup(
 	userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient,
 	rsAPI api.RoomserverInternalAPI,
 	cfg *config.SyncAPI,
+	lazyLoadCache *caching.LazyLoadCache,
 ) {
 	v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
 
@@ -51,7 +53,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg, srp)
+		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg, srp, lazyLoadCache)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/user/{userId}/filter",
@@ -89,6 +91,7 @@ func Setup(
 				req, device,
 				rsAPI, syncDB,
 				vars["roomId"], vars["eventId"],
+				lazyLoadCache,
 			)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index 2f9165d91..b2d333f74 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -148,5 +148,5 @@ func AddPublicRoutes(
 		logrus.WithError(err).Panicf("failed to start presence consumer")
 	}
 
-	routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg)
+	routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg, lazyLoadCache)
 }

From 6d78c4d67d5096f5bb2c7cd56f15ab79949edc86 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 22 Apr 2022 14:58:24 +0100
Subject: [PATCH 020/391] Fix retrieving cross-signing signatures in
 `/user/devices/{userId}` (#2368)

* Fix retrieving cross-signing signatures in `/user/devices/{userId}`

We need to know the target device IDs in order to get the signatures and we weren't populating those.

* Fix up signature retrieval

* Fix SQLite

* Always include the target's own signatures as well as the requesting user
---
 federationapi/routing/devices.go              |  3 ++
 keyserver/internal/cross_signing.go           | 21 ++++++-------
 keyserver/internal/internal.go                | 30 ++++++++++++++++---
 keyserver/storage/interface.go                |  2 +-
 .../postgres/cross_signing_sigs_table.go      |  6 ++--
 keyserver/storage/shared/storage.go           |  6 ++--
 .../sqlite3/cross_signing_sigs_table.go       |  8 ++---
 keyserver/storage/tables/interface.go         |  2 +-
 8 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go
index 4cd199960..8890eac4b 100644
--- a/federationapi/routing/devices.go
+++ b/federationapi/routing/devices.go
@@ -43,6 +43,9 @@ func GetUserDevices(
 		},
 	}
 	sigRes := &keyapi.QuerySignaturesResponse{}
+	for _, dev := range res.Devices {
+		sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID))
+	}
 	keyAPI.QuerySignatures(req.Context(), sigReq, sigRes)
 
 	response := gomatrixserverlib.RespUserDevices{
diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go
index 0d083b4ba..2281f4bbf 100644
--- a/keyserver/internal/cross_signing.go
+++ b/keyserver/internal/cross_signing.go
@@ -455,10 +455,10 @@ func (a *KeyInternalAPI) processOtherSignatures(
 func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
 	ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse,
 ) {
-	for userID := range req.UserToDevices {
-		keys, err := a.DB.CrossSigningKeysForUser(ctx, userID)
+	for targetUserID := range req.UserToDevices {
+		keys, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID)
 		if err != nil {
-			logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", userID)
+			logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", targetUserID)
 			continue
 		}
 
@@ -469,9 +469,9 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
 				break
 			}
 
-			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, keyID)
+			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, keyID)
 			if err != nil && err != sql.ErrNoRows {
-				logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", userID, keyID)
+				logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", targetUserID, keyID)
 				continue
 			}
 
@@ -491,7 +491,7 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
 					case req.UserID != "" && originUserID == req.UserID:
 						// Include signatures that we created
 						appendSignature(originUserID, originKeyID, signature)
-					case originUserID == userID:
+					case originUserID == targetUserID:
 						// Include signatures that were created by the person whose key
 						// we are processing
 						appendSignature(originUserID, originKeyID, signature)
@@ -501,13 +501,13 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
 
 			switch keyType {
 			case gomatrixserverlib.CrossSigningKeyPurposeMaster:
-				res.MasterKeys[userID] = key
+				res.MasterKeys[targetUserID] = key
 
 			case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning:
-				res.SelfSigningKeys[userID] = key
+				res.SelfSigningKeys[targetUserID] = key
 
 			case gomatrixserverlib.CrossSigningKeyPurposeUserSigning:
-				res.UserSigningKeys[userID] = key
+				res.UserSigningKeys[targetUserID] = key
 			}
 		}
 	}
@@ -546,7 +546,8 @@ func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySign
 		}
 
 		for _, targetKeyID := range forTargetUser {
-			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetKeyID)
+			// Get own signatures only.
+			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetUserID, targetKeyID)
 			if err != nil && err != sql.ErrNoRows {
 				res.Error = &api.KeyError{
 					Err: fmt.Sprintf("a.DB.CrossSigningSigsForTarget: %s", err),
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index a05476f5f..e70de7671 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -313,9 +313,31 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 	// Finally, append signatures that we know about
 	// TODO: This is horrible because we need to round-trip the signature from
 	// JSON, add the signatures and marshal it again, for some reason?
-	for userID, forUserID := range res.DeviceKeys {
-		for keyID, key := range forUserID {
-			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, gomatrixserverlib.KeyID(keyID))
+
+	for targetUserID, masterKey := range res.MasterKeys {
+		for targetKeyID := range masterKey.Keys {
+			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID)
+			if err != nil {
+				logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
+				continue
+			}
+			if len(sigMap) == 0 {
+				continue
+			}
+			for sourceUserID, forSourceUser := range sigMap {
+				for sourceKeyID, sourceSig := range forSourceUser {
+					if _, ok := masterKey.Signatures[sourceUserID]; !ok {
+						masterKey.Signatures[sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{}
+					}
+					masterKey.Signatures[sourceUserID][sourceKeyID] = sourceSig
+				}
+			}
+		}
+	}
+
+	for targetUserID, forUserID := range res.DeviceKeys {
+		for targetKeyID, key := range forUserID {
+			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID))
 			if err != nil {
 				logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
 				continue
@@ -339,7 +361,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 				}
 			}
 			if js, err := json.Marshal(deviceKey); err == nil {
-				res.DeviceKeys[userID][keyID] = js
+				res.DeviceKeys[targetUserID][targetKeyID] = js
 			}
 		}
 	}
diff --git a/keyserver/storage/interface.go b/keyserver/storage/interface.go
index 16e034776..242e16a06 100644
--- a/keyserver/storage/interface.go
+++ b/keyserver/storage/interface.go
@@ -81,7 +81,7 @@ type Database interface {
 
 	CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error)
 	CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error)
-	CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error)
+	CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error)
 
 	StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error
 	StoreCrossSigningSigsForTarget(ctx context.Context, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error
diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go
index e11853957..40633c05c 100644
--- a/keyserver/storage/postgres/cross_signing_sigs_table.go
+++ b/keyserver/storage/postgres/cross_signing_sigs_table.go
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs (
 
 const selectCrossSigningSigsForTargetSQL = "" +
 	"SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" +
-	" WHERE target_user_id = $1 AND target_key_id = $2"
+	" WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3"
 
 const upsertCrossSigningSigsForTargetSQL = "" +
 	"INSERT INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" +
@@ -72,9 +72,9 @@ func NewPostgresCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, erro
 }
 
 func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
-	ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
+	ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
 ) (r types.CrossSigningSigMap, err error) {
-	rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID)
+	rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetKeyID)
 	if err != nil {
 		return nil, err
 	}
diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go
index 7ba0b3ea1..0e587b5a8 100644
--- a/keyserver/storage/shared/storage.go
+++ b/keyserver/storage/shared/storage.go
@@ -190,7 +190,7 @@ func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) (
 				keyID: key,
 			},
 		}
-		sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, keyID)
+		sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, userID, keyID)
 		if err != nil {
 			continue
 		}
@@ -219,8 +219,8 @@ func (d *Database) CrossSigningKeysDataForUser(ctx context.Context, userID strin
 }
 
 // CrossSigningSigsForTarget returns the signatures for a given user's key ID, if any.
-func (d *Database) CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) {
-	return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, targetUserID, targetKeyID)
+func (d *Database) CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) {
+	return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, originUserID, targetUserID, targetKeyID)
 }
 
 // StoreCrossSigningKeysForUser stores the latest known cross-signing keys for a user.
diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go
index 9abf54363..29ee889fd 100644
--- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go
+++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs (
 
 const selectCrossSigningSigsForTargetSQL = "" +
 	"SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" +
-	" WHERE target_user_id = $1 AND target_key_id = $2"
+	" WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3"
 
 const upsertCrossSigningSigsForTargetSQL = "" +
 	"INSERT OR REPLACE INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" +
@@ -71,13 +71,13 @@ func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error)
 }
 
 func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget(
-	ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
+	ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID,
 ) (r types.CrossSigningSigMap, err error) {
-	rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID)
+	rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetKeyID)
 	if err != nil {
 		return nil, err
 	}
-	defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForTargetStmt: rows.close() failed")
+	defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForOriginTargetStmt: rows.close() failed")
 	r = types.CrossSigningSigMap{}
 	for rows.Next() {
 		var userID string
diff --git a/keyserver/storage/tables/interface.go b/keyserver/storage/tables/interface.go
index f840cd1f3..37a010a7c 100644
--- a/keyserver/storage/tables/interface.go
+++ b/keyserver/storage/tables/interface.go
@@ -64,7 +64,7 @@ type CrossSigningKeys interface {
 }
 
 type CrossSigningSigs interface {
-	SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error)
+	SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error)
 	UpsertCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error
 	DeleteCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) error
 }

From 67fb086c13b6845e6a76ab89d314895306e14e96 Mon Sep 17 00:00:00 2001
From: Till Faelligen <tfaelligen@gmail.com>
Date: Sun, 24 Apr 2022 20:26:20 +0200
Subject: [PATCH 021/391] Update README

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index cbb35ad59..75c827c33 100644
--- a/README.md
+++ b/README.md
@@ -6,11 +6,11 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
 
 - Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse.
 - Reliable: Implements the Matrix specification as written, using the
-   [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as
-   a [brand new Go test suite](https://github.com/matrix-org/complement).
+  [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as
+  a [brand new Go test suite](https://github.com/matrix-org/complement).
 - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments.
 
-As of October 2020, Dendrite has now entered **beta** which means:
+As of October 2020 (current [progress below](#progress)), Dendrite has now entered **beta** which means:
 
 - Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
 - Dendrite has periodic semver releases. We intend to release new versions as we land significant features.
@@ -21,7 +21,7 @@ This does not mean:
 
 - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
 - All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular,
-   presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
+  presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
 - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
@@ -78,7 +78,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
 
-## Progress
+## <a id="progress"></a> Progress
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it

From 446819e4ac405393ae7834107adc5761afce8a34 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 25 Apr 2022 11:56:50 +0200
Subject: [PATCH 022/391] Store the EDU type in the database (#2370)

---
 federationapi/queue/destinationqueue.go      | 1 +
 federationapi/storage/interface.go           | 2 +-
 federationapi/storage/shared/storage_edus.go | 3 ++-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go
index 09814b31f..a5f8c03b9 100644
--- a/federationapi/queue/destinationqueue.go
+++ b/federationapi/queue/destinationqueue.go
@@ -125,6 +125,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 		context.TODO(),
 		oq.destination, // the destination server name
 		receipt,        // NIDs from federationapi_queue_json table
+		event.Type,
 	); err != nil {
 		logrus.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination)
 		return
diff --git a/federationapi/storage/interface.go b/federationapi/storage/interface.go
index 3fa8d1f7a..e3038651b 100644
--- a/federationapi/storage/interface.go
+++ b/federationapi/storage/interface.go
@@ -39,7 +39,7 @@ type Database interface {
 	GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error)
 
 	AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error
-	AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error
+	AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string) error
 
 	CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
 	CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
diff --git a/federationapi/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go
index 6e3c7e367..02a23338f 100644
--- a/federationapi/storage/shared/storage_edus.go
+++ b/federationapi/storage/shared/storage_edus.go
@@ -31,12 +31,13 @@ func (d *Database) AssociateEDUWithDestination(
 	ctx context.Context,
 	serverName gomatrixserverlib.ServerName,
 	receipt *Receipt,
+	eduType string,
 ) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		if err := d.FederationQueueEDUs.InsertQueueEDU(
 			ctx,         // context
 			txn,         // SQL transaction
-			"",          // TODO: EDU type for coalescing
+			eduType,     // EDU type for coalescing
 			serverName,  // destination server name
 			receipt.nid, // NID from the federationapi_queue_json table
 		); err != nil {

From aad81b7b4dcf971508cde266c5ae99e35261bf27 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 25 Apr 2022 14:22:46 +0100
Subject: [PATCH 023/391] Only call key update process functions if there are
 updates, don't send things to ourselves over federation

---
 federationapi/queue/queue.go   |  2 ++
 federationapi/routing/send.go  | 49 +++++++++++++++++++++++++++++-----
 keyserver/internal/internal.go |  9 +++++--
 3 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go
index 5b5481274..c45bbd1d4 100644
--- a/federationapi/queue/queue.go
+++ b/federationapi/queue/queue.go
@@ -210,6 +210,7 @@ func (oqs *OutgoingQueues) SendEvent(
 		destmap[d] = struct{}{}
 	}
 	delete(destmap, oqs.origin)
+	delete(destmap, oqs.signing.ServerName)
 
 	// Check if any of the destinations are prohibited by server ACLs.
 	for destination := range destmap {
@@ -275,6 +276,7 @@ func (oqs *OutgoingQueues) SendEDU(
 		destmap[d] = struct{}{}
 	}
 	delete(destmap, oqs.origin)
+	delete(destmap, oqs.signing.ServerName)
 
 	// There is absolutely no guarantee that the EDU will have a room_id
 	// field, as it is not required by the spec. However, if it *does*
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index f2b902b6f..2c01afb1b 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -124,6 +124,7 @@ func Send(
 	t := txnReq{
 		rsAPI:                  rsAPI,
 		keys:                   keys,
+		ourServerName:          cfg.Matrix.ServerName,
 		federation:             federation,
 		servers:                servers,
 		keyAPI:                 keyAPI,
@@ -183,6 +184,7 @@ type txnReq struct {
 	gomatrixserverlib.Transaction
 	rsAPI                  api.RoomserverInternalAPI
 	keyAPI                 keyapi.KeyInternalAPI
+	ourServerName          gomatrixserverlib.ServerName
 	keys                   gomatrixserverlib.JSONVerifier
 	federation             txnFederationClient
 	roomsMu                *internal.MutexByRoom
@@ -303,6 +305,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res
 	return &gomatrixserverlib.RespSend{PDUs: results}, nil
 }
 
+// nolint:gocyclo
 func (t *txnReq) processEDUs(ctx context.Context) {
 	for _, e := range t.EDUs {
 		eduCountTotal.Inc()
@@ -318,13 +321,11 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal typing event")
 				continue
 			}
-			_, domain, err := gomatrixserverlib.SplitID('@', typingPayload.UserID)
-			if err != nil {
-				util.GetLogger(ctx).WithError(err).Debug("Failed to split domain from typing event sender")
+			if _, serverName, err := gomatrixserverlib.SplitID('@', typingPayload.UserID); err != nil {
 				continue
-			}
-			if domain != t.Origin {
-				util.GetLogger(ctx).Debugf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
+			} else if serverName == t.ourServerName {
+				continue
+			} else if serverName != t.Origin {
 				continue
 			}
 			if err := t.producer.SendTyping(ctx, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil {
@@ -337,6 +338,13 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal send-to-device events")
 				continue
 			}
+			if _, serverName, err := gomatrixserverlib.SplitID('@', directPayload.Sender); err != nil {
+				continue
+			} else if serverName == t.ourServerName {
+				continue
+			} else if serverName != t.Origin {
+				continue
+			}
 			for userID, byUser := range directPayload.Messages {
 				for deviceID, message := range byUser {
 					// TODO: check that the user and the device actually exist here
@@ -405,6 +413,13 @@ func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) e
 		return err
 	}
 	for _, content := range payload.Push {
+		if _, serverName, err := gomatrixserverlib.SplitID('@', content.UserID); err != nil {
+			continue
+		} else if serverName == t.ourServerName {
+			continue
+		} else if serverName != t.Origin {
+			continue
+		}
 		presence, ok := syncTypes.PresenceFromString(content.Presence)
 		if !ok {
 			continue
@@ -424,7 +439,13 @@ func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverli
 		}).Debug("Failed to unmarshal signing key update")
 		return err
 	}
-
+	if _, serverName, err := gomatrixserverlib.SplitID('@', updatePayload.UserID); err != nil {
+		return nil
+	} else if serverName == t.ourServerName {
+		return nil
+	} else if serverName != t.Origin {
+		return nil
+	}
 	keys := gomatrixserverlib.CrossSigningKeys{}
 	if updatePayload.MasterKey != nil {
 		keys.MasterKey = *updatePayload.MasterKey
@@ -450,6 +471,13 @@ func (t *txnReq) processReceiptEvent(ctx context.Context,
 	timestamp gomatrixserverlib.Timestamp,
 	eventIDs []string,
 ) error {
+	if _, serverName, err := gomatrixserverlib.SplitID('@', userID); err != nil {
+		return nil
+	} else if serverName == t.ourServerName {
+		return nil
+	} else if serverName != t.Origin {
+		return nil
+	}
 	// store every event
 	for _, eventID := range eventIDs {
 		if err := t.producer.SendReceipt(ctx, userID, roomID, eventID, receiptType, timestamp); err != nil {
@@ -466,6 +494,13 @@ func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverli
 		util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal device list update event")
 		return
 	}
+	if _, serverName, err := gomatrixserverlib.SplitID('@', payload.UserID); err != nil {
+		return
+	} else if serverName == t.ourServerName {
+		return
+	} else if serverName != t.Origin {
+		return
+	}
 	var inputRes keyapi.InputDeviceListUpdateResponse
 	t.keyAPI.InputDeviceListUpdate(context.Background(), &keyapi.InputDeviceListUpdateRequest{
 		Event: payload,
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index e70de7671..e571c7e56 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -71,8 +71,12 @@ func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyC
 
 func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) {
 	res.KeyErrors = make(map[string]map[string]*api.KeyError)
-	a.uploadLocalDeviceKeys(ctx, req, res)
-	a.uploadOneTimeKeys(ctx, req, res)
+	if len(req.DeviceKeys) > 0 {
+		a.uploadLocalDeviceKeys(ctx, req, res)
+	}
+	if len(req.OneTimeKeys) > 0 {
+		a.uploadOneTimeKeys(ctx, req, res)
+	}
 }
 
 func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformClaimKeysRequest, res *api.PerformClaimKeysResponse) {
@@ -663,6 +667,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per
 		// add the display name field from keysToStore into existingKeys
 		keysToStore = appendDisplayNames(existingKeys, keysToStore)
 	}
+
 	// store the device keys and emit changes
 	err = a.DB.StoreLocalDeviceKeys(ctx, keysToStore)
 	if err != nil {

From e95fc5c5e3e4945949fbc1e9036c599687925a4d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 25 Apr 2022 19:04:46 +0200
Subject: [PATCH 024/391] Use provided filter for account_data (#2372)

* Reuse IncrementalSync, use provided filter

* Inform SyncAPI about newly created push_rules
---
 syncapi/streams/stream_accountdata.go | 35 ++-------------------------
 sytest-blacklist                      |  4 ---
 sytest-whitelist                      |  2 +-
 userapi/internal/api.go               |  7 ++++++
 4 files changed, 10 insertions(+), 38 deletions(-)

diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 105d85260..094c51485 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -30,37 +30,7 @@ func (p *AccountDataStreamProvider) CompleteSync(
 	ctx context.Context,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	dataReq := &userapi.QueryAccountDataRequest{
-		UserID: req.Device.UserID,
-	}
-	dataRes := &userapi.QueryAccountDataResponse{}
-	if err := p.userAPI.QueryAccountData(ctx, dataReq, dataRes); err != nil {
-		req.Log.WithError(err).Error("p.userAPI.QueryAccountData failed")
-		return p.LatestPosition(ctx)
-	}
-	for datatype, databody := range dataRes.GlobalAccountData {
-		req.Response.AccountData.Events = append(
-			req.Response.AccountData.Events,
-			gomatrixserverlib.ClientEvent{
-				Type:    datatype,
-				Content: gomatrixserverlib.RawJSON(databody),
-			},
-		)
-	}
-	for r, j := range req.Response.Rooms.Join {
-		for datatype, databody := range dataRes.RoomAccountData[r] {
-			j.AccountData.Events = append(
-				j.AccountData.Events,
-				gomatrixserverlib.ClientEvent{
-					Type:    datatype,
-					Content: gomatrixserverlib.RawJSON(databody),
-				},
-			)
-			req.Response.Rooms.Join[r] = j
-		}
-	}
-
-	return p.LatestPosition(ctx)
+	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *AccountDataStreamProvider) IncrementalSync(
@@ -72,10 +42,9 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 		From: from,
 		To:   to,
 	}
-	accountDataFilter := gomatrixserverlib.DefaultEventFilter() // TODO: use filter provided in req instead
 
 	dataTypes, err := p.DB.GetAccountDataInRange(
-		ctx, req.Device.UserID, r, &accountDataFilter,
+		ctx, req.Device.UserID, r, &req.Filter.AccountData,
 	)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed")
diff --git a/sytest-blacklist b/sytest-blacklist
index f1bd60db1..713a5b631 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -1,7 +1,3 @@
-# Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug
-
-Latest account data appears in v2 /sync
-
 # Relies on a rejected PL event which will never be accepted into the DAG
 
 # Caused by <https://github.com/matrix-org/sytest/pull/911>
diff --git a/sytest-whitelist b/sytest-whitelist
index 979f12bf6..2052185fe 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -154,7 +154,7 @@ Can add account data
 Can add account data to room
 Can get account data without syncing
 Can get room account data without syncing
-#Latest account data appears in v2 /sync
+Latest account data appears in v2 /sync
 New account data appears in incremental v2 /sync
 Checking local federation server
 Inbound federation can query profile data
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index d1c12f05f..be58e2d8d 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -90,6 +90,13 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P
 		return nil
 	}
 
+	// Inform the SyncAPI about the newly created push_rules
+	if err = a.SyncProducer.SendAccountData(acc.UserID, "", "m.push_rules"); err != nil {
+		util.GetLogger(ctx).WithFields(logrus.Fields{
+			"user_id": acc.UserID,
+		}).WithError(err).Warn("failed to send account data to the SyncAPI")
+	}
+
 	if req.AccountType == api.AccountTypeGuest {
 		res.AccountCreated = true
 		res.Account = acc

From e8ab2154aa57d46bffa2db1e7c1333c2fa40cb21 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 25 Apr 2022 19:05:01 +0200
Subject: [PATCH 025/391] Return M_NOT_FOUND for rejected events (#2371)

* Return M_NOT_FOUND for rejected events

* Add passing tests
---
 federationapi/routing/state.go     |  7 +++++++
 roomserver/api/query.go            |  2 ++
 roomserver/internal/query/query.go | 24 +++++++++++++++++-------
 sytest-whitelist                   |  6 +++++-
 4 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go
index 37cbb9d1e..a202c92c2 100644
--- a/federationapi/routing/state.go
+++ b/federationapi/routing/state.go
@@ -129,6 +129,13 @@ func getState(
 		return nil, nil, &resErr
 	}
 
+	if response.IsRejected {
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Event not found"),
+		}
+	}
+
 	if !response.RoomExists {
 		return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
 	}
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index 8f84edcb5..ef2e6bb57 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -230,6 +230,8 @@ type QueryStateAndAuthChainResponse struct {
 	// The lists will be in an arbitrary order.
 	StateEvents     []*gomatrixserverlib.HeaderedEvent `json:"state_events"`
 	AuthChainEvents []*gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"`
+	// True if the queried event was rejected earlier.
+	IsRejected bool `json:"is_rejected"`
 }
 
 // QueryRoomVersionCapabilitiesRequest asks for the default room version
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 7e4d56684..5b33ec3c3 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -441,11 +441,11 @@ func (r *Queryer) QueryStateAndAuthChain(
 	}
 
 	var stateEvents []*gomatrixserverlib.Event
-	stateEvents, err = r.loadStateAtEventIDs(ctx, info, request.PrevEventIDs)
+	stateEvents, rejected, err := r.loadStateAtEventIDs(ctx, info, request.PrevEventIDs)
 	if err != nil {
 		return err
 	}
-
+	response.IsRejected = rejected
 	response.PrevEventsExist = true
 
 	// add the auth event IDs for the current state events too
@@ -480,15 +480,23 @@ func (r *Queryer) QueryStateAndAuthChain(
 	return err
 }
 
-func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]*gomatrixserverlib.Event, error) {
+func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]*gomatrixserverlib.Event, bool, error) {
 	roomState := state.NewStateResolution(r.DB, roomInfo)
 	prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs)
 	if err != nil {
 		switch err.(type) {
 		case types.MissingEventError:
-			return nil, nil
+			return nil, false, nil
 		default:
-			return nil, err
+			return nil, false, err
+		}
+	}
+	// Currently only used on /state and /state_ids
+	rejected := false
+	for i := range prevStates {
+		if prevStates[i].IsRejected {
+			rejected = true
+			break
 		}
 	}
 
@@ -497,10 +505,12 @@ func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomI
 		ctx, prevStates,
 	)
 	if err != nil {
-		return nil, err
+		return nil, rejected, err
 	}
 
-	return helpers.LoadStateEvents(ctx, r.DB, stateEntries)
+	events, err := helpers.LoadStateEvents(ctx, r.DB, stateEntries)
+
+	return events, rejected, err
 }
 
 type eventsFromIDs func(context.Context, []string) ([]types.Event, error)
diff --git a/sytest-whitelist b/sytest-whitelist
index 2052185fe..5d67aee6c 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -709,4 +709,8 @@ Gapped incremental syncs include all state changes
 Old leaves are present in gapped incremental syncs
 Leaves are present in non-gapped incremental syncs
 Members from the gap are included in gappy incr LL sync
-Presence can be set from sync
\ No newline at end of file
+Presence can be set from sync
+/state returns M_NOT_FOUND for a rejected message event
+/state_ids returns M_NOT_FOUND for a rejected message event
+/state returns M_NOT_FOUND for a rejected state event
+/state_ids returns M_NOT_FOUND for a rejected state event
\ No newline at end of file

From 7df5d69a5b606e226c5d83c23aa9dd90785c0b2d Mon Sep 17 00:00:00 2001
From: Till Faelligen <tfaelligen@gmail.com>
Date: Tue, 26 Apr 2022 08:07:27 +0200
Subject: [PATCH 026/391] Checkout correct branch for Sytest

---
 .github/workflows/dendrite.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 4f337a866..5d60301c7 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -250,6 +250,7 @@ jobs:
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
         API: ${{ matrix.api && 1 }}
+        SYTEST_BRANCH: ${{ github.head_ref }}
     steps:
       - uses: actions/checkout@v2
       - name: Run Sytest

From feac9db43fc459f1efa10424dfc96f8d54b55c64 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 26 Apr 2022 10:28:41 +0200
Subject: [PATCH 027/391] Add transactionsCache to redact endpoint (#2375)

---
 clientapi/routing/redaction.go | 20 +++++++++++++++++++-
 clientapi/routing/routing.go   |  5 +++--
 sytest-whitelist               |  3 ++-
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go
index 01ea818ab..e8d14ce34 100644
--- a/clientapi/routing/redaction.go
+++ b/clientapi/routing/redaction.go
@@ -22,6 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/internal/transactions"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -40,12 +41,21 @@ type redactionResponse struct {
 func SendRedaction(
 	req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
+	txnID *string,
+	txnCache *transactions.Cache,
 ) util.JSONResponse {
 	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
 	if resErr != nil {
 		return *resErr
 	}
 
+	if txnID != nil {
+		// Try to fetch response from transactionsCache
+		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
+			return *res
+		}
+	}
+
 	ev := roomserverAPI.GetEvent(req.Context(), rsAPI, eventID)
 	if ev == nil {
 		return util.JSONResponse{
@@ -124,10 +134,18 @@ func SendRedaction(
 		util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
 		return jsonerror.InternalServerError()
 	}
-	return util.JSONResponse{
+
+	res := util.JSONResponse{
 		Code: 200,
 		JSON: redactionResponse{
 			EventID: e.EventID(),
 		},
 	}
+
+	// Add response to transactionsCache
+	if txnID != nil {
+		txnCache.AddTransaction(device.AccessToken, *txnID, &res)
+	}
+
+	return res
 }
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 37d825b80..f370b4f8c 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -479,7 +479,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
+			return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, nil, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/redact/{eventID}/{txnId}",
@@ -488,7 +488,8 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
+			txnID := vars["txnId"]
+			return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, &txnID, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
diff --git a/sytest-whitelist b/sytest-whitelist
index 5d67aee6c..91304bd71 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -713,4 +713,5 @@ Presence can be set from sync
 /state returns M_NOT_FOUND for a rejected message event
 /state_ids returns M_NOT_FOUND for a rejected message event
 /state returns M_NOT_FOUND for a rejected state event
-/state_ids returns M_NOT_FOUND for a rejected state event
\ No newline at end of file
+/state_ids returns M_NOT_FOUND for a rejected state event
+PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
\ No newline at end of file

From e8be2b234f616c8422372665c845d9a7a1af245f Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 26 Apr 2022 10:53:17 +0200
Subject: [PATCH 028/391] Add heroes to the room summary (#2373)

* Implement room summary heroes

* Add passing tests

* Move MembershipCount to addRoomSummary

* Add comments, close Statement
---
 syncapi/storage/interface.go                  |  1 +
 syncapi/storage/postgres/memberships_table.go | 38 ++++++++++---
 syncapi/storage/shared/syncserver.go          |  4 ++
 syncapi/storage/sqlite3/memberships_table.go  | 51 ++++++++++++++---
 syncapi/storage/tables/interface.go           |  1 +
 syncapi/streams/stream_pdu.go                 | 56 +++++++++++++++----
 syncapi/streams/streams.go                    |  1 +
 sytest-whitelist                              |  5 +-
 8 files changed, 131 insertions(+), 26 deletions(-)

diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 14cb08a52..0fea88da6 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -39,6 +39,7 @@ type Database interface {
 	GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error)
 	RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error)
 	MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error)
+	GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error)
 
 	RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error)
 
diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go
index 39fa656cb..8c049977f 100644
--- a/syncapi/storage/postgres/memberships_table.go
+++ b/syncapi/storage/postgres/memberships_table.go
@@ -19,6 +19,8 @@ import (
 	"database/sql"
 	"fmt"
 
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
@@ -61,9 +63,13 @@ const selectMembershipCountSQL = "" +
 	" SELECT DISTINCT ON (room_id, user_id) room_id, user_id, membership FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 ORDER BY room_id, user_id, stream_pos DESC" +
 	") t WHERE t.membership = $3"
 
+const selectHeroesSQL = "" +
+	"SELECT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5"
+
 type membershipsStatements struct {
 	upsertMembershipStmt      *sql.Stmt
 	selectMembershipCountStmt *sql.Stmt
+	selectHeroesStmt          *sql.Stmt
 }
 
 func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) {
@@ -72,13 +78,11 @@ func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) {
 	if err != nil {
 		return nil, err
 	}
-	if s.upsertMembershipStmt, err = db.Prepare(upsertMembershipSQL); err != nil {
-		return nil, err
-	}
-	if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil {
-		return nil, err
-	}
-	return s, nil
+	return s, sqlutil.StatementList{
+		{&s.upsertMembershipStmt, upsertMembershipSQL},
+		{&s.selectMembershipCountStmt, selectMembershipCountSQL},
+		{&s.selectHeroesStmt, selectHeroesSQL},
+	}.Prepare(db)
 }
 
 func (s *membershipsStatements) UpsertMembership(
@@ -108,3 +112,23 @@ func (s *membershipsStatements) SelectMembershipCount(
 	err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count)
 	return
 }
+
+func (s *membershipsStatements) SelectHeroes(
+	ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string,
+) (heroes []string, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectHeroesStmt)
+	var rows *sql.Rows
+	rows, err = stmt.QueryContext(ctx, roomID, userID, pq.StringArray(memberships))
+	if err != nil {
+		return
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectHeroes: rows.close() failed")
+	var hero string
+	for rows.Next() {
+		if err = rows.Scan(&hero); err != nil {
+			return
+		}
+		heroes = append(heroes, hero)
+	}
+	return heroes, rows.Err()
+}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 2143fd672..3c431db48 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -124,6 +124,10 @@ func (d *Database) MembershipCount(ctx context.Context, roomID, membership strin
 	return d.Memberships.SelectMembershipCount(ctx, nil, roomID, membership, pos)
 }
 
+func (d *Database) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) {
+	return d.Memberships.SelectHeroes(ctx, nil, roomID, userID, memberships)
+}
+
 func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) {
 	return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents)
 }
diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go
index 9f3530ccd..e4daa99c1 100644
--- a/syncapi/storage/sqlite3/memberships_table.go
+++ b/syncapi/storage/sqlite3/memberships_table.go
@@ -18,7 +18,9 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
@@ -61,10 +63,14 @@ const selectMembershipCountSQL = "" +
 	" SELECT * FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 GROUP BY user_id HAVING(max(stream_pos))" +
 	") t WHERE t.membership = $3"
 
+const selectHeroesSQL = "" +
+	"SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership IN ($3) LIMIT 5"
+
 type membershipsStatements struct {
 	db                        *sql.DB
 	upsertMembershipStmt      *sql.Stmt
 	selectMembershipCountStmt *sql.Stmt
+	//selectHeroesStmt          *sql.Stmt - prepared at runtime due to variadic
 }
 
 func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) {
@@ -75,13 +81,11 @@ func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) {
 	if err != nil {
 		return nil, err
 	}
-	if s.upsertMembershipStmt, err = db.Prepare(upsertMembershipSQL); err != nil {
-		return nil, err
-	}
-	if s.selectMembershipCountStmt, err = db.Prepare(selectMembershipCountSQL); err != nil {
-		return nil, err
-	}
-	return s, nil
+	return s, sqlutil.StatementList{
+		{&s.upsertMembershipStmt, upsertMembershipSQL},
+		{&s.selectMembershipCountStmt, selectMembershipCountSQL},
+		// {&s.selectHeroesStmt, selectHeroesSQL}, - prepared at runtime due to variadic
+	}.Prepare(db)
 }
 
 func (s *membershipsStatements) UpsertMembership(
@@ -111,3 +115,36 @@ func (s *membershipsStatements) SelectMembershipCount(
 	err = stmt.QueryRowContext(ctx, roomID, pos, membership).Scan(&count)
 	return
 }
+
+func (s *membershipsStatements) SelectHeroes(
+	ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string,
+) (heroes []string, err error) {
+	stmtSQL := strings.Replace(selectHeroesSQL, "($3)", sqlutil.QueryVariadicOffset(len(memberships), 2), 1)
+	stmt, err := s.db.PrepareContext(ctx, stmtSQL)
+	if err != nil {
+		return
+	}
+	defer internal.CloseAndLogIfError(ctx, stmt, "SelectHeroes: stmt.close() failed")
+	params := []interface{}{
+		roomID, userID,
+	}
+	for _, membership := range memberships {
+		params = append(params, membership)
+	}
+
+	stmt = sqlutil.TxStmt(txn, stmt)
+	var rows *sql.Rows
+	rows, err = stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectHeroes: rows.close() failed")
+	var hero string
+	for rows.Next() {
+		if err = rows.Scan(&hero); err != nil {
+			return
+		}
+		heroes = append(heroes, hero)
+	}
+	return heroes, rows.Err()
+}
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 993e2022b..ac713dd5c 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -170,6 +170,7 @@ type Receipts interface {
 type Memberships interface {
 	UpsertMembership(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error
 	SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error)
+	SelectHeroes(ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string) (heroes []string, err error)
 }
 
 type NotificationData interface {
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index df5fb8e08..0d033095d 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -4,13 +4,16 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"sort"
 	"sync"
 	"time"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/tidwall/gjson"
 	"go.uber.org/atomic"
 )
 
@@ -30,6 +33,7 @@ type PDUStreamProvider struct {
 	workers atomic.Int32
 	// userID+deviceID -> lazy loading cache
 	lazyLoadCache *caching.LazyLoadCache
+	rsAPI         roomserverAPI.RoomserverInternalAPI
 }
 
 func (p *PDUStreamProvider) worker() {
@@ -290,16 +294,11 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		}
 	}
 
-	// Work out how many members are in the room.
-	joinedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Join, latestPosition)
-	invitedCount, _ := p.DB.MembershipCount(ctx, delta.RoomID, gomatrixserverlib.Invite, latestPosition)
-
 	switch delta.Membership {
 	case gomatrixserverlib.Join:
 		jr := types.NewJoinResponse()
 		if hasMembershipChange {
-			jr.Summary.JoinedMemberCount = &joinedCount
-			jr.Summary.InvitedMemberCount = &invitedCount
+			p.addRoomSummary(ctx, jr, delta.RoomID, device.UserID, latestPosition)
 		}
 		jr.Timeline.PrevBatch = &prevBatch
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
@@ -332,6 +331,45 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	return latestPosition, nil
 }
 
+func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) {
+	// Work out how many members are in the room.
+	joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition)
+	invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, latestPosition)
+
+	jr.Summary.JoinedMemberCount = &joinedCount
+	jr.Summary.InvitedMemberCount = &invitedCount
+
+	fetchStates := []gomatrixserverlib.StateKeyTuple{
+		{EventType: gomatrixserverlib.MRoomName},
+		{EventType: gomatrixserverlib.MRoomCanonicalAlias},
+	}
+	// Check if the room has a name or a canonical alias
+	latestState := &roomserverAPI.QueryLatestEventsAndStateResponse{}
+	err := p.rsAPI.QueryLatestEventsAndState(ctx, &roomserverAPI.QueryLatestEventsAndStateRequest{StateToFetch: fetchStates, RoomID: roomID}, latestState)
+	if err != nil {
+		return
+	}
+	// Check if the room has a name or canonical alias, if so, return.
+	for _, ev := range latestState.StateEvents {
+		switch ev.Type() {
+		case gomatrixserverlib.MRoomName:
+			if gjson.GetBytes(ev.Content(), "name").Str != "" {
+				return
+			}
+		case gomatrixserverlib.MRoomCanonicalAlias:
+			if gjson.GetBytes(ev.Content(), "alias").Str != "" {
+				return
+			}
+		}
+	}
+	heroes, err := p.DB.GetRoomHeroes(ctx, roomID, userID, []string{"join", "invite"})
+	if err != nil {
+		return
+	}
+	sort.Strings(heroes)
+	jr.Summary.Heroes = heroes
+}
+
 func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	ctx context.Context,
 	roomID string,
@@ -416,9 +454,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		prevBatch.Decrement()
 	}
 
-	// Work out how many members are in the room.
-	joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, r.From)
-	invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, r.From)
+	p.addRoomSummary(ctx, jr, roomID, device.UserID, r.From)
 
 	// We don't include a device here as we don't need to send down
 	// transaction IDs for complete syncs, but we do it anyway because Sytest demands it for:
@@ -439,8 +475,6 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		}
 	}
 
-	jr.Summary.JoinedMemberCount = &joinedCount
-	jr.Summary.InvitedMemberCount = &invitedCount
 	jr.Timeline.PrevBatch = prevBatch
 	jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 	jr.Timeline.Limited = limited
diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go
index d3195b78f..a18a0cc41 100644
--- a/syncapi/streams/streams.go
+++ b/syncapi/streams/streams.go
@@ -33,6 +33,7 @@ func NewSyncStreamProviders(
 		PDUStreamProvider: &PDUStreamProvider{
 			StreamProvider: StreamProvider{DB: d},
 			lazyLoadCache:  lazyLoadCache,
+			rsAPI:          rsAPI,
 		},
 		TypingStreamProvider: &TypingStreamProvider{
 			StreamProvider: StreamProvider{DB: d},
diff --git a/sytest-whitelist b/sytest-whitelist
index 91304bd71..c9829606f 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -714,4 +714,7 @@ Presence can be set from sync
 /state_ids returns M_NOT_FOUND for a rejected message event
 /state returns M_NOT_FOUND for a rejected state event
 /state_ids returns M_NOT_FOUND for a rejected state event
-PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
\ No newline at end of file
+PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
+Unnamed room comes with a name summary
+Named room comes with just joined member count summary
+Room summary only has 5 heroes
\ No newline at end of file

From 5306c73b008567d855ca548d195abf3dfaf8917c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 26 Apr 2022 13:08:54 +0100
Subject: [PATCH 029/391] Fix bug when uploading device signatures (#2377)

* Find the complete key ID when uploading signatures

* Try that again

* Try splitting the right thing

* Don't do it for device keys

* Refactor `QuerySignatures`

* Revert "Refactor `QuerySignatures`"

This reverts commit c02832a3e92569f64f180dec1555056dc8f8c3e3.

* Both requested key IDs and master/self/user keys

* Fix uniqueness

* Try tweaking GMSL

* Update GMSL again

* Revert "Update GMSL again"

This reverts commit bd6916cc379dd8d9e3f38d979c6550bd658938aa.

* Revert "Try tweaking GMSL"

This reverts commit 2a054524da9d64c6a2a5228262fbba5fde28798c.

* Database migrations
---
 keyserver/internal/cross_signing.go           |  7 ++
 .../postgres/cross_signing_sigs_table.go      |  6 +-
 .../deltas/2022042612000000_xsigning_idx.go   | 52 +++++++++++++
 keyserver/storage/postgres/storage.go         |  1 +
 .../sqlite3/cross_signing_sigs_table.go       |  4 +-
 .../deltas/2022042612000000_xsigning_idx.go   | 76 +++++++++++++++++++
 keyserver/storage/sqlite3/storage.go          |  1 +
 7 files changed, 144 insertions(+), 3 deletions(-)
 create mode 100644 keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go
 create mode 100644 keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go

diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go
index 2281f4bbf..08bbfedb8 100644
--- a/keyserver/internal/cross_signing.go
+++ b/keyserver/internal/cross_signing.go
@@ -362,6 +362,13 @@ func (a *KeyInternalAPI) processSelfSignatures(
 		for targetKeyID, signature := range forTargetUserID {
 			switch sig := signature.CrossSigningBody.(type) {
 			case *gomatrixserverlib.CrossSigningKey:
+				for keyID := range sig.Keys {
+					split := strings.SplitN(string(keyID), ":", 2)
+					if len(split) > 1 && gomatrixserverlib.KeyID(split[1]) == targetKeyID {
+						targetKeyID = keyID // contains the ed25519: or other scheme
+						break
+					}
+				}
 				for originUserID, forOriginUserID := range sig.Signatures {
 					for originKeyID, originSig := range forOriginUserID {
 						if err := a.DB.StoreCrossSigningSigsForTarget(
diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go
index 40633c05c..b101e7ce5 100644
--- a/keyserver/storage/postgres/cross_signing_sigs_table.go
+++ b/keyserver/storage/postgres/cross_signing_sigs_table.go
@@ -33,8 +33,10 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs (
 	target_user_id TEXT NOT NULL,
 	target_key_id TEXT NOT NULL,
 	signature TEXT NOT NULL,
-	PRIMARY KEY (origin_user_id, target_user_id, target_key_id)
+	PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id)
 );
+
+CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id);
 `
 
 const selectCrossSigningSigsForTargetSQL = "" +
@@ -44,7 +46,7 @@ const selectCrossSigningSigsForTargetSQL = "" +
 const upsertCrossSigningSigsForTargetSQL = "" +
 	"INSERT INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" +
 	" VALUES($1, $2, $3, $4, $5)" +
-	" ON CONFLICT (origin_user_id, target_user_id, target_key_id) DO UPDATE SET (origin_key_id, signature) = ($2, $5)"
+	" ON CONFLICT (origin_user_id, origin_key_id, target_user_id, target_key_id) DO UPDATE SET signature = $5"
 
 const deleteCrossSigningSigsForTargetSQL = "" +
 	"DELETE FROM keyserver_cross_signing_sigs WHERE target_user_id=$1 AND target_key_id=$2"
diff --git a/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go b/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go
new file mode 100644
index 000000000..12956e3b4
--- /dev/null
+++ b/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go
@@ -0,0 +1,52 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadFixCrossSigningSignatureIndexes(m *sqlutil.Migrations) {
+	m.AddMigration(UpFixCrossSigningSignatureIndexes, DownFixCrossSigningSignatureIndexes)
+}
+
+func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+		ALTER TABLE keyserver_cross_signing_sigs DROP CONSTRAINT keyserver_cross_signing_sigs_pkey;
+		ALTER TABLE keyserver_cross_signing_sigs ADD PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id);
+
+		CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id);
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+		ALTER TABLE keyserver_cross_signing_sigs DROP CONSTRAINT keyserver_cross_signing_sigs_pkey;
+		ALTER TABLE keyserver_cross_signing_sigs ADD PRIMARY KEY (origin_user_id, target_user_id, target_key_id);
+
+		DROP INDEX IF EXISTS keyserver_cross_signing_sigs_idx;
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}
diff --git a/keyserver/storage/postgres/storage.go b/keyserver/storage/postgres/storage.go
index 136986885..d4c7e2cc7 100644
--- a/keyserver/storage/postgres/storage.go
+++ b/keyserver/storage/postgres/storage.go
@@ -54,6 +54,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error)
 	}
 	m := sqlutil.NewMigrations()
 	deltas.LoadRefactorKeyChanges(m)
+	deltas.LoadFixCrossSigningSignatureIndexes(m)
 	if err = m.RunDeltas(db, dbProperties); err != nil {
 		return nil, err
 	}
diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go
index 29ee889fd..36d562b8a 100644
--- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go
+++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go
@@ -33,8 +33,10 @@ CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs (
 	target_user_id TEXT NOT NULL,
 	target_key_id TEXT NOT NULL,
 	signature TEXT NOT NULL,
-	PRIMARY KEY (origin_user_id, target_user_id, target_key_id)
+	PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id)
 );
+
+CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id);
 `
 
 const selectCrossSigningSigsForTargetSQL = "" +
diff --git a/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go b/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go
new file mode 100644
index 000000000..230e39fef
--- /dev/null
+++ b/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go
@@ -0,0 +1,76 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadFixCrossSigningSignatureIndexes(m *sqlutil.Migrations) {
+	m.AddMigration(UpFixCrossSigningSignatureIndexes, DownFixCrossSigningSignatureIndexes)
+}
+
+func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+		CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs_tmp (
+			origin_user_id TEXT NOT NULL,
+			origin_key_id TEXT NOT NULL,
+			target_user_id TEXT NOT NULL,
+			target_key_id TEXT NOT NULL,
+			signature TEXT NOT NULL,
+			PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id)
+		);
+
+		INSERT INTO keyserver_cross_signing_sigs_tmp (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)
+			SELECT origin_user_id, origin_key_id, target_user_id, target_key_id, signature FROM keyserver_cross_signing_sigs;
+
+		DROP TABLE keyserver_cross_signing_sigs;
+		ALTER TABLE keyserver_cross_signing_sigs_tmp RENAME TO keyserver_cross_signing_sigs;
+
+		CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_signing_sigs (origin_user_id, target_user_id, target_key_id);
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+		CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs_tmp (
+			origin_user_id TEXT NOT NULL,
+			origin_key_id TEXT NOT NULL,
+			target_user_id TEXT NOT NULL,
+			target_key_id TEXT NOT NULL,
+			signature TEXT NOT NULL,
+			PRIMARY KEY (origin_user_id, target_user_id, target_key_id)
+		);
+
+		INSERT INTO keyserver_cross_signing_sigs_tmp (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)
+			SELECT origin_user_id, origin_key_id, target_user_id, target_key_id, signature FROM keyserver_cross_signing_sigs;
+
+		DROP TABLE keyserver_cross_signing_sigs;
+		ALTER TABLE keyserver_cross_signing_sigs_tmp RENAME TO keyserver_cross_signing_sigs;
+		
+		DELETE INDEX IF EXISTS keyserver_cross_signing_sigs_idx;
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}
diff --git a/keyserver/storage/sqlite3/storage.go b/keyserver/storage/sqlite3/storage.go
index 0e0adceef..84d4cdf55 100644
--- a/keyserver/storage/sqlite3/storage.go
+++ b/keyserver/storage/sqlite3/storage.go
@@ -53,6 +53,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error)
 
 	m := sqlutil.NewMigrations()
 	deltas.LoadRefactorKeyChanges(m)
+	deltas.LoadFixCrossSigningSignatureIndexes(m)
 	if err = m.RunDeltas(db, dbProperties); err != nil {
 		return nil, err
 	}

From 4c19f22725b8f534163ad37845650005b32172ad Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 26 Apr 2022 15:50:56 +0200
Subject: [PATCH 030/391] Fix account_data not correctly send in a complete
 sync (#2379)

* Return the StreamPosition from the database and not the latest

* Fix linter issue
---
 syncapi/storage/interface.go                   |  2 +-
 syncapi/storage/postgres/account_data_table.go | 18 +++++++++++-------
 syncapi/storage/shared/syncserver.go           |  2 +-
 syncapi/storage/sqlite3/account_data_table.go  | 18 +++++++++++-------
 syncapi/storage/tables/interface.go            |  2 +-
 syncapi/streams/stream_accountdata.go          |  4 ++--
 6 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 0fea88da6..13065fa6b 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -81,7 +81,7 @@ type Database interface {
 	// Returns a map following the format data[roomID] = []dataTypes
 	// If no data is retrieved, returns an empty map
 	// If there was an issue with the retrieval, returns an error
-	GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error)
+	GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, types.StreamPosition, error)
 	// UpsertAccountData keeps track of new or updated account data, by saving the type
 	// of the new/updated data, and the user ID and room ID the data is related to (empty)
 	// room ID means the data isn't specific to any room)
diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go
index 25bdb1da3..22bb4d7fa 100644
--- a/syncapi/storage/postgres/account_data_table.go
+++ b/syncapi/storage/postgres/account_data_table.go
@@ -57,7 +57,7 @@ const insertAccountDataSQL = "" +
 	" RETURNING id"
 
 const selectAccountDataInRangeSQL = "" +
-	"SELECT room_id, type FROM syncapi_account_data_type" +
+	"SELECT id, room_id, type FROM syncapi_account_data_type" +
 	" WHERE user_id = $1 AND id > $2 AND id <= $3" +
 	" AND ( $4::text[] IS NULL OR     type LIKE ANY($4)  )" +
 	" AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" +
@@ -103,7 +103,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	userID string,
 	r types.Range,
 	accountDataEventFilter *gomatrixserverlib.EventFilter,
-) (data map[string][]string, err error) {
+) (data map[string][]string, pos types.StreamPosition, err error) {
 	data = make(map[string][]string)
 
 	rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(),
@@ -116,11 +116,12 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed")
 
-	for rows.Next() {
-		var dataType string
-		var roomID string
+	var dataType string
+	var roomID string
+	var id types.StreamPosition
 
-		if err = rows.Scan(&roomID, &dataType); err != nil {
+	for rows.Next() {
+		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
 			return
 		}
 
@@ -129,8 +130,11 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 		} else {
 			data[roomID] = []string{dataType}
 		}
+		if id > pos {
+			pos = id
+		}
 	}
-	return data, rows.Err()
+	return data, pos, rows.Err()
 }
 
 func (s *accountDataStatements) SelectMaxAccountDataID(
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 3c431db48..69bceb624 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -265,7 +265,7 @@ func (d *Database) DeletePeeks(
 func (d *Database) GetAccountDataInRange(
 	ctx context.Context, userID string, r types.Range,
 	accountDataFilterPart *gomatrixserverlib.EventFilter,
-) (map[string][]string, error) {
+) (map[string][]string, types.StreamPosition, error) {
 	return d.AccountData.SelectAccountDataInRange(ctx, userID, r, accountDataFilterPart)
 }
 
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index 71a098177..e0d97ec32 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -43,7 +43,7 @@ const insertAccountDataSQL = "" +
 
 // further parameters are added by prepareWithFilters
 const selectAccountDataInRangeSQL = "" +
-	"SELECT room_id, type FROM syncapi_account_data_type" +
+	"SELECT id, room_id, type FROM syncapi_account_data_type" +
 	" WHERE user_id = $1 AND id > $2 AND id <= $3"
 
 const selectMaxAccountDataIDSQL = "" +
@@ -95,7 +95,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	userID string,
 	r types.Range,
 	filter *gomatrixserverlib.EventFilter,
-) (data map[string][]string, err error) {
+) (data map[string][]string, pos types.StreamPosition, err error) {
 	data = make(map[string][]string)
 	stmt, params, err := prepareWithFilters(
 		s.db, nil, selectAccountDataInRangeSQL,
@@ -112,11 +112,12 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed")
 
-	for rows.Next() {
-		var dataType string
-		var roomID string
+	var dataType string
+	var roomID string
+	var id types.StreamPosition
 
-		if err = rows.Scan(&roomID, &dataType); err != nil {
+	for rows.Next() {
+		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
 			return
 		}
 
@@ -125,9 +126,12 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 		} else {
 			data[roomID] = []string{dataType}
 		}
+		if id > pos {
+			pos = id
+		}
 	}
 
-	return data, nil
+	return data, pos, nil
 }
 
 func (s *accountDataStatements) SelectMaxAccountDataID(
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index ac713dd5c..32b1c34ef 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -27,7 +27,7 @@ import (
 type AccountData interface {
 	InsertAccountData(ctx context.Context, txn *sql.Tx, userID, roomID, dataType string) (pos types.StreamPosition, err error)
 	// SelectAccountDataInRange returns a map of room ID to a list of `dataType`.
-	SelectAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, err error)
+	SelectAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, pos types.StreamPosition, err error)
 	SelectMaxAccountDataID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 }
 
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 094c51485..99cd4a92a 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -43,7 +43,7 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 		To:   to,
 	}
 
-	dataTypes, err := p.DB.GetAccountDataInRange(
+	dataTypes, pos, err := p.DB.GetAccountDataInRange(
 		ctx, req.Device.UserID, r, &req.Filter.AccountData,
 	)
 	if err != nil {
@@ -95,5 +95,5 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 		}
 	}
 
-	return to
+	return pos
 }

From 6892e0f0e02466be3cac6fc6f17267aeecb5961b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 26 Apr 2022 16:02:21 +0100
Subject: [PATCH 031/391] Start account data ID from `from`

---
 syncapi/storage/postgres/account_data_table.go | 2 +-
 syncapi/storage/sqlite3/account_data_table.go  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go
index 22bb4d7fa..0a7146913 100644
--- a/syncapi/storage/postgres/account_data_table.go
+++ b/syncapi/storage/postgres/account_data_table.go
@@ -118,7 +118,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 
 	var dataType string
 	var roomID string
-	var id types.StreamPosition
+	id := r.From
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index e0d97ec32..d84159ac8 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -114,7 +114,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 
 	var dataType string
 	var roomID string
-	var id types.StreamPosition
+	id := r.From
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {

From f6d07768a82cdea07c56cf4ae463449292fa9fe4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 26 Apr 2022 16:07:13 +0100
Subject: [PATCH 032/391] Fix account data position

---
 syncapi/storage/postgres/account_data_table.go | 3 ++-
 syncapi/storage/sqlite3/account_data_table.go  | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go
index 0a7146913..ec1919fca 100644
--- a/syncapi/storage/postgres/account_data_table.go
+++ b/syncapi/storage/postgres/account_data_table.go
@@ -105,6 +105,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	accountDataEventFilter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
 	data = make(map[string][]string)
+	pos = r.Low()
 
 	rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(),
 		pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)),
@@ -118,7 +119,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 
 	var dataType string
 	var roomID string
-	id := r.From
+	var id types.StreamPosition
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index d84159ac8..2c7272ea8 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -96,6 +96,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	r types.Range,
 	filter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
+	pos = r.Low()
 	data = make(map[string][]string)
 	stmt, params, err := prepareWithFilters(
 		s.db, nil, selectAccountDataInRangeSQL,
@@ -114,7 +115,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 
 	var dataType string
 	var roomID string
-	id := r.From
+	var id types.StreamPosition
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {

From b527e33c16d70ee6f94ac12c077b43283ff1fd86 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 26 Apr 2022 16:58:20 +0100
Subject: [PATCH 033/391] Send all account data on complete sync by default

Squashed commit of the following:

commit 0ec8de57261d573a5f88577aa9d7a1174d3999b9
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Tue Apr 26 16:56:30 2022 +0100

    Select filter onto provided target filter

commit da40b6fffbf5737864b223f49900048f557941f9
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Tue Apr 26 16:48:00 2022 +0100

    Specify other field too

commit ffc0b0801f63bb4d3061b6813e3ce5f3b4c8fbcb
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Tue Apr 26 16:45:44 2022 +0100

    Send as much account data as possible during complete sync
---
 syncapi/routing/filter.go                |  4 ++--
 syncapi/storage/interface.go             |  6 +++---
 syncapi/storage/postgres/filter_table.go | 13 ++++++-------
 syncapi/storage/shared/syncserver.go     |  6 +++---
 syncapi/storage/sqlite3/filter_table.go  | 13 ++++++-------
 syncapi/storage/tables/interface.go      |  2 +-
 syncapi/sync/request.go                  | 12 +++++++++---
 7 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/syncapi/routing/filter.go b/syncapi/routing/filter.go
index baa4d841c..1a10bd649 100644
--- a/syncapi/routing/filter.go
+++ b/syncapi/routing/filter.go
@@ -44,8 +44,8 @@ func GetFilter(
 		return jsonerror.InternalServerError()
 	}
 
-	filter, err := syncDB.GetFilter(req.Context(), localpart, filterID)
-	if err != nil {
+	filter := gomatrixserverlib.DefaultFilter()
+	if err := syncDB.GetFilter(req.Context(), &filter, localpart, filterID); err != nil {
 		//TODO better error handling. This error message is *probably* right,
 		// but if there are obscure db errors, this will also be returned,
 		// even though it is not correct.
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 13065fa6b..43aaa3588 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -125,10 +125,10 @@ type Database interface {
 	// CleanSendToDeviceUpdates removes all send-to-device messages BEFORE the specified
 	// from position, preventing the send-to-device table from growing indefinitely.
 	CleanSendToDeviceUpdates(ctx context.Context, userID, deviceID string, before types.StreamPosition) (err error)
-	// GetFilter looks up the filter associated with a given local user and filter ID.
-	// Returns a filter structure. Otherwise returns an error if no such filter exists
+	// GetFilter looks up the filter associated with a given local user and filter ID
+	// and populates the target filter. Otherwise returns an error if no such filter exists
 	// or if there was an error talking to the database.
-	GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error)
+	GetFilter(ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string) error
 	// PutFilter puts the passed filter into the database.
 	// Returns the filterID as a string. Otherwise returns an error if something
 	// goes wrong.
diff --git a/syncapi/storage/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go
index dfd3d6963..c82ef092f 100644
--- a/syncapi/storage/postgres/filter_table.go
+++ b/syncapi/storage/postgres/filter_table.go
@@ -73,21 +73,20 @@ func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) {
 }
 
 func (s *filterStatements) SelectFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
+	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
+) error {
 	// Retrieve filter from database (stored as canonical JSON)
 	var filterData []byte
 	err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	// Unmarshal JSON into Filter struct
-	filter := gomatrixserverlib.DefaultFilter()
-	if err = json.Unmarshal(filterData, &filter); err != nil {
-		return nil, err
+	if err = json.Unmarshal(filterData, &target); err != nil {
+		return err
 	}
-	return &filter, nil
+	return nil
 }
 
 func (s *filterStatements) InsertFilter(
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 69bceb624..25aca50ae 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -513,9 +513,9 @@ func (d *Database) StreamToTopologicalPosition(
 }
 
 func (d *Database) GetFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
-	return d.Filter.SelectFilter(ctx, localpart, filterID)
+	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
+) error {
+	return d.Filter.SelectFilter(ctx, target, localpart, filterID)
 }
 
 func (d *Database) PutFilter(
diff --git a/syncapi/storage/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go
index 0cfebef2a..6081a48b1 100644
--- a/syncapi/storage/sqlite3/filter_table.go
+++ b/syncapi/storage/sqlite3/filter_table.go
@@ -77,21 +77,20 @@ func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) {
 }
 
 func (s *filterStatements) SelectFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
+	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
+) error {
 	// Retrieve filter from database (stored as canonical JSON)
 	var filterData []byte
 	err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	// Unmarshal JSON into Filter struct
-	filter := gomatrixserverlib.DefaultFilter()
-	if err = json.Unmarshal(filterData, &filter); err != nil {
-		return nil, err
+	if err = json.Unmarshal(filterData, &target); err != nil {
+		return err
 	}
-	return &filter, nil
+	return nil
 }
 
 func (s *filterStatements) InsertFilter(
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 32b1c34ef..4ff4689ed 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -157,7 +157,7 @@ type SendToDevice interface {
 }
 
 type Filter interface {
-	SelectFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error)
+	SelectFilter(ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string) error
 	InsertFilter(ctx context.Context, filter *gomatrixserverlib.Filter, localpart string) (filterID string, err error)
 }
 
diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go
index f04f172d3..c9ee8e4a8 100644
--- a/syncapi/sync/request.go
+++ b/syncapi/sync/request.go
@@ -18,6 +18,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"math"
 	"net/http"
 	"strconv"
 	"time"
@@ -47,6 +48,13 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
 	}
 	// TODO: read from stored filters too
 	filter := gomatrixserverlib.DefaultFilter()
+	if since.IsEmpty() {
+		// Send as much account data down for complete syncs as possible
+		// by default, otherwise clients do weird things while waiting
+		// for the rest of the data to trickle down.
+		filter.AccountData.Limit = math.MaxInt
+		filter.Room.AccountData.Limit = math.MaxInt
+	}
 	filterQuery := req.URL.Query().Get("filter")
 	if filterQuery != "" {
 		if filterQuery[0] == '{' {
@@ -61,11 +69,9 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
 				util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
 				return nil, fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 			}
-			if f, err := syncDB.GetFilter(req.Context(), localpart, filterQuery); err != nil && err != sql.ErrNoRows {
+			if err := syncDB.GetFilter(req.Context(), &filter, localpart, filterQuery); err != nil && err != sql.ErrNoRows {
 				util.GetLogger(req.Context()).WithError(err).Error("syncDB.GetFilter failed")
 				return nil, fmt.Errorf("syncDB.GetFilter: %w", err)
-			} else if f != nil {
-				filter = *f
 			}
 		}
 	}

From 6c5c6d73d771e0ea5cc325e4251bcbfc48b7d55e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 26 Apr 2022 17:05:31 +0100
Subject: [PATCH 034/391] Use a value that is Go 1.16-friendly

---
 syncapi/sync/request.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go
index c9ee8e4a8..9d4740e93 100644
--- a/syncapi/sync/request.go
+++ b/syncapi/sync/request.go
@@ -52,8 +52,8 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
 		// Send as much account data down for complete syncs as possible
 		// by default, otherwise clients do weird things while waiting
 		// for the rest of the data to trickle down.
-		filter.AccountData.Limit = math.MaxInt
-		filter.Room.AccountData.Limit = math.MaxInt
+		filter.AccountData.Limit = math.MaxInt32
+		filter.Room.AccountData.Limit = math.MaxInt32
 	}
 	filterQuery := req.URL.Query().Get("filter")
 	if filterQuery != "" {

From 66b397b3c60c51bba36e4bce858733b2fda26f6a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 11:25:07 +0100
Subject: [PATCH 035/391] Don't create fictitious presence entries (#2381)

* Don't create fictitious presence entries for users that don't have any

* Update whitelist, since that test probably shouldn't be passing

* Fix panics
---
 syncapi/consumers/presence.go              |  9 ++++++++-
 syncapi/storage/postgres/presence_table.go |  3 +++
 syncapi/storage/sqlite3/presence_table.go  |  3 +++
 syncapi/streams/stream_presence.go         | 12 +++++------
 syncapi/sync/requestpool.go                | 23 ++++++++++++----------
 sytest-whitelist                           |  2 --
 6 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go
index b198b2292..6bcca48f4 100644
--- a/syncapi/consumers/presence.go
+++ b/syncapi/consumers/presence.go
@@ -88,6 +88,11 @@ func (s *PresenceConsumer) Start() error {
 			}
 			return
 		}
+		if presence == nil {
+			presence = &types.PresenceInternal{
+				UserID: userID,
+			}
+		}
 
 		deviceRes := api.QueryDevicesResponse{}
 		if err = s.deviceAPI.QueryDevices(s.ctx, &api.QueryDevicesRequest{UserID: userID}, &deviceRes); err != nil {
@@ -106,7 +111,9 @@ func (s *PresenceConsumer) Start() error {
 
 		m.Header.Set(jetstream.UserID, presence.UserID)
 		m.Header.Set("presence", presence.ClientFields.Presence)
-		m.Header.Set("status_msg", *presence.ClientFields.StatusMsg)
+		if presence.ClientFields.StatusMsg != nil {
+			m.Header.Set("status_msg", *presence.ClientFields.StatusMsg)
+		}
 		m.Header.Set("last_active_ts", strconv.Itoa(int(presence.LastActiveTS)))
 
 		if err = msg.RespondMsg(m); err != nil {
diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go
index 49336c4eb..9f1e37f79 100644
--- a/syncapi/storage/postgres/presence_table.go
+++ b/syncapi/storage/postgres/presence_table.go
@@ -127,6 +127,9 @@ func (p *presenceStatements) GetPresenceForUser(
 	}
 	stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt)
 	err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
 	result.ClientFields.Presence = result.Presence.String()
 	return result, err
 }
diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go
index 00b16458d..177a01bf3 100644
--- a/syncapi/storage/sqlite3/presence_table.go
+++ b/syncapi/storage/sqlite3/presence_table.go
@@ -142,6 +142,9 @@ func (p *presenceStatements) GetPresenceForUser(
 	}
 	stmt := sqlutil.TxStmt(txn, p.selectPresenceForUsersStmt)
 	err := stmt.QueryRowContext(ctx, userID).Scan(&result.Presence, &result.ClientFields.StatusMsg, &result.LastActiveTS)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
 	result.ClientFields.Presence = result.Presence.String()
 	return result, err
 }
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 9a6c5c130..614b88d48 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -16,7 +16,6 @@ package streams
 
 import (
 	"context"
-	"database/sql"
 	"encoding/json"
 	"sync"
 
@@ -80,11 +79,10 @@ func (p *PresenceStreamProvider) IncrementalSync(
 				if _, ok := presences[roomUsers[i]]; ok {
 					continue
 				}
+				// Bear in mind that this might return nil, but at least populating
+				// a nil means that there's a map entry so we won't repeat this call.
 				presences[roomUsers[i]], err = p.DB.GetPresence(ctx, roomUsers[i])
 				if err != nil {
-					if err == sql.ErrNoRows {
-						continue
-					}
 					req.Log.WithError(err).Error("unable to query presence for user")
 					return from
 				}
@@ -93,8 +91,10 @@ func (p *PresenceStreamProvider) IncrementalSync(
 	}
 
 	lastPos := to
-	for i := range presences {
-		presence := presences[i]
+	for _, presence := range presences {
+		if presence == nil {
+			continue
+		}
 		// Ignore users we don't share a room with
 		if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) {
 			continue
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 703340997..76d550a65 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -127,14 +127,23 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
 	if !ok { // this should almost never happen
 		return
 	}
+
 	newPresence := types.PresenceInternal{
-		ClientFields: types.PresenceClientResponse{
-			Presence: presenceID.String(),
-		},
 		Presence:     presenceID,
 		UserID:       userID,
 		LastActiveTS: gomatrixserverlib.AsTimestamp(time.Now()),
 	}
+
+	// ensure we also send the current status_msg to federated servers and not nil
+	dbPresence, err := db.GetPresence(context.Background(), userID)
+	if err != nil && err != sql.ErrNoRows {
+		return
+	}
+	if dbPresence != nil {
+		newPresence.ClientFields = dbPresence.ClientFields
+	}
+	newPresence.ClientFields.Presence = presenceID.String()
+
 	defer rp.presence.Store(userID, newPresence)
 	// avoid spamming presence updates when syncing
 	existingPresence, ok := rp.presence.LoadOrStore(userID, newPresence)
@@ -145,13 +154,7 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
 		}
 	}
 
-	// ensure we also send the current status_msg to federated servers and not nil
-	dbPresence, err := db.GetPresence(context.Background(), userID)
-	if err != nil && err != sql.ErrNoRows {
-		return
-	}
-
-	if err := rp.producer.SendPresence(userID, presenceID, dbPresence.ClientFields.StatusMsg); err != nil {
+	if err := rp.producer.SendPresence(userID, presenceID, newPresence.ClientFields.StatusMsg); err != nil {
 		logrus.WithError(err).Error("Unable to publish presence message from sync")
 		return
 	}
diff --git a/sytest-whitelist b/sytest-whitelist
index c9829606f..6af8d89ff 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -681,8 +681,6 @@ GET /presence/:user_id/status fetches initial status
 PUT /presence/:user_id/status updates my presence
 Presence change reports an event to myself
 Existing members see new members' presence
-#Existing members see new member's presence
-Newly joined room includes presence in incremental sync
 Get presence for newly joined members in incremental sync
 User sees their own presence in a sync
 User sees updates to presence from other users in the incremental sync.

From dca4afd2f0871ed53109121dff17048e69cc4935 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 12:03:34 +0100
Subject: [PATCH 036/391] Don't send account data or receipts for
 left/forgotten rooms (#2382)

* Only include account data and receipts for rooms in a complete sync that we care about

* Fix global account data
---
 syncapi/streams/stream_accountdata.go |  6 ++++++
 syncapi/streams/stream_receipt.go     |  6 ++++++
 syncapi/types/provider.go             | 17 +++++++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 99cd4a92a..2cddbcf04 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -53,6 +53,12 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 
 	// Iterate over the rooms
 	for roomID, dataTypes := range dataTypes {
+		// For a complete sync, make sure we're only including this room if
+		// that room was present in the joined rooms.
+		if from == 0 && roomID != "" && !req.IsRoomPresent(roomID) {
+			continue
+		}
+
 		// Request the missing data from the database
 		for _, dataType := range dataTypes {
 			dataReq := userapi.QueryAccountDataRequest{
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 9d7d479a2..f4e84c7d0 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -62,6 +62,12 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 	}
 
 	for roomID, receipts := range receiptsByRoom {
+		// For a complete sync, make sure we're only including this room if
+		// that room was present in the joined rooms.
+		if from == 0 && !req.IsRoomPresent(roomID) {
+			continue
+		}
+
 		jr := *types.NewJoinResponse()
 		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
 			jr = existing
diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go
index e6777f643..a9ea234d0 100644
--- a/syncapi/types/provider.go
+++ b/syncapi/types/provider.go
@@ -25,6 +25,23 @@ type SyncRequest struct {
 	IgnoredUsers IgnoredUsers
 }
 
+func (r *SyncRequest) IsRoomPresent(roomID string) bool {
+	membership, ok := r.Rooms[roomID]
+	if !ok {
+		return false
+	}
+	switch membership {
+	case gomatrixserverlib.Join:
+		return true
+	case gomatrixserverlib.Invite:
+		return true
+	case gomatrixserverlib.Peek:
+		return true
+	default:
+		return false
+	}
+}
+
 type StreamProvider interface {
 	Setup()
 

From 54ff4cf690918886c7e7a59a65ccff970c3aa1fc Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 12:23:55 +0100
Subject: [PATCH 037/391] Don't try to federated-join via ourselves (#2383)

---
 federationapi/internal/perform.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 8cd944346..aac36cc76 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -75,7 +75,7 @@ func (r *FederationInternalAPI) PerformJoin(
 	seenSet := make(map[gomatrixserverlib.ServerName]bool)
 	var uniqueList []gomatrixserverlib.ServerName
 	for _, srv := range request.ServerNames {
-		if seenSet[srv] {
+		if seenSet[srv] || srv == r.cfg.Matrix.ServerName {
 			continue
 		}
 		seenSet[srv] = true

From d7cc187ec00410b949ffae1625835f8ac9f36c29 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 13:36:40 +0100
Subject: [PATCH 038/391] Prevent JetStream from handling OS signals, allow
 running as a Windows service (#2385)

* Prevent JetStream from handling OS signals, allow running as a Windows service (fixes #2374)

* Remove double import
---
 go.mod                  | 1 +
 go.sum                  | 1 +
 setup/base/base.go      | 9 +++++++--
 setup/jetstream/nats.go | 1 +
 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index ba222ed8f..d51c3f75d 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,7 @@ require (
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
+	github.com/kardianos/minwinsvc v1.0.0 // indirect
 	github.com/lib/pq v1.10.5
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
diff --git a/go.sum b/go.sum
index 8bb306a82..f8daca79e 100644
--- a/go.sum
+++ b/go.sum
@@ -721,6 +721,7 @@ github.com/julienschmidt/httprouter v1.1.1-0.20151013225520-77a895ad01eb/go.mod
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA=
 github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc=
 github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=
 github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=
diff --git a/setup/base/base.go b/setup/base/base.go
index 43d613b0c..281153444 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -42,6 +42,7 @@ import (
 	userdb "github.com/matrix-org/dendrite/userapi/storage"
 
 	"github.com/gorilla/mux"
+	"github.com/kardianos/minwinsvc"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	asinthttp "github.com/matrix-org/dendrite/appservice/inthttp"
@@ -462,7 +463,8 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		}()
 	}
 
-	<-b.ProcessContext.WaitForShutdown()
+	minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite)
+	b.WaitForShutdown()
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
@@ -475,7 +477,10 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 func (b *BaseDendrite) WaitForShutdown() {
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	<-sigs
+	select {
+	case <-sigs:
+	case <-b.ProcessContext.WaitForShutdown():
+	}
 	signal.Reset(syscall.SIGINT, syscall.SIGTERM)
 
 	logrus.Warnf("Shutdown signal received")
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 1c8a89e8d..8d5289697 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -44,6 +44,7 @@ func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient
 			StoreDir:        string(cfg.StoragePath),
 			NoSystemAccount: true,
 			MaxPayload:      16 * 1024 * 1024,
+			NoSigs:          true,
 		})
 		if err != nil {
 			panic(err)

From f023cdf8c42cc1a4bb850b478dbbf7d901b5e1bd Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 27 Apr 2022 15:05:49 +0200
Subject: [PATCH 039/391] Add UserAPI storage tests (#2384)

* Add tests for parts of the userapi storage

* Add tests for keybackup

* Add LoginToken tests

* Add OpenID tests

* Add profile tests

* Add pusher tests

* Add ThreePID tests

* Add notification tests

* Add more device tests, fix numeric localpart query

* Fix failing CI

* Fix numeric local part query
---
 go.mod                                     |   1 +
 setup/base/base.go                         |   5 +-
 userapi/storage/interface.go               |  91 ++--
 userapi/storage/postgres/accounts_table.go |   6 +-
 userapi/storage/postgres/devices_table.go  |  22 +-
 userapi/storage/shared/storage.go          |  15 -
 userapi/storage/sqlite3/accounts_table.go  |   8 +-
 userapi/storage/sqlite3/devices_table.go   |  22 +-
 userapi/storage/storage.go                 |   4 +-
 userapi/storage/storage_test.go            | 539 +++++++++++++++++++++
 userapi/storage/storage_wasm.go            |   2 +-
 userapi/userapi_test.go                    |   2 +-
 12 files changed, 640 insertions(+), 77 deletions(-)
 create mode 100644 userapi/storage/storage_test.go

diff --git a/go.mod b/go.mod
index d51c3f75d..a7caadfb5 100644
--- a/go.mod
+++ b/go.mod
@@ -47,6 +47,7 @@ require (
 	github.com/pressly/goose v2.7.0+incompatible
 	github.com/prometheus/client_golang v1.12.1
 	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.7.0
 	github.com/tidwall/gjson v1.14.0
 	github.com/tidwall/sjson v1.2.4
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
diff --git a/setup/base/base.go b/setup/base/base.go
index 281153444..dbc5d2394 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -21,6 +21,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
@@ -56,8 +57,6 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp"
 	"github.com/sirupsen/logrus"
-
-	_ "net/http/pprof"
 )
 
 // BaseDendrite is a base for creating new instances of dendrite. It parses
@@ -273,7 +272,7 @@ func (b *BaseDendrite) PushGatewayHTTPClient() pushgateway.Client {
 // CreateAccountsDB creates a new instance of the accounts database. Should only
 // be called once per component.
 func (b *BaseDendrite) CreateAccountsDB() userdb.Database {
-	db, err := userdb.NewDatabase(
+	db, err := userdb.NewUserAPIDatabase(
 		&b.Cfg.UserAPI.AccountDatabase,
 		b.Cfg.Global.ServerName,
 		b.Cfg.UserAPI.BCryptCost,
diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go
index b15470dd4..a4562cf19 100644
--- a/userapi/storage/interface.go
+++ b/userapi/storage/interface.go
@@ -27,18 +27,24 @@ import (
 type Profile interface {
 	GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error)
 	SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error)
-	SetPassword(ctx context.Context, localpart string, plaintextPassword string) error
 	SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error
 	SetDisplayName(ctx context.Context, localpart string, displayName string) error
 }
 
-type Database interface {
-	Profile
-	GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error)
+type Account interface {
 	// CreateAccount makes a new account with the given login name and password, and creates an empty profile
 	// for this account. If no password is supplied, the account will be a passwordless account. If the
 	// account already exists, it will return nil, ErrUserExists.
 	CreateAccount(ctx context.Context, localpart string, plaintextPassword string, appserviceID string, accountType api.AccountType) (*api.Account, error)
+	GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error)
+	GetNewNumericLocalpart(ctx context.Context) (int64, error)
+	CheckAccountAvailability(ctx context.Context, localpart string) (bool, error)
+	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
+	DeactivateAccount(ctx context.Context, localpart string) (err error)
+	SetPassword(ctx context.Context, localpart string, plaintextPassword string) error
+}
+
+type AccountData interface {
 	SaveAccountData(ctx context.Context, localpart, roomID, dataType string, content json.RawMessage) error
 	GetAccountData(ctx context.Context, localpart string) (global map[string]json.RawMessage, rooms map[string]map[string]json.RawMessage, err error)
 	// GetAccountDataByType returns account data matching a given
@@ -46,26 +52,9 @@ type Database interface {
 	// If no account data could be found, returns nil
 	// Returns an error if there was an issue with the retrieval
 	GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data json.RawMessage, err error)
-	GetNewNumericLocalpart(ctx context.Context) (int64, error)
-	SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error)
-	RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error)
-	GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error)
-	GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error)
-	CheckAccountAvailability(ctx context.Context, localpart string) (bool, error)
-	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
-	DeactivateAccount(ctx context.Context, localpart string) (err error)
-	CreateOpenIDToken(ctx context.Context, token, localpart string) (exp int64, err error)
-	GetOpenIDTokenAttributes(ctx context.Context, token string) (*api.OpenIDTokenAttributes, error)
-
-	// Key backups
-	CreateKeyBackup(ctx context.Context, userID, algorithm string, authData json.RawMessage) (version string, err error)
-	UpdateKeyBackupAuthData(ctx context.Context, userID, version string, authData json.RawMessage) (err error)
-	DeleteKeyBackup(ctx context.Context, userID, version string) (exists bool, err error)
-	GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error)
-	UpsertBackupKeys(ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession) (count int64, etag string, err error)
-	GetBackupKeys(ctx context.Context, version, userID, filterRoomID, filterSessionID string) (result map[string]map[string]api.KeyBackupSession, err error)
-	CountBackupKeys(ctx context.Context, version, userID string) (count int64, err error)
+}
 
+type Device interface {
 	GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error)
 	GetDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error)
 	GetDevicesByLocalpart(ctx context.Context, localpart string) ([]api.Device, error)
@@ -79,11 +68,22 @@ type Database interface {
 	CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error)
 	UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error
 	UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error
-	RemoveDevice(ctx context.Context, deviceID, localpart string) error
 	RemoveDevices(ctx context.Context, localpart string, devices []string) error
 	// RemoveAllDevices deleted all devices for this user. Returns the devices deleted.
 	RemoveAllDevices(ctx context.Context, localpart, exceptDeviceID string) (devices []api.Device, err error)
+}
 
+type KeyBackup interface {
+	CreateKeyBackup(ctx context.Context, userID, algorithm string, authData json.RawMessage) (version string, err error)
+	UpdateKeyBackupAuthData(ctx context.Context, userID, version string, authData json.RawMessage) (err error)
+	DeleteKeyBackup(ctx context.Context, userID, version string) (exists bool, err error)
+	GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error)
+	UpsertBackupKeys(ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession) (count int64, etag string, err error)
+	GetBackupKeys(ctx context.Context, version, userID, filterRoomID, filterSessionID string) (result map[string]map[string]api.KeyBackupSession, err error)
+	CountBackupKeys(ctx context.Context, version, userID string) (count int64, err error)
+}
+
+type LoginToken interface {
 	// CreateLoginToken generates a token, stores and returns it. The lifetime is
 	// determined by the loginTokenLifetime given to the Database constructor.
 	CreateLoginToken(ctx context.Context, data *api.LoginTokenData) (*api.LoginTokenMetadata, error)
@@ -94,21 +94,50 @@ type Database interface {
 	// GetLoginTokenDataByToken returns the data associated with the given token.
 	// May return sql.ErrNoRows.
 	GetLoginTokenDataByToken(ctx context.Context, token string) (*api.LoginTokenData, error)
+}
 
-	InsertNotification(ctx context.Context, localpart, eventID string, pos int64, tweaks map[string]interface{}, n *api.Notification) error
-	DeleteNotificationsUpTo(ctx context.Context, localpart, roomID string, pos int64) (affected bool, err error)
-	SetNotificationsRead(ctx context.Context, localpart, roomID string, pos int64, b bool) (affected bool, err error)
-	GetNotifications(ctx context.Context, localpart string, fromID int64, limit int, filter tables.NotificationFilter) ([]*api.Notification, int64, error)
-	GetNotificationCount(ctx context.Context, localpart string, filter tables.NotificationFilter) (int64, error)
-	GetRoomNotificationCounts(ctx context.Context, localpart, roomID string) (total int64, highlight int64, _ error)
-	DeleteOldNotifications(ctx context.Context) error
+type OpenID interface {
+	CreateOpenIDToken(ctx context.Context, token, userID string) (exp int64, err error)
+	GetOpenIDTokenAttributes(ctx context.Context, token string) (*api.OpenIDTokenAttributes, error)
+}
 
+type Pusher interface {
 	UpsertPusher(ctx context.Context, p api.Pusher, localpart string) error
 	GetPushers(ctx context.Context, localpart string) ([]api.Pusher, error)
 	RemovePusher(ctx context.Context, appid, pushkey, localpart string) error
 	RemovePushers(ctx context.Context, appid, pushkey string) error
 }
 
+type ThreePID interface {
+	SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error)
+	RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error)
+	GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error)
+	GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error)
+}
+
+type Notification interface {
+	InsertNotification(ctx context.Context, localpart, eventID string, pos int64, tweaks map[string]interface{}, n *api.Notification) error
+	DeleteNotificationsUpTo(ctx context.Context, localpart, roomID string, pos int64) (affected bool, err error)
+	SetNotificationsRead(ctx context.Context, localpart, roomID string, pos int64, read bool) (affected bool, err error)
+	GetNotifications(ctx context.Context, localpart string, fromID int64, limit int, filter tables.NotificationFilter) ([]*api.Notification, int64, error)
+	GetNotificationCount(ctx context.Context, localpart string, filter tables.NotificationFilter) (int64, error)
+	GetRoomNotificationCounts(ctx context.Context, localpart, roomID string) (total int64, highlight int64, _ error)
+	DeleteOldNotifications(ctx context.Context) error
+}
+
+type Database interface {
+	Account
+	AccountData
+	Device
+	KeyBackup
+	LoginToken
+	Notification
+	OpenID
+	Profile
+	Pusher
+	ThreePID
+}
+
 // Err3PIDInUse is the error returned when trying to save an association involving
 // a third-party identifier which is already associated to a local user.
 var Err3PIDInUse = errors.New("this third-party identifier is already in use")
diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go
index 92311d56d..f86812f17 100644
--- a/userapi/storage/postgres/accounts_table.go
+++ b/userapi/storage/postgres/accounts_table.go
@@ -47,8 +47,6 @@ CREATE TABLE IF NOT EXISTS account_accounts (
     -- TODO:
     -- upgraded_ts, devices, any email reset stuff?
 );
--- Create sequence for autogenerated numeric usernames
-CREATE SEQUENCE IF NOT EXISTS numeric_username_seq START 1;
 `
 
 const insertAccountSQL = "" +
@@ -67,7 +65,7 @@ const selectPasswordHashSQL = "" +
 	"SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE"
 
 const selectNewNumericLocalpartSQL = "" +
-	"SELECT nextval('numeric_username_seq')"
+	"SELECT COALESCE(MAX(localpart::integer), 0) FROM account_accounts WHERE localpart ~ '^[0-9]*$'"
 
 type accountsStatements struct {
 	insertAccountStmt             *sql.Stmt
@@ -178,5 +176,5 @@ func (s *accountsStatements) SelectNewNumericLocalpart(
 		stmt = sqlutil.TxStmt(txn, stmt)
 	}
 	err = stmt.QueryRowContext(ctx).Scan(&id)
-	return
+	return id + 1, err
 }
diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go
index 7bc5dc69b..fe8c54e04 100644
--- a/userapi/storage/postgres/devices_table.go
+++ b/userapi/storage/postgres/devices_table.go
@@ -78,7 +78,7 @@ const selectDeviceByIDSQL = "" +
 	"SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2"
 
 const selectDevicesByLocalpartSQL = "" +
-	"SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2"
+	"SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC"
 
 const updateDeviceNameSQL = "" +
 	"UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3"
@@ -93,7 +93,7 @@ const deleteDevicesSQL = "" +
 	"DELETE FROM device_devices WHERE localpart = $1 AND device_id = ANY($2)"
 
 const selectDevicesByIDSQL = "" +
-	"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)"
+	"SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id = ANY($1) ORDER BY last_seen_ts DESC"
 
 const updateDeviceLastSeen = "" +
 	"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
@@ -235,16 +235,20 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByID: rows.close() failed")
 	var devices []api.Device
+	var dev api.Device
+	var localpart string
+	var lastseents sql.NullInt64
+	var displayName sql.NullString
 	for rows.Next() {
-		var dev api.Device
-		var localpart string
-		var displayName sql.NullString
-		if err := rows.Scan(&dev.ID, &localpart, &displayName); err != nil {
+		if err := rows.Scan(&dev.ID, &localpart, &displayName, &lastseents); err != nil {
 			return nil, err
 		}
 		if displayName.Valid {
 			dev.DisplayName = displayName.String
 		}
+		if lastseents.Valid {
+			dev.LastSeenTS = lastseents.Int64
+		}
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
 		devices = append(devices, dev)
 	}
@@ -262,10 +266,10 @@ func (s *devicesStatements) SelectDevicesByLocalpart(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed")
 
+	var dev api.Device
+	var lastseents sql.NullInt64
+	var id, displayname, ip, useragent sql.NullString
 	for rows.Next() {
-		var dev api.Device
-		var lastseents sql.NullInt64
-		var id, displayname, ip, useragent sql.NullString
 		err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent)
 		if err != nil {
 			return devices, err
diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index 72ae96ecc..f7212e030 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -577,21 +577,6 @@ func (d *Database) UpdateDevice(
 	})
 }
 
-// RemoveDevice revokes a device by deleting the entry in the database
-// matching with the given device ID and user ID localpart.
-// If the device doesn't exist, it will not return an error
-// If something went wrong during the deletion, it will return the SQL error.
-func (d *Database) RemoveDevice(
-	ctx context.Context, deviceID, localpart string,
-) error {
-	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		if err := d.Devices.DeleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows {
-			return err
-		}
-		return nil
-	})
-}
-
 // RemoveDevices revokes one or more devices by deleting the entry in the database
 // matching with the given device IDs and user ID localpart.
 // If the devices don't exist, it will not return an error
diff --git a/userapi/storage/sqlite3/accounts_table.go b/userapi/storage/sqlite3/accounts_table.go
index e6c37e58e..6c5fe3071 100644
--- a/userapi/storage/sqlite3/accounts_table.go
+++ b/userapi/storage/sqlite3/accounts_table.go
@@ -65,7 +65,7 @@ const selectPasswordHashSQL = "" +
 	"SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = 0"
 
 const selectNewNumericLocalpartSQL = "" +
-	"SELECT COUNT(localpart) FROM account_accounts"
+	"SELECT COALESCE(MAX(CAST(localpart AS INT)), 0) FROM account_accounts WHERE CAST(localpart AS INT) <> 0"
 
 type accountsStatements struct {
 	db                            *sql.DB
@@ -121,6 +121,7 @@ func (s *accountsStatements) InsertAccount(
 		UserID:       userutil.MakeUserID(localpart, s.serverName),
 		ServerName:   s.serverName,
 		AppServiceID: appserviceID,
+		AccountType:  accountType,
 	}, nil
 }
 
@@ -177,5 +178,8 @@ func (s *accountsStatements) SelectNewNumericLocalpart(
 		stmt = sqlutil.TxStmt(txn, stmt)
 	}
 	err = stmt.QueryRowContext(ctx).Scan(&id)
-	return
+	if err == sql.ErrNoRows {
+		return 1, nil
+	}
+	return id + 1, err
 }
diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go
index 423640e90..7860bd6a2 100644
--- a/userapi/storage/sqlite3/devices_table.go
+++ b/userapi/storage/sqlite3/devices_table.go
@@ -63,7 +63,7 @@ const selectDeviceByIDSQL = "" +
 	"SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2"
 
 const selectDevicesByLocalpartSQL = "" +
-	"SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2"
+	"SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC"
 
 const updateDeviceNameSQL = "" +
 	"UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3"
@@ -78,7 +78,7 @@ const deleteDevicesSQL = "" +
 	"DELETE FROM device_devices WHERE localpart = $1 AND device_id IN ($2)"
 
 const selectDevicesByIDSQL = "" +
-	"SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)"
+	"SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id IN ($1) ORDER BY last_seen_ts DESC"
 
 const updateDeviceLastSeen = "" +
 	"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
@@ -235,10 +235,10 @@ func (s *devicesStatements) SelectDevicesByLocalpart(
 		return devices, err
 	}
 
+	var dev api.Device
+	var lastseents sql.NullInt64
+	var id, displayname, ip, useragent sql.NullString
 	for rows.Next() {
-		var dev api.Device
-		var lastseents sql.NullInt64
-		var id, displayname, ip, useragent sql.NullString
 		err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent)
 		if err != nil {
 			return devices, err
@@ -279,16 +279,20 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByID: rows.close() failed")
 	var devices []api.Device
+	var dev api.Device
+	var localpart string
+	var displayName sql.NullString
+	var lastseents sql.NullInt64
 	for rows.Next() {
-		var dev api.Device
-		var localpart string
-		var displayName sql.NullString
-		if err := rows.Scan(&dev.ID, &localpart, &displayName); err != nil {
+		if err := rows.Scan(&dev.ID, &localpart, &displayName, &lastseents); err != nil {
 			return nil, err
 		}
 		if displayName.Valid {
 			dev.DisplayName = displayName.String
 		}
+		if lastseents.Valid {
+			dev.LastSeenTS = lastseents.Int64
+		}
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
 		devices = append(devices, dev)
 	}
diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go
index f372fe7dc..faf1ce75c 100644
--- a/userapi/storage/storage.go
+++ b/userapi/storage/storage.go
@@ -28,9 +28,9 @@ import (
 	"github.com/matrix-org/dendrite/userapi/storage/sqlite3"
 )
 
-// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// NewUserAPIDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets postgres connection parameters
-func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) {
+func NewUserAPIDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
 		return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
new file mode 100644
index 000000000..e6c7d35fc
--- /dev/null
+++ b/userapi/storage/storage_test.go
@@ -0,0 +1,539 @@
+package storage_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/pushrules"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/dendrite/userapi/storage/tables"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/bcrypt"
+)
+
+const loginTokenLifetime = time.Minute
+
+var (
+	openIDLifetimeMS = time.Minute.Milliseconds()
+	ctx              = context.Background()
+)
+
+func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := storage.NewUserAPIDatabase(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server")
+	if err != nil {
+		t.Fatalf("NewUserAPIDatabase returned %s", err)
+	}
+	return db, close
+}
+
+// Tests storing and getting account data
+func Test_AccountData(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		alice := test.NewUser()
+		localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
+		assert.NoError(t, err)
+
+		room := test.NewRoom(t, alice)
+		events := room.Events()
+
+		contentRoom := json.RawMessage(fmt.Sprintf(`{"event_id":"%s"}`, events[len(events)-1].EventID()))
+		err = db.SaveAccountData(ctx, localpart, room.ID, "m.fully_read", contentRoom)
+		assert.NoError(t, err, "unable to save account data")
+
+		contentGlobal := json.RawMessage(fmt.Sprintf(`{"recent_rooms":["%s"]}`, room.ID))
+		err = db.SaveAccountData(ctx, localpart, "", "im.vector.setting.breadcrumbs", contentGlobal)
+		assert.NoError(t, err, "unable to save account data")
+
+		accountData, err := db.GetAccountDataByType(ctx, localpart, room.ID, "m.fully_read")
+		assert.NoError(t, err, "unable to get account data by type")
+		assert.Equal(t, contentRoom, accountData)
+
+		globalData, roomData, err := db.GetAccountData(ctx, localpart)
+		assert.NoError(t, err)
+		assert.Equal(t, contentRoom, roomData[room.ID]["m.fully_read"])
+		assert.Equal(t, contentGlobal, globalData["im.vector.setting.breadcrumbs"])
+	})
+}
+
+// Tests the creation of accounts
+func Test_Accounts(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		alice := test.NewUser()
+		aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
+		assert.NoError(t, err)
+
+		accAlice, err := db.CreateAccount(ctx, aliceLocalpart, "testing", "", api.AccountTypeAdmin)
+		assert.NoError(t, err, "failed to create account")
+		// verify the newly create account is the same as returned by CreateAccount
+		var accGet *api.Account
+		accGet, err = db.GetAccountByPassword(ctx, aliceLocalpart, "testing")
+		assert.NoError(t, err, "failed to get account by password")
+		assert.Equal(t, accAlice, accGet)
+		accGet, err = db.GetAccountByLocalpart(ctx, aliceLocalpart)
+		assert.NoError(t, err, "failed to get account by localpart")
+		assert.Equal(t, accAlice, accGet)
+
+		// check account availability
+		available, err := db.CheckAccountAvailability(ctx, aliceLocalpart)
+		assert.NoError(t, err, "failed to checkout account availability")
+		assert.Equal(t, false, available)
+
+		available, err = db.CheckAccountAvailability(ctx, "unusedname")
+		assert.NoError(t, err, "failed to checkout account availability")
+		assert.Equal(t, true, available)
+
+		// get guest account numeric aliceLocalpart
+		first, err := db.GetNewNumericLocalpart(ctx)
+		assert.NoError(t, err, "failed to get new numeric localpart")
+		// Create a new account to verify the numeric localpart is updated
+		_, err = db.CreateAccount(ctx, "", "testing", "", api.AccountTypeGuest)
+		assert.NoError(t, err, "failed to create account")
+		second, err := db.GetNewNumericLocalpart(ctx)
+		assert.NoError(t, err)
+		assert.Greater(t, second, first)
+
+		// update password for alice
+		err = db.SetPassword(ctx, aliceLocalpart, "newPassword")
+		assert.NoError(t, err, "failed to update password")
+		accGet, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword")
+		assert.NoError(t, err, "failed to get account by new password")
+		assert.Equal(t, accAlice, accGet)
+
+		// deactivate account
+		err = db.DeactivateAccount(ctx, aliceLocalpart)
+		assert.NoError(t, err, "failed to deactivate account")
+		// This should fail now, as the account is deactivated
+		_, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword")
+		assert.Error(t, err, "expected an error, got none")
+
+		_, err = db.GetAccountByLocalpart(ctx, "unusename")
+		assert.Error(t, err, "expected an error for non existent localpart")
+	})
+}
+
+func Test_Devices(t *testing.T) {
+	alice := test.NewUser()
+	localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
+	assert.NoError(t, err)
+	deviceID := util.RandomString(8)
+	accessToken := util.RandomString(16)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		deviceWithID, err := db.CreateDevice(ctx, localpart, &deviceID, accessToken, nil, "", "")
+		assert.NoError(t, err, "unable to create deviceWithoutID")
+
+		gotDevice, err := db.GetDeviceByID(ctx, localpart, deviceID)
+		assert.NoError(t, err, "unable to get device by id")
+		assert.Equal(t, deviceWithID.ID, gotDevice.ID) // GetDeviceByID doesn't populate all fields
+
+		gotDeviceAccessToken, err := db.GetDeviceByAccessToken(ctx, accessToken)
+		assert.NoError(t, err, "unable to get device by access token")
+		assert.Equal(t, deviceWithID.ID, gotDeviceAccessToken.ID) // GetDeviceByAccessToken doesn't populate all fields
+
+		// create a device without existing device ID
+		accessToken = util.RandomString(16)
+		deviceWithoutID, err := db.CreateDevice(ctx, localpart, nil, accessToken, nil, "", "")
+		assert.NoError(t, err, "unable to create deviceWithoutID")
+		gotDeviceWithoutID, err := db.GetDeviceByID(ctx, localpart, deviceWithoutID.ID)
+		assert.NoError(t, err, "unable to get device by id")
+		assert.Equal(t, deviceWithoutID.ID, gotDeviceWithoutID.ID) // GetDeviceByID doesn't populate all fields
+
+		// Get devices
+		devices, err := db.GetDevicesByLocalpart(ctx, localpart)
+		assert.NoError(t, err, "unable to get devices by localpart")
+		assert.Equal(t, 2, len(devices))
+		deviceIDs := make([]string, 0, len(devices))
+		for _, dev := range devices {
+			deviceIDs = append(deviceIDs, dev.ID)
+		}
+
+		devices2, err := db.GetDevicesByID(ctx, deviceIDs)
+		assert.NoError(t, err, "unable to get devices by id")
+		assert.Equal(t, devices, devices2)
+
+		// Update device
+		newName := "new display name"
+		err = db.UpdateDevice(ctx, localpart, deviceWithID.ID, &newName)
+		assert.NoError(t, err, "unable to update device displayname")
+		err = db.UpdateDeviceLastSeen(ctx, localpart, deviceWithID.ID, "127.0.0.1")
+		assert.NoError(t, err, "unable to update device last seen")
+
+		deviceWithID.DisplayName = newName
+		deviceWithID.LastSeenIP = "127.0.0.1"
+		deviceWithID.LastSeenTS = int64(gomatrixserverlib.AsTimestamp(time.Now().Truncate(time.Second)))
+		devices, err = db.GetDevicesByLocalpart(ctx, localpart)
+		assert.NoError(t, err, "unable to get device by id")
+		assert.Equal(t, 2, len(devices))
+		assert.Equal(t, deviceWithID.DisplayName, devices[0].DisplayName)
+		assert.Equal(t, deviceWithID.LastSeenIP, devices[0].LastSeenIP)
+		truncatedTime := gomatrixserverlib.Timestamp(devices[0].LastSeenTS).Time().Truncate(time.Second)
+		assert.Equal(t, gomatrixserverlib.Timestamp(deviceWithID.LastSeenTS), gomatrixserverlib.AsTimestamp(truncatedTime))
+
+		// create one more device and remove the devices step by step
+		newDeviceID := util.RandomString(16)
+		accessToken = util.RandomString(16)
+		_, err = db.CreateDevice(ctx, localpart, &newDeviceID, accessToken, nil, "", "")
+		assert.NoError(t, err, "unable to create new device")
+
+		devices, err = db.GetDevicesByLocalpart(ctx, localpart)
+		assert.NoError(t, err, "unable to get device by id")
+		assert.Equal(t, 3, len(devices))
+
+		err = db.RemoveDevices(ctx, localpart, deviceIDs)
+		assert.NoError(t, err, "unable to remove devices")
+		devices, err = db.GetDevicesByLocalpart(ctx, localpart)
+		assert.NoError(t, err, "unable to get device by id")
+		assert.Equal(t, 1, len(devices))
+
+		deleted, err := db.RemoveAllDevices(ctx, localpart, "")
+		assert.NoError(t, err, "unable to remove all devices")
+		assert.Equal(t, 1, len(deleted))
+		assert.Equal(t, newDeviceID, deleted[0].ID)
+	})
+}
+
+func Test_KeyBackup(t *testing.T) {
+	alice := test.NewUser()
+	room := test.NewRoom(t, alice)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		wantAuthData := json.RawMessage("my auth data")
+		wantVersion, err := db.CreateKeyBackup(ctx, alice.ID, "dummyAlgo", wantAuthData)
+		assert.NoError(t, err, "unable to create key backup")
+		// get key backup by version
+		gotVersion, gotAlgo, gotAuthData, _, _, err := db.GetKeyBackup(ctx, alice.ID, wantVersion)
+		assert.NoError(t, err, "unable to get key backup")
+		assert.Equal(t, wantVersion, gotVersion, "backup version mismatch")
+		assert.Equal(t, "dummyAlgo", gotAlgo, "backup algorithm mismatch")
+		assert.Equal(t, wantAuthData, gotAuthData, "backup auth data mismatch")
+
+		// get any key backup
+		gotVersion, gotAlgo, gotAuthData, _, _, err = db.GetKeyBackup(ctx, alice.ID, "")
+		assert.NoError(t, err, "unable to get key backup")
+		assert.Equal(t, wantVersion, gotVersion, "backup version mismatch")
+		assert.Equal(t, "dummyAlgo", gotAlgo, "backup algorithm mismatch")
+		assert.Equal(t, wantAuthData, gotAuthData, "backup auth data mismatch")
+
+		err = db.UpdateKeyBackupAuthData(ctx, alice.ID, wantVersion, json.RawMessage("my updated auth data"))
+		assert.NoError(t, err, "unable to update key backup auth data")
+
+		uploads := []api.InternalKeyBackupSession{
+			{
+				KeyBackupSession: api.KeyBackupSession{
+					IsVerified:  true,
+					SessionData: wantAuthData,
+				},
+				RoomID:    room.ID,
+				SessionID: "1",
+			},
+			{
+				KeyBackupSession: api.KeyBackupSession{},
+				RoomID:           room.ID,
+				SessionID:        "2",
+			},
+		}
+		count, _, err := db.UpsertBackupKeys(ctx, wantVersion, alice.ID, uploads)
+		assert.NoError(t, err, "unable to upsert backup keys")
+		assert.Equal(t, int64(len(uploads)), count, "unexpected backup count")
+
+		// do it again to update a key
+		uploads[1].IsVerified = true
+		count, _, err = db.UpsertBackupKeys(ctx, wantVersion, alice.ID, uploads[1:])
+		assert.NoError(t, err, "unable to upsert backup keys")
+		assert.Equal(t, int64(len(uploads)), count, "unexpected backup count")
+
+		// get backup keys by session id
+		gotBackupKeys, err := db.GetBackupKeys(ctx, wantVersion, alice.ID, room.ID, "1")
+		assert.NoError(t, err, "unable to get backup keys")
+		assert.Equal(t, uploads[0].KeyBackupSession, gotBackupKeys[room.ID]["1"])
+
+		// get backup keys by room id
+		gotBackupKeys, err = db.GetBackupKeys(ctx, wantVersion, alice.ID, room.ID, "")
+		assert.NoError(t, err, "unable to get backup keys")
+		assert.Equal(t, uploads[0].KeyBackupSession, gotBackupKeys[room.ID]["1"])
+
+		gotCount, err := db.CountBackupKeys(ctx, wantVersion, alice.ID)
+		assert.NoError(t, err, "unable to get backup keys count")
+		assert.Equal(t, count, gotCount, "unexpected backup count")
+
+		// finally delete a key
+		exists, err := db.DeleteKeyBackup(ctx, alice.ID, wantVersion)
+		assert.NoError(t, err, "unable to delete key backup")
+		assert.True(t, exists)
+
+		// this key should not exist
+		exists, err = db.DeleteKeyBackup(ctx, alice.ID, "3")
+		assert.NoError(t, err, "unable to delete key backup")
+		assert.False(t, exists)
+	})
+}
+
+func Test_LoginToken(t *testing.T) {
+	alice := test.NewUser()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		// create a new token
+		wantLoginToken := &api.LoginTokenData{UserID: alice.ID}
+
+		gotMetadata, err := db.CreateLoginToken(ctx, wantLoginToken)
+		assert.NoError(t, err, "unable to create login token")
+		assert.NotNil(t, gotMetadata)
+		assert.Equal(t, time.Now().Add(loginTokenLifetime).Truncate(loginTokenLifetime), gotMetadata.Expiration.Truncate(loginTokenLifetime))
+
+		// get the new token
+		gotLoginToken, err := db.GetLoginTokenDataByToken(ctx, gotMetadata.Token)
+		assert.NoError(t, err, "unable to get login token")
+		assert.NotNil(t, gotLoginToken)
+		assert.Equal(t, wantLoginToken, gotLoginToken, "unexpected login token")
+
+		// remove the login token again
+		err = db.RemoveLoginToken(ctx, gotMetadata.Token)
+		assert.NoError(t, err, "unable to remove login token")
+
+		// check if the token was actually deleted
+		_, err = db.GetLoginTokenDataByToken(ctx, gotMetadata.Token)
+		assert.Error(t, err, "expected an error, but got none")
+	})
+}
+
+func Test_OpenID(t *testing.T) {
+	alice := test.NewUser()
+	token := util.RandomString(24)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		expiresAtMS := time.Now().UnixNano()/int64(time.Millisecond) + openIDLifetimeMS
+		expires, err := db.CreateOpenIDToken(ctx, token, alice.ID)
+		assert.NoError(t, err, "unable to create OpenID token")
+		assert.Equal(t, expiresAtMS, expires)
+
+		attributes, err := db.GetOpenIDTokenAttributes(ctx, token)
+		assert.NoError(t, err, "unable to get OpenID token attributes")
+		assert.Equal(t, alice.ID, attributes.UserID)
+		assert.Equal(t, expiresAtMS, attributes.ExpiresAtMS)
+	})
+}
+
+func Test_Profile(t *testing.T) {
+	alice := test.NewUser()
+	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
+	assert.NoError(t, err)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		// create account, which also creates a profile
+		_, err = db.CreateAccount(ctx, aliceLocalpart, "testing", "", api.AccountTypeAdmin)
+		assert.NoError(t, err, "failed to create account")
+
+		gotProfile, err := db.GetProfileByLocalpart(ctx, aliceLocalpart)
+		assert.NoError(t, err, "unable to get profile by localpart")
+		wantProfile := &authtypes.Profile{Localpart: aliceLocalpart}
+		assert.Equal(t, wantProfile, gotProfile)
+
+		// set avatar & displayname
+		wantProfile.DisplayName = "Alice"
+		wantProfile.AvatarURL = "mxc://aliceAvatar"
+		err = db.SetDisplayName(ctx, aliceLocalpart, "Alice")
+		assert.NoError(t, err, "unable to set displayname")
+		err = db.SetAvatarURL(ctx, aliceLocalpart, "mxc://aliceAvatar")
+		assert.NoError(t, err, "unable to set avatar url")
+		// verify profile
+		gotProfile, err = db.GetProfileByLocalpart(ctx, aliceLocalpart)
+		assert.NoError(t, err, "unable to get profile by localpart")
+		assert.Equal(t, wantProfile, gotProfile)
+
+		// search profiles
+		searchRes, err := db.SearchProfiles(ctx, "Alice", 2)
+		assert.NoError(t, err, "unable to search profiles")
+		assert.Equal(t, 1, len(searchRes))
+		assert.Equal(t, *wantProfile, searchRes[0])
+	})
+}
+
+func Test_Pusher(t *testing.T) {
+	alice := test.NewUser()
+	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
+	assert.NoError(t, err)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		appID := util.RandomString(8)
+		var pushKeys []string
+		var gotPushers []api.Pusher
+		for i := 0; i < 2; i++ {
+			pushKey := util.RandomString(8)
+
+			wantPusher := api.Pusher{
+				PushKey:           pushKey,
+				Kind:              api.HTTPKind,
+				AppID:             appID,
+				AppDisplayName:    util.RandomString(8),
+				DeviceDisplayName: util.RandomString(8),
+				ProfileTag:        util.RandomString(8),
+				Language:          util.RandomString(2),
+			}
+			err = db.UpsertPusher(ctx, wantPusher, aliceLocalpart)
+			assert.NoError(t, err, "unable to upsert pusher")
+
+			// check it was actually persisted
+			gotPushers, err = db.GetPushers(ctx, aliceLocalpart)
+			assert.NoError(t, err, "unable to get pushers")
+			assert.Equal(t, i+1, len(gotPushers))
+			assert.Equal(t, wantPusher, gotPushers[i])
+			pushKeys = append(pushKeys, pushKey)
+		}
+
+		// remove single pusher
+		err = db.RemovePusher(ctx, appID, pushKeys[0], aliceLocalpart)
+		assert.NoError(t, err, "unable to remove pusher")
+		gotPushers, err := db.GetPushers(ctx, aliceLocalpart)
+		assert.NoError(t, err, "unable to get pushers")
+		assert.Equal(t, 1, len(gotPushers))
+
+		// remove last pusher
+		err = db.RemovePushers(ctx, appID, pushKeys[1])
+		assert.NoError(t, err, "unable to remove pusher")
+		gotPushers, err = db.GetPushers(ctx, aliceLocalpart)
+		assert.NoError(t, err, "unable to get pushers")
+		assert.Equal(t, 0, len(gotPushers))
+	})
+}
+
+func Test_ThreePID(t *testing.T) {
+	alice := test.NewUser()
+	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
+	assert.NoError(t, err)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		threePID := util.RandomString(8)
+		medium := util.RandomString(8)
+		err = db.SaveThreePIDAssociation(ctx, threePID, aliceLocalpart, medium)
+		assert.NoError(t, err, "unable to save threepid association")
+
+		// get the stored threepid
+		gotLocalpart, err := db.GetLocalpartForThreePID(ctx, threePID, medium)
+		assert.NoError(t, err, "unable to get localpart for threepid")
+		assert.Equal(t, aliceLocalpart, gotLocalpart)
+
+		threepids, err := db.GetThreePIDsForLocalpart(ctx, aliceLocalpart)
+		assert.NoError(t, err, "unable to get threepids for localpart")
+		assert.Equal(t, 1, len(threepids))
+		assert.Equal(t, authtypes.ThreePID{
+			Address: threePID,
+			Medium:  medium,
+		}, threepids[0])
+
+		// remove threepid association
+		err = db.RemoveThreePIDAssociation(ctx, threePID, medium)
+		assert.NoError(t, err, "unexpected error")
+
+		// verify it was deleted
+		threepids, err = db.GetThreePIDsForLocalpart(ctx, aliceLocalpart)
+		assert.NoError(t, err, "unable to get threepids for localpart")
+		assert.Equal(t, 0, len(threepids))
+	})
+}
+
+func Test_Notification(t *testing.T) {
+	alice := test.NewUser()
+	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
+	assert.NoError(t, err)
+	room := test.NewRoom(t, alice)
+	room2 := test.NewRoom(t, alice)
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		// generate some dummy notifications
+		for i := 0; i < 10; i++ {
+			eventID := util.RandomString(16)
+			roomID := room.ID
+			ts := time.Now()
+			if i > 5 {
+				roomID = room2.ID
+				// create some old notifications to test DeleteOldNotifications
+				ts = ts.AddDate(0, -2, 0)
+			}
+			notification := &api.Notification{
+				Actions: []*pushrules.Action{
+					{},
+				},
+				Event: gomatrixserverlib.ClientEvent{
+					Content: gomatrixserverlib.RawJSON("{}"),
+				},
+				Read:   false,
+				RoomID: roomID,
+				TS:     gomatrixserverlib.AsTimestamp(ts),
+			}
+			err = db.InsertNotification(ctx, aliceLocalpart, eventID, int64(i+1), nil, notification)
+			assert.NoError(t, err, "unable to insert notification")
+		}
+
+		// get notifications
+		count, err := db.GetNotificationCount(ctx, aliceLocalpart, tables.AllNotifications)
+		assert.NoError(t, err, "unable to get notification count")
+		assert.Equal(t, int64(10), count)
+		notifs, count, err := db.GetNotifications(ctx, aliceLocalpart, 0, 15, tables.AllNotifications)
+		assert.NoError(t, err, "unable to get notifications")
+		assert.Equal(t, int64(10), count)
+		assert.Equal(t, 10, len(notifs))
+		// ... for a specific room
+		total, _, err := db.GetRoomNotificationCounts(ctx, aliceLocalpart, room2.ID)
+		assert.NoError(t, err, "unable to get notifications for room")
+		assert.Equal(t, int64(4), total)
+
+		// mark notification as read
+		affected, err := db.SetNotificationsRead(ctx, aliceLocalpart, room2.ID, 7, true)
+		assert.NoError(t, err, "unable to set notifications read")
+		assert.True(t, affected)
+
+		// this should delete 2 notifications
+		affected, err = db.DeleteNotificationsUpTo(ctx, aliceLocalpart, room2.ID, 8)
+		assert.NoError(t, err, "unable to set notifications read")
+		assert.True(t, affected)
+
+		total, _, err = db.GetRoomNotificationCounts(ctx, aliceLocalpart, room2.ID)
+		assert.NoError(t, err, "unable to get notifications for room")
+		assert.Equal(t, int64(2), total)
+
+		// delete old notifications
+		err = db.DeleteOldNotifications(ctx)
+		assert.NoError(t, err)
+
+		// this should now return 0 notifications
+		total, _, err = db.GetRoomNotificationCounts(ctx, aliceLocalpart, room2.ID)
+		assert.NoError(t, err, "unable to get notifications for room")
+		assert.Equal(t, int64(0), total)
+	})
+}
diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go
index 779f77568..a8e6f031c 100644
--- a/userapi/storage/storage_wasm.go
+++ b/userapi/storage/storage_wasm.go
@@ -23,7 +23,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(
+func NewUserAPIDatabase(
 	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 	bcryptCost int,
diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index 8c3608bd8..076b4f3c6 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -52,7 +52,7 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts) (api.UserInternalAPI, s
 		MaxOpenConnections: 1,
 		MaxIdleConnections: 1,
 	}
-	accountDB, err := storage.NewDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
+	accountDB, err := storage.NewUserAPIDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
 	if err != nil {
 		t.Fatalf("failed to create account DB: %s", err)
 	}

From 6ee8507955f2b9674649acc928768b1a4d96f7c0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 14:45:51 +0100
Subject: [PATCH 040/391] Correct account data position mapping

---
 syncapi/storage/postgres/account_data_table.go | 10 +++++++---
 syncapi/storage/sqlite3/account_data_table.go  | 11 +++++++----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go
index ec1919fca..7c0d03030 100644
--- a/syncapi/storage/postgres/account_data_table.go
+++ b/syncapi/storage/postgres/account_data_table.go
@@ -105,7 +105,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	accountDataEventFilter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
 	data = make(map[string][]string)
-	pos = r.Low()
+	pos = r.High()
 
 	rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(),
 		pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)),
@@ -120,6 +120,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	var dataType string
 	var roomID string
 	var id types.StreamPosition
+	var highest types.StreamPosition
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
@@ -131,10 +132,13 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 		} else {
 			data[roomID] = []string{dataType}
 		}
-		if id > pos {
-			pos = id
+		if id > highest {
+			highest = id
 		}
 	}
+	if highest < pos {
+		pos = highest
+	}
 	return data, pos, rows.Err()
 }
 
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index 2c7272ea8..1bbfe9c96 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -96,7 +96,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	r types.Range,
 	filter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
-	pos = r.Low()
+	pos = r.High()
 	data = make(map[string][]string)
 	stmt, params, err := prepareWithFilters(
 		s.db, nil, selectAccountDataInRangeSQL,
@@ -116,6 +116,7 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	var dataType string
 	var roomID string
 	var id types.StreamPosition
+	var highest types.StreamPosition
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
@@ -127,11 +128,13 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 		} else {
 			data[roomID] = []string{dataType}
 		}
-		if id > pos {
-			pos = id
+		if id > highest {
+			highest = id
 		}
 	}
-
+	if highest < pos {
+		pos = highest
+	}
 	return data, pos, nil
 }
 

From 655ac3e8fb83e1cb9b670ab420a0f661dc19786e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 14:53:11 +0100
Subject: [PATCH 041/391] Try that again

---
 syncapi/storage/postgres/account_data_table.go | 10 ++++------
 syncapi/storage/sqlite3/account_data_table.go  | 10 ++++------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go
index 7c0d03030..e9c72058b 100644
--- a/syncapi/storage/postgres/account_data_table.go
+++ b/syncapi/storage/postgres/account_data_table.go
@@ -105,7 +105,6 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	accountDataEventFilter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
 	data = make(map[string][]string)
-	pos = r.High()
 
 	rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(),
 		pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)),
@@ -120,7 +119,6 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	var dataType string
 	var roomID string
 	var id types.StreamPosition
-	var highest types.StreamPosition
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
@@ -132,12 +130,12 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 		} else {
 			data[roomID] = []string{dataType}
 		}
-		if id > highest {
-			highest = id
+		if id > pos {
+			pos = id
 		}
 	}
-	if highest < pos {
-		pos = highest
+	if pos == 0 {
+		pos = r.High()
 	}
 	return data, pos, rows.Err()
 }
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index 1bbfe9c96..21a16dcd3 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -96,7 +96,6 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	r types.Range,
 	filter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
-	pos = r.High()
 	data = make(map[string][]string)
 	stmt, params, err := prepareWithFilters(
 		s.db, nil, selectAccountDataInRangeSQL,
@@ -116,7 +115,6 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 	var dataType string
 	var roomID string
 	var id types.StreamPosition
-	var highest types.StreamPosition
 
 	for rows.Next() {
 		if err = rows.Scan(&id, &roomID, &dataType); err != nil {
@@ -128,12 +126,12 @@ func (s *accountDataStatements) SelectAccountDataInRange(
 		} else {
 			data[roomID] = []string{dataType}
 		}
-		if id > highest {
-			highest = id
+		if id > pos {
+			pos = id
 		}
 	}
-	if highest < pos {
-		pos = highest
+	if pos == 0 {
+		pos = r.High()
 	}
 	return data, pos, nil
 }

From cafa2853c5d67b3dd4d247abdd1ad5806f0c951b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 15:01:57 +0100
Subject: [PATCH 042/391] Use process context as base context for all HTTP

---
 setup/base/base.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/setup/base/base.go b/setup/base/base.go
index dbc5d2394..51c43198a 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -346,6 +346,9 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		Addr:         string(externalAddr),
 		WriteTimeout: HTTPServerTimeout,
 		Handler:      externalRouter,
+		BaseContext: func(_ net.Listener) context.Context {
+			return b.ProcessContext.Context()
+		},
 	}
 	internalServ := externalServ
 
@@ -361,6 +364,9 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		internalServ = &http.Server{
 			Addr:    string(internalAddr),
 			Handler: h2c.NewHandler(internalRouter, internalH2S),
+			BaseContext: func(_ net.Listener) context.Context {
+				return b.ProcessContext.Context()
+			},
 		}
 	}
 

From 103795d33a09728d7619e73014d507505ff121e2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 15:06:20 +0100
Subject: [PATCH 043/391] Defer cancel on shutdown context

---
 setup/base/base.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/base/base.go b/setup/base/base.go
index 51c43198a..03ea2ad7e 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -472,7 +472,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	b.WaitForShutdown()
 
 	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
+	defer cancel()
 
 	_ = internalServ.Shutdown(ctx)
 	_ = externalServ.Shutdown(ctx)

From 923f789ca3174a685bd53ce5e64a5e86cabd38cb Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 15:29:49 +0100
Subject: [PATCH 044/391] Fix graceful shutdown

---
 federationapi/queue/destinationqueue.go | 13 ++++++++-----
 federationapi/queue/queue.go            | 13 ++++++-------
 setup/base/base.go                      | 12 ++++++------
 setup/jetstream/helpers.go              | 16 +++++++++++++---
 4 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go
index a5f8c03b9..747940403 100644
--- a/federationapi/queue/destinationqueue.go
+++ b/federationapi/queue/destinationqueue.go
@@ -78,7 +78,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re
 	// this destination queue. We'll then be able to retrieve the PDU
 	// later.
 	if err := oq.db.AssociatePDUWithDestination(
-		context.TODO(),
+		oq.process.Context(),
 		"",             // TODO: remove this, as we don't need to persist the transaction ID
 		oq.destination, // the destination server name
 		receipt,        // NIDs from federationapi_queue_json table
@@ -122,7 +122,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 	// this destination queue. We'll then be able to retrieve the PDU
 	// later.
 	if err := oq.db.AssociateEDUWithDestination(
-		context.TODO(),
+		oq.process.Context(),
 		oq.destination, // the destination server name
 		receipt,        // NIDs from federationapi_queue_json table
 		event.Type,
@@ -177,7 +177,7 @@ func (oq *destinationQueue) getPendingFromDatabase() {
 	// Check to see if there's anything to do for this server
 	// in the database.
 	retrieved := false
-	ctx := context.Background()
+	ctx := oq.process.Context()
 	oq.pendingMutex.Lock()
 	defer oq.pendingMutex.Unlock()
 
@@ -271,6 +271,9 @@ func (oq *destinationQueue) backgroundSend() {
 			// restarted automatically the next time we have an event to
 			// send.
 			return
+		case <-oq.process.Context().Done():
+			// The parent process is shutting down, so stop.
+			return
 		}
 
 		// If we are backing off this server then wait for the
@@ -420,13 +423,13 @@ func (oq *destinationQueue) nextTransaction(
 		// Clean up the transaction in the database.
 		if pduReceipts != nil {
 			//logrus.Infof("Cleaning PDUs %q", pduReceipt.String())
-			if err = oq.db.CleanPDUs(context.Background(), oq.destination, pduReceipts); err != nil {
+			if err = oq.db.CleanPDUs(oq.process.Context(), oq.destination, pduReceipts); err != nil {
 				logrus.WithError(err).Errorf("Failed to clean PDUs for server %q", t.Destination)
 			}
 		}
 		if eduReceipts != nil {
 			//logrus.Infof("Cleaning EDUs %q", eduReceipt.String())
-			if err = oq.db.CleanEDUs(context.Background(), oq.destination, eduReceipts); err != nil {
+			if err = oq.db.CleanEDUs(oq.process.Context(), oq.destination, eduReceipts); err != nil {
 				logrus.WithError(err).Errorf("Failed to clean EDUs for server %q", t.Destination)
 			}
 		}
diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go
index c45bbd1d4..d152886f5 100644
--- a/federationapi/queue/queue.go
+++ b/federationapi/queue/queue.go
@@ -15,7 +15,6 @@
 package queue
 
 import (
-	"context"
 	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
@@ -105,14 +104,14 @@ func NewOutgoingQueues(
 	// Look up which servers we have pending items for and then rehydrate those queues.
 	if !disabled {
 		serverNames := map[gomatrixserverlib.ServerName]struct{}{}
-		if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil {
+		if names, err := db.GetPendingPDUServerNames(process.Context()); err == nil {
 			for _, serverName := range names {
 				serverNames[serverName] = struct{}{}
 			}
 		} else {
 			log.WithError(err).Error("Failed to get PDU server names for destination queue hydration")
 		}
-		if names, err := db.GetPendingEDUServerNames(context.Background()); err == nil {
+		if names, err := db.GetPendingEDUServerNames(process.Context()); err == nil {
 			for _, serverName := range names {
 				serverNames[serverName] = struct{}{}
 			}
@@ -215,7 +214,7 @@ func (oqs *OutgoingQueues) SendEvent(
 	// Check if any of the destinations are prohibited by server ACLs.
 	for destination := range destmap {
 		if api.IsServerBannedFromRoom(
-			context.TODO(),
+			oqs.process.Context(),
 			oqs.rsAPI,
 			ev.RoomID(),
 			destination,
@@ -238,7 +237,7 @@ func (oqs *OutgoingQueues) SendEvent(
 		return fmt.Errorf("json.Marshal: %w", err)
 	}
 
-	nid, err := oqs.db.StoreJSON(context.TODO(), string(headeredJSON))
+	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(headeredJSON))
 	if err != nil {
 		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
 	}
@@ -286,7 +285,7 @@ func (oqs *OutgoingQueues) SendEDU(
 	if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() {
 		for destination := range destmap {
 			if api.IsServerBannedFromRoom(
-				context.TODO(),
+				oqs.process.Context(),
 				oqs.rsAPI,
 				result.Str,
 				destination,
@@ -310,7 +309,7 @@ func (oqs *OutgoingQueues) SendEDU(
 		return fmt.Errorf("json.Marshal: %w", err)
 	}
 
-	nid, err := oqs.db.StoreJSON(context.TODO(), string(ephemeralJSON))
+	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON))
 	if err != nil {
 		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
 	}
diff --git a/setup/base/base.go b/setup/base/base.go
index 03ea2ad7e..e67b034a3 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -469,14 +469,14 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	}
 
 	minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite)
-	b.WaitForShutdown()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	_ = internalServ.Shutdown(ctx)
-	_ = externalServ.Shutdown(ctx)
+	<-b.ProcessContext.WaitForShutdown()
+	logrus.Infof("Stopping HTTP listeners")
+	_ = internalServ.Shutdown(context.Background())
+	_ = externalServ.Shutdown(context.Background())
 	logrus.Infof("Stopped HTTP listeners")
+
+	b.WaitForShutdown()
 }
 
 func (b *BaseDendrite) WaitForShutdown() {
diff --git a/setup/jetstream/helpers.go b/setup/jetstream/helpers.go
index 78cecb6ae..1c07583e9 100644
--- a/setup/jetstream/helpers.go
+++ b/setup/jetstream/helpers.go
@@ -35,6 +35,16 @@ func JetStreamConsumer(
 	}
 	go func() {
 		for {
+			// If the parent context has given up then there's no point in
+			// carrying on doing anything, so stop the listener.
+			select {
+			case <-ctx.Done():
+				if err := sub.Unsubscribe(); err != nil {
+					logrus.WithContext(ctx).Warnf("Failed to unsubscribe %q", durable)
+				}
+				return
+			default:
+			}
 			// The context behaviour here is surprising — we supply a context
 			// so that we can interrupt the fetch if we want, but NATS will still
 			// enforce its own deadline (roughly 5 seconds by default). Therefore
@@ -65,18 +75,18 @@ func JetStreamConsumer(
 				continue
 			}
 			msg := msgs[0]
-			if err = msg.InProgress(); err != nil {
+			if err = msg.InProgress(nats.Context(ctx)); err != nil {
 				logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err))
 				sentry.CaptureException(err)
 				continue
 			}
 			if f(ctx, msg) {
-				if err = msg.AckSync(); err != nil {
+				if err = msg.AckSync(nats.Context(ctx)); err != nil {
 					logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err))
 					sentry.CaptureException(err)
 				}
 			} else {
-				if err = msg.Nak(); err != nil {
+				if err = msg.Nak(nats.Context(ctx)); err != nil {
 					logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err))
 					sentry.CaptureException(err)
 				}

From 34221938ccb1f3a885ac9e5a36b79d3d74850d38 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 27 Apr 2022 16:04:11 +0100
Subject: [PATCH 045/391] Version 0.8.2 (#2386)

* Version 0.8.2

* Correct account data position mapping

* Try that again

* Don't duplicate wait-for-shutdowns
---
 CHANGES.md          | 29 +++++++++++++++++++++++++++++
 internal/version.go |  2 +-
 setup/base/base.go  |  4 +---
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 831a8969d..6278bcba4 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,34 @@
 # Changelog
 
+## Dendrite 0.8.2 (2022-04-27)
+
+### Features
+
+* Lazy-loading has been added to the `/sync` endpoint, which should speed up syncs considerably
+* Filtering has been added to the `/messages` endpoint
+* The room summary now contains "heroes" (up to 5 users in the room) for clients to display when no room name is set
+* The existing lazy-loading caches will now be used by `/messages` and `/context` so that member events will not be sent to clients more times than necessary
+* The account data stream now uses the provided filters
+* The built-in NATS Server has been updated to version 2.8.0
+* The `/state` and `/state_ids` endpoints will now return `M_NOT_FOUND` for rejected events
+* Repeated calls to the `/redact` endpoint will now be idempotent when a transaction ID is given
+* Dendrite should now be able to run as a Windows service under Service Control Manager
+
+### Fixes
+
+* Fictitious presence updates will no longer be created for users which have not sent us presence updates, which should speed up complete syncs considerably
+* Uploading cross-signing device signatures should now be more reliable, fixing a number of bugs with cross-signing
+* All account data should now be sent properly on a complete sync, which should eliminate problems with client settings or key backups appearing to be missing
+* Account data will now be limited correctly on incremental syncs, returning the stream position of the most recent update rather than the latest stream position
+* Account data will not be sent for parted rooms, which should reduce the number of left/forgotten rooms reappearing in clients as empty rooms
+* The TURN username hash has been fixed which should help to resolve some problems when using TURN for voice calls (contributed by [fcwoknhenuxdfiyv](https://github.com/fcwoknhenuxdfiyv))
+* Push rules can no longer be modified using the account data endpoints
+* Querying account availability should now work properly in polylith deployments
+* A number of bugs with sync filters have been fixed
+* A default sync filter will now be used if the request contains a filter ID that does not exist
+* The `pushkey_ts` field is now using seconds instead of milliseconds
+* A race condition when gracefully shutting down has been fixed, so JetStream should no longer cause the process to exit before other Dendrite components are finished shutting down
+
 ## Dendrite 0.8.1 (2022-04-07)
 
 ### Fixes
diff --git a/internal/version.go b/internal/version.go
index 5227a03bf..2477bc9ac 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 1
+	VersionPatch = 2
 	VersionTag   = "" // example: "rc1"
 )
 
diff --git a/setup/base/base.go b/setup/base/base.go
index e67b034a3..7091c6ba5 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -469,14 +469,12 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	}
 
 	minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite)
-
 	<-b.ProcessContext.WaitForShutdown()
+
 	logrus.Infof("Stopping HTTP listeners")
 	_ = internalServ.Shutdown(context.Background())
 	_ = externalServ.Shutdown(context.Background())
 	logrus.Infof("Stopped HTTP listeners")
-
-	b.WaitForShutdown()
 }
 
 func (b *BaseDendrite) WaitForShutdown() {

From 8d69e2f0b897dd1ecd99c7bd348ad8bfc8999c4e Mon Sep 17 00:00:00 2001
From: 0x1a8510f2 <admin@0x1a8510f2.space>
Date: Wed, 27 Apr 2022 20:19:46 +0100
Subject: [PATCH 046/391] Use Go 1.18 to build Docker images (#2391)

Go 1.18 has now been released for a while and the CI already tests Dendrite with Go 1.18
so there should be no issues. Go 1.18 brings some performance improvements for ARM via
the register calling convention so it makes sense to switch to it.
---
 build/docker/Dockerfile.monolith | 4 ++--
 build/docker/Dockerfile.polylith | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
index 0d2a141ad..891a3a9e0 100644
--- a/build/docker/Dockerfile.monolith
+++ b/build/docker/Dockerfile.monolith
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.17-alpine AS base
+FROM docker.io/golang:1.18-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 
@@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
\ No newline at end of file
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
index c266fd480..ffdc35586 100644
--- a/build/docker/Dockerfile.polylith
+++ b/build/docker/Dockerfile.polylith
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.17-alpine AS base
+FROM docker.io/golang:1.18-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 
@@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
\ No newline at end of file
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

From 74259f296f225510e9fbb6c5aae191c3f86c729e Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 27 Apr 2022 21:31:30 +0200
Subject: [PATCH 047/391] Fix #2390 (#2392)

Fix duplicate heroes in `/sync` response.
---
 syncapi/storage/postgres/memberships_table.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go
index 8c049977f..00223c57a 100644
--- a/syncapi/storage/postgres/memberships_table.go
+++ b/syncapi/storage/postgres/memberships_table.go
@@ -64,7 +64,7 @@ const selectMembershipCountSQL = "" +
 	") t WHERE t.membership = $3"
 
 const selectHeroesSQL = "" +
-	"SELECT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5"
+	"SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5"
 
 type membershipsStatements struct {
 	upsertMembershipStmt      *sql.Stmt

From 2ff75b7c806829b211be0c310497728055e898cc Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 28 Apr 2022 11:34:19 +0100
Subject: [PATCH 048/391] Ensure signature map exists (fixes #2393) (#2397)

---
 keyserver/internal/internal.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index e571c7e56..1677cf8e3 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -319,6 +319,9 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 	// JSON, add the signatures and marshal it again, for some reason?
 
 	for targetUserID, masterKey := range res.MasterKeys {
+		if masterKey.Signatures == nil {
+			masterKey.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{}
+		}
 		for targetKeyID := range masterKey.Keys {
 			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID)
 			if err != nil {

From 6deb10f3f61e4da0360a8910f6b5375a2d31f182 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 28 Apr 2022 11:45:56 +0100
Subject: [PATCH 049/391] Don't answer expensive federation requests for rooms
 we no longer belong to (#2398)

This includes `/state`, `/state_ids`, `/get_missing_events` and `/backfill`.

This should fix #2396.
---
 federationapi/routing/backfill.go      |  6 ++++++
 federationapi/routing/eventauth.go     |  6 ++++++
 federationapi/routing/missingevents.go |  6 ++++++
 federationapi/routing/routing.go       | 27 ++++++++++++++++++++++++++
 federationapi/routing/state.go         |  6 ++++++
 5 files changed, 51 insertions(+)

diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go
index 31005209f..82f6cbabf 100644
--- a/federationapi/routing/backfill.go
+++ b/federationapi/routing/backfill.go
@@ -51,6 +51,12 @@ func Backfill(
 		}
 	}
 
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil {
+		return *err
+	}
+
 	// Check if all of the required parameters are there.
 	eIDs, exists = httpReq.URL.Query()["v"]
 	if !exists {
diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go
index 0a03a0cb4..e83cb8ad2 100644
--- a/federationapi/routing/eventauth.go
+++ b/federationapi/routing/eventauth.go
@@ -30,6 +30,12 @@ func GetEventAuth(
 	roomID string,
 	eventID string,
 ) util.JSONResponse {
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil {
+		return *err
+	}
+
 	event, resErr := fetchEvent(ctx, rsAPI, eventID)
 	if resErr != nil {
 		return *resErr
diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go
index dd3df7aa9..b826d69c4 100644
--- a/federationapi/routing/missingevents.go
+++ b/federationapi/routing/missingevents.go
@@ -45,6 +45,12 @@ func GetMissingEvents(
 		}
 	}
 
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(httpReq.Context(), rsAPI, roomID); err != nil {
+		return *err
+	}
+
 	var eventsResponse api.QueryMissingEventsResponse
 	if err := rsAPI.QueryMissingEvents(
 		httpReq.Context(), &api.QueryMissingEventsRequest{
diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index a085ed780..6d24c8b40 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -15,6 +15,8 @@
 package routing
 
 import (
+	"context"
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -24,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -491,3 +494,27 @@ func Setup(
 		}),
 	).Methods(http.MethodGet)
 }
+
+func ErrorIfLocalServerNotInRoom(
+	ctx context.Context,
+	rsAPI api.RoomserverInternalAPI,
+	roomID string,
+) *util.JSONResponse {
+	// Check if we think we're in this room. If we aren't then
+	// we won't waste CPU cycles serving this request.
+	joinedReq := &api.QueryServerJoinedToRoomRequest{
+		RoomID: roomID,
+	}
+	joinedRes := &api.QueryServerJoinedToRoomResponse{}
+	if err := rsAPI.QueryServerJoinedToRoom(ctx, joinedReq, joinedRes); err != nil {
+		res := util.ErrorResponse(err)
+		return &res
+	}
+	if !joinedRes.IsInRoom {
+		return &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(fmt.Sprintf("This server is not joined to room %s", roomID)),
+		}
+	}
+	return nil
+}
diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go
index a202c92c2..e2b67776a 100644
--- a/federationapi/routing/state.go
+++ b/federationapi/routing/state.go
@@ -101,6 +101,12 @@ func getState(
 	roomID string,
 	eventID string,
 ) (stateEvents, authEvents []*gomatrixserverlib.HeaderedEvent, errRes *util.JSONResponse) {
+	// If we don't think we belong to this room then don't waste the effort
+	// responding to expensive requests for it.
+	if err := ErrorIfLocalServerNotInRoom(ctx, rsAPI, roomID); err != nil {
+		return nil, nil, err
+	}
+
 	event, resErr := fetchEvent(ctx, rsAPI, eventID)
 	if resErr != nil {
 		return nil, nil, resErr

From 65034d1f227de45e88d39ec5a3e83d854e840875 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 28 Apr 2022 11:46:15 +0100
Subject: [PATCH 050/391] Unlist test since it no longer seems to be flakey
 (hopefully?)

---
 sytest-blacklist | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sytest-blacklist b/sytest-blacklist
index 713a5b631..be0826eee 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -48,4 +48,3 @@ Notifications can be viewed with GET /notifications
 # More flakey
 
 If remote user leaves room we no longer receive device updates
-Local device key changes get to remote servers

From 8683ff78b1bee6b7c35e7befb9903c794a17510c Mon Sep 17 00:00:00 2001
From: Till Faelligen <tfaelligen@gmail.com>
Date: Thu, 28 Apr 2022 15:06:34 +0200
Subject: [PATCH 051/391] Make tests more reliable

---
 userapi/storage/postgres/devices_table.go | 13 ++++++++++---
 userapi/storage/sqlite3/devices_table.go  | 13 ++++++++++---
 userapi/storage/storage_test.go           |  8 ++++----
 3 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go
index fe8c54e04..6c777982f 100644
--- a/userapi/storage/postgres/devices_table.go
+++ b/userapi/storage/postgres/devices_table.go
@@ -75,7 +75,7 @@ const selectDeviceByTokenSQL = "" +
 	"SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1"
 
 const selectDeviceByIDSQL = "" +
-	"SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2"
+	"SELECT display_name, last_seen_ts, ip FROM device_devices WHERE localpart = $1 and device_id = $2"
 
 const selectDevicesByLocalpartSQL = "" +
 	"SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC"
@@ -215,15 +215,22 @@ func (s *devicesStatements) SelectDeviceByID(
 	ctx context.Context, localpart, deviceID string,
 ) (*api.Device, error) {
 	var dev api.Device
-	var displayName sql.NullString
+	var displayName, ip sql.NullString
+	var lastseenTS sql.NullInt64
 	stmt := s.selectDeviceByIDStmt
-	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName)
+	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName, &lastseenTS, &ip)
 	if err == nil {
 		dev.ID = deviceID
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
 		if displayName.Valid {
 			dev.DisplayName = displayName.String
 		}
+		if lastseenTS.Valid {
+			dev.LastSeenTS = lastseenTS.Int64
+		}
+		if ip.Valid {
+			dev.LastSeenIP = ip.String
+		}
 	}
 	return &dev, err
 }
diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go
index 7860bd6a2..b86ed1cc2 100644
--- a/userapi/storage/sqlite3/devices_table.go
+++ b/userapi/storage/sqlite3/devices_table.go
@@ -60,7 +60,7 @@ const selectDeviceByTokenSQL = "" +
 	"SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1"
 
 const selectDeviceByIDSQL = "" +
-	"SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2"
+	"SELECT display_name, last_seen_ts, ip FROM device_devices WHERE localpart = $1 and device_id = $2"
 
 const selectDevicesByLocalpartSQL = "" +
 	"SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC"
@@ -212,15 +212,22 @@ func (s *devicesStatements) SelectDeviceByID(
 	ctx context.Context, localpart, deviceID string,
 ) (*api.Device, error) {
 	var dev api.Device
-	var displayName sql.NullString
+	var displayName, ip sql.NullString
 	stmt := s.selectDeviceByIDStmt
-	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName)
+	var lastseenTS sql.NullInt64
+	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&displayName, &lastseenTS, &ip)
 	if err == nil {
 		dev.ID = deviceID
 		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
 		if displayName.Valid {
 			dev.DisplayName = displayName.String
 		}
+		if lastseenTS.Valid {
+			dev.LastSeenTS = lastseenTS.Int64
+		}
+		if ip.Valid {
+			dev.LastSeenIP = ip.String
+		}
 	}
 	return &dev, err
 }
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index e6c7d35fc..2eb57d0bc 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -180,12 +180,12 @@ func Test_Devices(t *testing.T) {
 		deviceWithID.DisplayName = newName
 		deviceWithID.LastSeenIP = "127.0.0.1"
 		deviceWithID.LastSeenTS = int64(gomatrixserverlib.AsTimestamp(time.Now().Truncate(time.Second)))
-		devices, err = db.GetDevicesByLocalpart(ctx, localpart)
+		gotDevice, err = db.GetDeviceByID(ctx, localpart, deviceWithID.ID)
 		assert.NoError(t, err, "unable to get device by id")
 		assert.Equal(t, 2, len(devices))
-		assert.Equal(t, deviceWithID.DisplayName, devices[0].DisplayName)
-		assert.Equal(t, deviceWithID.LastSeenIP, devices[0].LastSeenIP)
-		truncatedTime := gomatrixserverlib.Timestamp(devices[0].LastSeenTS).Time().Truncate(time.Second)
+		assert.Equal(t, deviceWithID.DisplayName, gotDevice.DisplayName)
+		assert.Equal(t, deviceWithID.LastSeenIP, gotDevice.LastSeenIP)
+		truncatedTime := gomatrixserverlib.Timestamp(gotDevice.LastSeenTS).Time().Truncate(time.Second)
 		assert.Equal(t, gomatrixserverlib.Timestamp(deviceWithID.LastSeenTS), gomatrixserverlib.AsTimestamp(truncatedTime))
 
 		// create one more device and remove the devices step by step

From 21ee5b36a41f2cb3960f63ef6f19106d36312aae Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Apr 2022 16:12:40 +0200
Subject: [PATCH 052/391] Limit presence in `/sync` responses (#2394)

* Use filter and limit presence count

* More limiting

* More limiting

* Fix unit test

* Also limit presence by last_active_ts

* Update query, use "from" as the initial lastPos

* Get 1000 presence events, they are filtered later

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 syncapi/storage/interface.go               |  2 +-
 syncapi/storage/postgres/presence_table.go |  9 ++++++---
 syncapi/storage/shared/syncserver.go       |  4 ++--
 syncapi/storage/sqlite3/presence_table.go  | 10 ++++++----
 syncapi/storage/tables/interface.go        |  2 +-
 syncapi/streams/stream_presence.go         | 12 ++++++++++--
 syncapi/sync/requestpool_test.go           |  2 +-
 7 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 43aaa3588..486978598 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -159,6 +159,6 @@ type Database interface {
 type Presence interface {
 	UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error)
 	GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
-	PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error)
+	PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
 	MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
 }
diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go
index 9f1e37f79..7194afea6 100644
--- a/syncapi/storage/postgres/presence_table.go
+++ b/syncapi/storage/postgres/presence_table.go
@@ -17,6 +17,7 @@ package postgres
 import (
 	"context"
 	"database/sql"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -72,7 +73,8 @@ const selectMaxPresenceSQL = "" +
 const selectPresenceAfter = "" +
 	" SELECT id, user_id, presence, status_msg, last_active_ts" +
 	" FROM syncapi_presence" +
-	" WHERE id > $1"
+	" WHERE id > $1 AND last_active_ts >= $2" +
+	" ORDER BY id ASC LIMIT $3"
 
 type presenceStatements struct {
 	upsertPresenceStmt         *sql.Stmt
@@ -144,11 +146,12 @@ func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx)
 func (p *presenceStatements) GetPresenceAfter(
 	ctx context.Context, txn *sql.Tx,
 	after types.StreamPosition,
+	filter gomatrixserverlib.EventFilter,
 ) (presences map[string]*types.PresenceInternal, err error) {
 	presences = make(map[string]*types.PresenceInternal)
 	stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt)
-
-	rows, err := stmt.QueryContext(ctx, after)
+	afterTS := gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute * -5))
+	rows, err := stmt.QueryContext(ctx, after, afterTS, filter.Limit)
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 25aca50ae..b7d2d3a29 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -1056,8 +1056,8 @@ func (s *Database) GetPresence(ctx context.Context, userID string) (*types.Prese
 	return s.Presence.GetPresenceForUser(ctx, nil, userID)
 }
 
-func (s *Database) PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) {
-	return s.Presence.GetPresenceAfter(ctx, nil, after)
+func (s *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
+	return s.Presence.GetPresenceAfter(ctx, nil, after, filter)
 }
 
 func (s *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go
index 177a01bf3..b61a825df 100644
--- a/syncapi/storage/sqlite3/presence_table.go
+++ b/syncapi/storage/sqlite3/presence_table.go
@@ -17,6 +17,7 @@ package sqlite3
 import (
 	"context"
 	"database/sql"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -71,7 +72,8 @@ const selectMaxPresenceSQL = "" +
 const selectPresenceAfter = "" +
 	" SELECT id, user_id, presence, status_msg, last_active_ts" +
 	" FROM syncapi_presence" +
-	" WHERE id > $1"
+	" WHERE id > $1 AND last_active_ts >= $2" +
+	" ORDER BY id ASC LIMIT $3"
 
 type presenceStatements struct {
 	db                         *sql.DB
@@ -158,12 +160,12 @@ func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx)
 // GetPresenceAfter returns the changes presences after a given stream id
 func (p *presenceStatements) GetPresenceAfter(
 	ctx context.Context, txn *sql.Tx,
-	after types.StreamPosition,
+	after types.StreamPosition, filter gomatrixserverlib.EventFilter,
 ) (presences map[string]*types.PresenceInternal, err error) {
 	presences = make(map[string]*types.PresenceInternal)
 	stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt)
-
-	rows, err := stmt.QueryContext(ctx, after)
+	afterTS := gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute * -5))
+	rows, err := stmt.QueryContext(ctx, after, afterTS, filter.Limit)
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 4ff4689ed..6fdd9483e 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -188,5 +188,5 @@ type Presence interface {
 	UpsertPresence(ctx context.Context, txn *sql.Tx, userID string, statusMsg *string, presence types.Presence, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (pos types.StreamPosition, err error)
 	GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error)
 	GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error)
-	GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition) (presences map[string]*types.PresenceInternal, err error)
+	GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error)
 }
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 614b88d48..675a7a178 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -53,7 +53,8 @@ func (p *PresenceStreamProvider) IncrementalSync(
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
-	presences, err := p.DB.PresenceAfter(ctx, from)
+	// We pull out a larger number than the filter asks for, since we're filtering out events later
+	presences, err := p.DB.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000})
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PresenceAfter failed")
 		return from
@@ -72,6 +73,7 @@ func (p *PresenceStreamProvider) IncrementalSync(
 			req.Log.WithError(err).Error("unable to refresh notifier lists")
 			return from
 		}
+	NewlyJoinedLoop:
 		for _, roomID := range newlyJoined {
 			roomUsers := p.notifier.JoinedUsers(roomID)
 			for i := range roomUsers {
@@ -86,11 +88,14 @@ func (p *PresenceStreamProvider) IncrementalSync(
 					req.Log.WithError(err).Error("unable to query presence for user")
 					return from
 				}
+				if len(presences) > req.Filter.Presence.Limit {
+					break NewlyJoinedLoop
+				}
 			}
 		}
 	}
 
-	lastPos := to
+	lastPos := from
 	for _, presence := range presences {
 		if presence == nil {
 			continue
@@ -135,6 +140,9 @@ func (p *PresenceStreamProvider) IncrementalSync(
 		if presence.StreamPos > lastPos {
 			lastPos = presence.StreamPos
 		}
+		if len(req.Response.Presence.Events) == req.Filter.Presence.Limit {
+			break
+		}
 		p.cache.Store(cacheKey, presence)
 	}
 
diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go
index a80089945..5e52bc7c9 100644
--- a/syncapi/sync/requestpool_test.go
+++ b/syncapi/sync/requestpool_test.go
@@ -30,7 +30,7 @@ func (d dummyDB) GetPresence(ctx context.Context, userID string) (*types.Presenc
 	return &types.PresenceInternal{}, nil
 }
 
-func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition) (map[string]*types.PresenceInternal, error) {
+func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
 	return map[string]*types.PresenceInternal{}, nil
 }
 

From c6ea2c9ff26ca6ae4c799db08a3f72c6b4d99256 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 28 Apr 2022 16:02:30 +0100
Subject: [PATCH 053/391] Add `/_dendrite/admin/evacuateRoom/{roomID}` (#2401)

* Add new endpoint to allow admins to evacuate the local server from the room

* Guard endpoint

* Use right prefix

* Auth API

* More useful return error rather than a panic

* More useful return value again

* Update the path

* Try using inputer instead

* oh provide the config

* Try that again

* Return affected user IDs

* Don't create so many forward extremities

* Add missing `Path` to name

Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>
---
 build/gobind-pinecone/monolith.go             |   1 +
 build/gobind-yggdrasil/monolith.go            |   1 +
 clientapi/clientapi.go                        |   4 +-
 clientapi/routing/routing.go                  |  42 ++++-
 cmd/dendrite-demo-pinecone/main.go            |   1 +
 cmd/dendrite-demo-yggdrasil/main.go           |   1 +
 cmd/dendrite-monolith-server/main.go          |   1 +
 .../personalities/clientapi.go                |   6 +-
 cmd/dendritejs-pinecone/main.go               |   1 +
 roomserver/api/api.go                         |   6 +
 roomserver/api/api_trace.go                   |   9 +
 roomserver/api/perform.go                     |   9 +
 roomserver/internal/api.go                    |   7 +
 roomserver/internal/perform/perform_admin.go  | 162 ++++++++++++++++++
 roomserver/inthttp/client.go                  |  38 ++--
 roomserver/inthttp/server.go                  |  11 ++
 setup/monolith.go                             |   4 +-
 17 files changed, 288 insertions(+), 16 deletions(-)
 create mode 100644 roomserver/internal/perform/perform_admin.go

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 9cc94d650..6b2533491 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -314,6 +314,7 @@ func (m *DendriteMonolith) Start() {
 		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 	)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 87dcad2e8..b9c6c1b78 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -152,6 +152,7 @@ func (m *DendriteMonolith) Start() {
 		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 	)
 
 	httpRouter := mux.NewRouter()
diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index e2f8d3f32..ad277056c 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -36,6 +36,7 @@ func AddPublicRoutes(
 	process *process.ProcessContext,
 	router *mux.Router,
 	synapseAdminRouter *mux.Router,
+	dendriteAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
 	federation *gomatrixserverlib.FederationClient,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
@@ -62,7 +63,8 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		router, synapseAdminRouter, cfg, rsAPI, asAPI,
+		router, synapseAdminRouter, dendriteAdminRouter,
+		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI,
 		extRoomsProvider, mscCfg, natsClient,
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index f370b4f8c..ec90b80db 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -48,7 +48,8 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI,
+	publicAPIMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
+	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	userAPI userapi.UserInternalAPI,
@@ -119,6 +120,45 @@ func Setup(
 		).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 	}
 
+	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
+		httputil.MakeAuthAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			if device.AccountType != userapi.AccountTypeAdmin {
+				return util.JSONResponse{
+					Code: http.StatusForbidden,
+					JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+				}
+			}
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			roomID, ok := vars["roomID"]
+			if !ok {
+				return util.JSONResponse{
+					Code: http.StatusBadRequest,
+					JSON: jsonerror.MissingArgument("Expecting room ID."),
+				}
+			}
+			res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
+			rsAPI.PerformAdminEvacuateRoom(
+				req.Context(),
+				&roomserverAPI.PerformAdminEvacuateRoomRequest{
+					RoomID: roomID,
+				},
+				res,
+			)
+			if err := res.Error; err != nil {
+				return err.JSONResponse()
+			}
+			return util.JSONResponse{
+				Code: 200,
+				JSON: map[string]interface{}{
+					"affected": res.Affected,
+				},
+			}
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index dd1ab3697..7ec810c95 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -193,6 +193,7 @@ func main() {
 		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 	)
 
 	wsUpgrader := websocket.Upgrader{
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index b840eb2b8..54231f30c 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -150,6 +150,7 @@ func main() {
 		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 	)
 	if err := mscs.Enable(base, &monolith); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index 1443ab5b1..5fd5c0b5d 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -153,6 +153,7 @@ func main() {
 		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 	)
 
 	if len(base.Cfg.MSCs.MSCs) > 0 {
diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go
index 1e509f88a..7ed2075aa 100644
--- a/cmd/dendrite-polylith-multi/personalities/clientapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go
@@ -31,8 +31,10 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	keyAPI := base.KeyServerHTTPClient()
 
 	clientapi.AddPublicRoutes(
-		base.ProcessContext, base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI,
-		federation, rsAPI, asQuery, transactions.New(), fsAPI, userAPI, userAPI,
+		base.ProcessContext, base.PublicClientAPIMux,
+		base.SynapseAdminMux, base.DendriteAdminMux,
+		&base.Cfg.ClientAPI, federation, rsAPI, asQuery,
+		transactions.New(), fsAPI, userAPI, userAPI,
 		keyAPI, nil, &cfg.MSCs,
 	)
 
diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
index 211b3e131..0d4b2fbc5 100644
--- a/cmd/dendritejs-pinecone/main.go
+++ b/cmd/dendritejs-pinecone/main.go
@@ -220,6 +220,7 @@ func startup() {
 		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 	)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index fb77423f8..f0ca8a615 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -66,6 +66,12 @@ type RoomserverInternalAPI interface {
 		res *PerformInboundPeekResponse,
 	) error
 
+	PerformAdminEvacuateRoom(
+		ctx context.Context,
+		req *PerformAdminEvacuateRoomRequest,
+		res *PerformAdminEvacuateRoomResponse,
+	)
+
 	QueryPublishedRooms(
 		ctx context.Context,
 		req *QueryPublishedRoomsRequest,
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index ec7211ef8..61c06e886 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -104,6 +104,15 @@ func (t *RoomserverInternalAPITrace) PerformPublish(
 	util.GetLogger(ctx).Infof("PerformPublish req=%+v res=%+v", js(req), js(res))
 }
 
+func (t *RoomserverInternalAPITrace) PerformAdminEvacuateRoom(
+	ctx context.Context,
+	req *PerformAdminEvacuateRoomRequest,
+	res *PerformAdminEvacuateRoomResponse,
+) {
+	t.Impl.PerformAdminEvacuateRoom(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformAdminEvacuateRoom req=%+v res=%+v", js(req), js(res))
+}
+
 func (t *RoomserverInternalAPITrace) PerformInboundPeek(
 	ctx context.Context,
 	req *PerformInboundPeekRequest,
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index cda4b3ee4..30aa2cf1b 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -214,3 +214,12 @@ type PerformRoomUpgradeResponse struct {
 	NewRoomID string
 	Error     *PerformError
 }
+
+type PerformAdminEvacuateRoomRequest struct {
+	RoomID string `json:"room_id"`
+}
+
+type PerformAdminEvacuateRoomResponse struct {
+	Affected []string `json:"affected"`
+	Error    *PerformError
+}
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index 59f485cf7..267cd4099 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -35,6 +35,7 @@ type RoomserverInternalAPI struct {
 	*perform.Backfiller
 	*perform.Forgetter
 	*perform.Upgrader
+	*perform.Admin
 	ProcessContext         *process.ProcessContext
 	DB                     storage.Database
 	Cfg                    *config.RoomServer
@@ -164,6 +165,12 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalA
 		Cfg:    r.Cfg,
 		URSAPI: r,
 	}
+	r.Admin = &perform.Admin{
+		DB:      r.DB,
+		Cfg:     r.Cfg,
+		Inputer: r.Inputer,
+		Queryer: r.Queryer,
+	}
 
 	if err := r.Inputer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start roomserver input API")
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
new file mode 100644
index 000000000..2de6477cc
--- /dev/null
+++ b/roomserver/internal/perform/perform_admin.go
@@ -0,0 +1,162 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package perform
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/internal/input"
+	"github.com/matrix-org/dendrite/roomserver/internal/query"
+	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Admin struct {
+	DB      storage.Database
+	Cfg     *config.RoomServer
+	Queryer *query.Queryer
+	Inputer *input.Inputer
+}
+
+// PerformEvacuateRoom will remove all local users from the given room.
+func (r *Admin) PerformAdminEvacuateRoom(
+	ctx context.Context,
+	req *api.PerformAdminEvacuateRoomRequest,
+	res *api.PerformAdminEvacuateRoomResponse,
+) {
+	roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.RoomInfo: %s", err),
+		}
+		return
+	}
+	if roomInfo == nil || roomInfo.IsStub {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorNoRoom,
+			Msg:  fmt.Sprintf("Room %s not found", req.RoomID),
+		}
+		return
+	}
+
+	memberNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, roomInfo.RoomNID, true, true)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.GetMembershipEventNIDsForRoom: %s", err),
+		}
+		return
+	}
+
+	memberEvents, err := r.DB.Events(ctx, memberNIDs)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.Events: %s", err),
+		}
+		return
+	}
+
+	inputEvents := make([]api.InputRoomEvent, 0, len(memberEvents))
+	res.Affected = make([]string, 0, len(memberEvents))
+	latestReq := &api.QueryLatestEventsAndStateRequest{
+		RoomID: req.RoomID,
+	}
+	latestRes := &api.QueryLatestEventsAndStateResponse{}
+	if err = r.Queryer.QueryLatestEventsAndState(ctx, latestReq, latestRes); err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.Queryer.QueryLatestEventsAndState: %s", err),
+		}
+		return
+	}
+
+	prevEvents := latestRes.LatestEvents
+	for _, memberEvent := range memberEvents {
+		if memberEvent.StateKey() == nil {
+			continue
+		}
+
+		var memberContent gomatrixserverlib.MemberContent
+		if err = json.Unmarshal(memberEvent.Content(), &memberContent); err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("json.Unmarshal: %s", err),
+			}
+			return
+		}
+		memberContent.Membership = gomatrixserverlib.Leave
+
+		stateKey := *memberEvent.StateKey()
+		fledglingEvent := &gomatrixserverlib.EventBuilder{
+			RoomID:     req.RoomID,
+			Type:       gomatrixserverlib.MRoomMember,
+			StateKey:   &stateKey,
+			Sender:     stateKey,
+			PrevEvents: prevEvents,
+		}
+
+		if fledglingEvent.Content, err = json.Marshal(memberContent); err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("json.Marshal: %s", err),
+			}
+			return
+		}
+
+		eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(fledglingEvent)
+		if err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("gomatrixserverlib.StateNeededForEventBuilder: %s", err),
+			}
+			return
+		}
+
+		event, err := eventutil.BuildEvent(ctx, fledglingEvent, r.Cfg.Matrix, time.Now(), &eventsNeeded, latestRes)
+		if err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("eventutil.BuildEvent: %s", err),
+			}
+			return
+		}
+
+		inputEvents = append(inputEvents, api.InputRoomEvent{
+			Kind:         api.KindNew,
+			Event:        event,
+			Origin:       r.Cfg.Matrix.ServerName,
+			SendAsServer: string(r.Cfg.Matrix.ServerName),
+		})
+		res.Affected = append(res.Affected, stateKey)
+		prevEvents = []gomatrixserverlib.EventReference{
+			event.EventReference(),
+		}
+	}
+
+	inputReq := &api.InputRoomEventsRequest{
+		InputRoomEvents: inputEvents,
+		Asynchronous:    true,
+	}
+	inputRes := &api.InputRoomEventsResponse{}
+	r.Inputer.InputRoomEvents(ctx, inputReq, inputRes)
+}
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index d55805a91..3b29001e9 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -29,16 +29,17 @@ const (
 	RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents"
 
 	// Perform operations
-	RoomserverPerformInvitePath      = "/roomserver/performInvite"
-	RoomserverPerformPeekPath        = "/roomserver/performPeek"
-	RoomserverPerformUnpeekPath      = "/roomserver/performUnpeek"
-	RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade"
-	RoomserverPerformJoinPath        = "/roomserver/performJoin"
-	RoomserverPerformLeavePath       = "/roomserver/performLeave"
-	RoomserverPerformBackfillPath    = "/roomserver/performBackfill"
-	RoomserverPerformPublishPath     = "/roomserver/performPublish"
-	RoomserverPerformInboundPeekPath = "/roomserver/performInboundPeek"
-	RoomserverPerformForgetPath      = "/roomserver/performForget"
+	RoomserverPerformInvitePath            = "/roomserver/performInvite"
+	RoomserverPerformPeekPath              = "/roomserver/performPeek"
+	RoomserverPerformUnpeekPath            = "/roomserver/performUnpeek"
+	RoomserverPerformRoomUpgradePath       = "/roomserver/performRoomUpgrade"
+	RoomserverPerformJoinPath              = "/roomserver/performJoin"
+	RoomserverPerformLeavePath             = "/roomserver/performLeave"
+	RoomserverPerformBackfillPath          = "/roomserver/performBackfill"
+	RoomserverPerformPublishPath           = "/roomserver/performPublish"
+	RoomserverPerformInboundPeekPath       = "/roomserver/performInboundPeek"
+	RoomserverPerformForgetPath            = "/roomserver/performForget"
+	RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom"
 
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
@@ -299,6 +300,23 @@ func (h *httpRoomserverInternalAPI) PerformPublish(
 	}
 }
 
+func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom(
+	ctx context.Context,
+	req *api.PerformAdminEvacuateRoomRequest,
+	res *api.PerformAdminEvacuateRoomResponse,
+) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAdminEvacuateRoom")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverPerformAdminEvacuateRoomPath
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
+		}
+	}
+}
+
 // QueryLatestEventsAndState implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
 	ctx context.Context,
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index 0b27b5a8d..c5159a63c 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -118,6 +118,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverPerformAdminEvacuateRoomPath,
+		httputil.MakeInternalAPI("performAdminEvacuateRoom", func(req *http.Request) util.JSONResponse {
+			var request api.PerformAdminEvacuateRoomRequest
+			var response api.PerformAdminEvacuateRoomResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			r.PerformAdminEvacuateRoom(req.Context(), &request, &response)
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(
 		RoomserverQueryPublishedRoomsPath,
 		httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse {
diff --git a/setup/monolith.go b/setup/monolith.go
index 32f1a6494..c86ec7b69 100644
--- a/setup/monolith.go
+++ b/setup/monolith.go
@@ -54,13 +54,13 @@ type Monolith struct {
 }
 
 // AddAllPublicRoutes attaches all public paths to the given router
-func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux *mux.Router) {
+func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux, dendriteMux *mux.Router) {
 	userDirectoryProvider := m.ExtUserDirectoryProvider
 	if userDirectoryProvider == nil {
 		userDirectoryProvider = m.UserAPI
 	}
 	clientapi.AddPublicRoutes(
-		process, csMux, synapseMux, &m.Config.ClientAPI,
+		process, csMux, synapseMux, dendriteMux, &m.Config.ClientAPI,
 		m.FedClient, m.RoomserverAPI,
 		m.AppserviceAPI, transactions.New(),
 		m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI,

From 2a5b8e0306a283aa8ca64822d59d71479ffba59a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 28 Apr 2022 18:53:28 +0200
Subject: [PATCH 054/391] Only load members of newly joined rooms (#2389)

* Only load members of newly joined rooms

* Comment that the query is prepared at runtime

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 syncapi/notifier/notifier.go                  | 14 ++++++
 syncapi/storage/interface.go                  |  3 ++
 .../postgres/current_room_state_table.go      | 34 ++++++++++++-
 syncapi/storage/shared/syncserver.go          |  4 ++
 .../sqlite3/current_room_state_table.go       | 49 +++++++++++++++++--
 syncapi/storage/tables/interface.go           |  2 +
 syncapi/streams/stream_presence.go            |  5 +-
 7 files changed, 102 insertions(+), 9 deletions(-)

diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index 82834239b..87f0d86d7 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -333,6 +333,20 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error {
 	return nil
 }
 
+// LoadRooms loads the membership states required to notify users correctly.
+func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs []string) error {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	roomToUsers, err := db.AllJoinedUsersInRoom(ctx, roomIDs)
+	if err != nil {
+		return err
+	}
+	n.setUsersJoinedToRooms(roomToUsers)
+
+	return nil
+}
+
 // CurrentPosition returns the current sync position
 func (n *Notifier) CurrentPosition() types.StreamingToken {
 	n.lock.RLock()
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 486978598..5a036d889 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -52,6 +52,9 @@ type Database interface {
 
 	// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs.
 	AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error)
+	// AllJoinedUsersInRoom returns a map of room ID to a list of all joined user IDs for a given room.
+	AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error)
+
 	// AllPeekingDevicesInRooms returns a map of room ID to a list of all peeking devices.
 	AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error)
 	// Events lookups a list of event by their event ID.
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index fe68788d1..8ee387b39 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -93,6 +93,9 @@ const selectCurrentStateSQL = "" +
 const selectJoinedUsersSQL = "" +
 	"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'"
 
+const selectJoinedUsersInRoomSQL = "" +
+	"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join' AND room_id = ANY($1)"
+
 const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
 
@@ -112,6 +115,7 @@ type currentRoomStateStatements struct {
 	selectRoomIDsWithAnyMembershipStmt *sql.Stmt
 	selectCurrentStateStmt             *sql.Stmt
 	selectJoinedUsersStmt              *sql.Stmt
+	selectJoinedUsersInRoomStmt        *sql.Stmt
 	selectEventsWithEventIDsStmt       *sql.Stmt
 	selectStateEventStmt               *sql.Stmt
 }
@@ -143,6 +147,9 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
 	if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil {
 		return nil, err
 	}
+	if s.selectJoinedUsersInRoomStmt, err = db.Prepare(selectJoinedUsersInRoomSQL); err != nil {
+		return nil, err
+	}
 	if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil {
 		return nil, err
 	}
@@ -163,9 +170,32 @@ func (s *currentRoomStateStatements) SelectJoinedUsers(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed")
 
 	result := make(map[string][]string)
+	var roomID string
+	var userID string
+	for rows.Next() {
+		if err := rows.Scan(&roomID, &userID); err != nil {
+			return nil, err
+		}
+		users := result[roomID]
+		users = append(users, userID)
+		result[roomID] = users
+	}
+	return result, rows.Err()
+}
+
+// SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
+func (s *currentRoomStateStatements) SelectJoinedUsersInRoom(
+	ctx context.Context, roomIDs []string,
+) (map[string][]string, error) {
+	rows, err := s.selectJoinedUsersInRoomStmt.QueryContext(ctx, pq.StringArray(roomIDs))
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed")
+
+	result := make(map[string][]string)
+	var userID, roomID string
 	for rows.Next() {
-		var roomID string
-		var userID string
 		if err := rows.Scan(&roomID, &userID); err != nil {
 			return nil, err
 		}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index b7d2d3a29..ec5edd355 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -168,6 +168,10 @@ func (d *Database) AllJoinedUsersInRooms(ctx context.Context) (map[string][]stri
 	return d.CurrentRoomState.SelectJoinedUsers(ctx)
 }
 
+func (d *Database) AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) {
+	return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, roomIDs)
+}
+
 func (d *Database) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) {
 	return d.Peeks.SelectPeekingDevices(ctx)
 }
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index ccda005c1..f0a1c7bb7 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -77,6 +77,9 @@ const selectCurrentStateSQL = "" +
 const selectJoinedUsersSQL = "" +
 	"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'"
 
+const selectJoinedUsersInRoomSQL = "" +
+	"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join' AND room_id IN ($1)"
+
 const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
 
@@ -97,7 +100,8 @@ type currentRoomStateStatements struct {
 	selectRoomIDsWithMembershipStmt    *sql.Stmt
 	selectRoomIDsWithAnyMembershipStmt *sql.Stmt
 	selectJoinedUsersStmt              *sql.Stmt
-	selectStateEventStmt               *sql.Stmt
+	//selectJoinedUsersInRoomStmt        *sql.Stmt - prepared at runtime due to variadic
+	selectStateEventStmt *sql.Stmt
 }
 
 func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (tables.CurrentRoomState, error) {
@@ -127,13 +131,16 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (t
 	if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil {
 		return nil, err
 	}
+	//if s.selectJoinedUsersInRoomStmt, err = db.Prepare(selectJoinedUsersInRoomSQL); err != nil {
+	//	return nil, err
+	//}
 	if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
 		return nil, err
 	}
 	return s, nil
 }
 
-// JoinedMemberLists returns a map of room ID to a list of joined user IDs.
+// SelectJoinedUsers returns a map of room ID to a list of joined user IDs.
 func (s *currentRoomStateStatements) SelectJoinedUsers(
 	ctx context.Context,
 ) (map[string][]string, error) {
@@ -144,9 +151,9 @@ func (s *currentRoomStateStatements) SelectJoinedUsers(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed")
 
 	result := make(map[string][]string)
+	var roomID string
+	var userID string
 	for rows.Next() {
-		var roomID string
-		var userID string
 		if err := rows.Scan(&roomID, &userID); err != nil {
 			return nil, err
 		}
@@ -157,6 +164,40 @@ func (s *currentRoomStateStatements) SelectJoinedUsers(
 	return result, nil
 }
 
+// SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
+func (s *currentRoomStateStatements) SelectJoinedUsersInRoom(
+	ctx context.Context, roomIDs []string,
+) (map[string][]string, error) {
+	query := strings.Replace(selectJoinedUsersInRoomSQL, "($1)", sqlutil.QueryVariadic(len(roomIDs)), 1)
+	params := make([]interface{}, 0, len(roomIDs))
+	for _, roomID := range roomIDs {
+		params = append(params, roomID)
+	}
+	stmt, err := s.db.Prepare(query)
+	if err != nil {
+		return nil, fmt.Errorf("SelectJoinedUsersInRoom s.db.Prepare: %w", err)
+	}
+	defer internal.CloseAndLogIfError(ctx, stmt, "SelectJoinedUsersInRoom: stmt.close() failed")
+
+	rows, err := stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectJoinedUsersInRoom: rows.close() failed")
+
+	result := make(map[string][]string)
+	var userID, roomID string
+	for rows.Next() {
+		if err := rows.Scan(&roomID, &userID); err != nil {
+			return nil, err
+		}
+		users := result[roomID]
+		users = append(users, userID)
+		result[roomID] = users
+	}
+	return result, rows.Err()
+}
+
 // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
 func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 	ctx context.Context,
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 6fdd9483e..ccdebfdbd 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -102,6 +102,8 @@ type CurrentRoomState interface {
 	SelectRoomIDsWithAnyMembership(ctx context.Context, txn *sql.Tx, userID string) (map[string]string, error)
 	// SelectJoinedUsers returns a map of room ID to a list of joined user IDs.
 	SelectJoinedUsers(ctx context.Context) (map[string][]string, error)
+	// SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
+	SelectJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error)
 }
 
 // BackwardsExtremities keeps track of backwards extremities for a room.
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 675a7a178..a84d19878 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -67,9 +67,8 @@ func (p *PresenceStreamProvider) IncrementalSync(
 	// add newly joined rooms user presences
 	newlyJoined := joinedRooms(req.Response, req.Device.UserID)
 	if len(newlyJoined) > 0 {
-		// TODO: This refreshes all lists and is quite expensive
-		// The notifier should update the lists itself
-		if err = p.notifier.Load(ctx, p.DB); err != nil {
+		// TODO: Check if this is working better than before.
+		if err = p.notifier.LoadRooms(ctx, p.DB, newlyJoined); err != nil {
 			req.Log.WithError(err).Error("unable to refresh notifier lists")
 			return from
 		}

From 26a1512808282c954a141b3376c47b05ef1e6ab4 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 29 Apr 2022 09:31:11 +0200
Subject: [PATCH 055/391] Add restrictions for open registration (#2402)

* Add restrications for open registration

* Make enable open registration a parameter

* Enable registration for CI

* Update error message

* Shuffle things around a bit

* Add a warning at every startup just to be extra annoying

* Ignore shared secret when warning about open registration, since it's not strictly required when it is set if registration is otherwise enabled

* Make CI happy?

* Add missing parameter; try new parameter in upgrade-test

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 build/docker/config/dendrite.yaml           |  2 +-
 build/gobind-pinecone/monolith.go           |  2 ++
 build/gobind-yggdrasil/monolith.go          |  2 ++
 build/scripts/Complement.Dockerfile         |  2 +-
 build/scripts/ComplementLocal.Dockerfile    |  2 +-
 build/scripts/ComplementPostgres.Dockerfile |  2 +-
 cmd/dendrite-demo-pinecone/main.go          |  2 ++
 cmd/dendrite-demo-yggdrasil/main.go         |  2 ++
 cmd/dendrite-upgrade-tests/main.go          |  3 ++-
 cmd/dendritejs-pinecone/main.go             |  2 ++
 cmd/generate-config/main.go                 |  2 ++
 dendrite-config.yaml                        |  2 +-
 setup/base/base.go                          |  4 ++++
 setup/config/config_clientapi.go            | 23 ++++++++++++++++++++-
 setup/flags.go                              |  9 ++++++--
 15 files changed, 52 insertions(+), 9 deletions(-)

diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml
index e3a0316dc..94dcf4558 100644
--- a/build/docker/config/dendrite.yaml
+++ b/build/docker/config/dendrite.yaml
@@ -140,7 +140,7 @@ client_api:
 
   # Prevents new users from being able to register on this homeserver, except when
   # using the registration shared secret below.
-  registration_disabled: false
+  registration_disabled: true
 
   # If set, allows registration by anyone who knows the shared secret, regardless of
   # whether registration is otherwise disabled.
diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 6b2533491..d047f3fff 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -259,6 +259,8 @@ func (m *DendriteMonolith) Start() {
 	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
 	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
 	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	cfg.ClientAPI.RegistrationDisabled = false
+	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 	if err := cfg.Derive(); err != nil {
 		panic(err)
 	}
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index b9c6c1b78..4e95e3972 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -97,6 +97,8 @@ func (m *DendriteMonolith) Start() {
 	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory))
 	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
 	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
+	cfg.ClientAPI.RegistrationDisabled = false
+	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile
index 6b2942d97..63e3890ee 100644
--- a/build/scripts/Complement.Dockerfile
+++ b/build/scripts/Complement.Dockerfile
@@ -29,4 +29,4 @@ EXPOSE 8008 8448
 CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
  ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
  cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+ ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile
index 60b4d983a..a9feb4cd1 100644
--- a/build/scripts/ComplementLocal.Dockerfile
+++ b/build/scripts/ComplementLocal.Dockerfile
@@ -32,7 +32,7 @@ RUN echo '\
 ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
 ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
 cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
-./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
 ' > run.sh && chmod +x run.sh
 
 
diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile
index b98f4671c..4e26faa58 100644
--- a/build/scripts/ComplementPostgres.Dockerfile
+++ b/build/scripts/ComplementPostgres.Dockerfile
@@ -51,4 +51,4 @@ CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert s
  sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
  sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
  cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
+ ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 7ec810c95..785e7b460 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -140,6 +140,8 @@ func main() {
 	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
 	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
 	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	cfg.ClientAPI.RegistrationDisabled = false
+	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 	if err := cfg.Derive(); err != nil {
 		panic(err)
 	}
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 54231f30c..f9234319a 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -89,6 +89,8 @@ func main() {
 		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
 		cfg.MSCs.MSCs = []string{"msc2836"}
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
+		cfg.ClientAPI.RegistrationDisabled = false
+		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 		if err = cfg.Derive(); err != nil {
 			panic(err)
 		}
diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index 3241234ac..b7e7da07d 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -83,7 +83,8 @@ do \n\
 done \n\
  \n\
 sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
-./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
+./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
 ' > run_dendrite.sh && chmod +x run_dendrite.sh
 
 ENV SERVER_NAME=localhost
diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
index 0d4b2fbc5..5ecf1f2fb 100644
--- a/cmd/dendritejs-pinecone/main.go
+++ b/cmd/dendritejs-pinecone/main.go
@@ -171,6 +171,8 @@ func startup() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.PrivateKey = sk
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.ClientAPI.RegistrationDisabled = false
+	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 
 	if err := cfg.Derive(); err != nil {
 		logrus.Fatalf("Failed to derive values from config: %s", err)
diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go
index 24085afaa..1c585d916 100644
--- a/cmd/generate-config/main.go
+++ b/cmd/generate-config/main.go
@@ -90,6 +90,8 @@ func main() {
 		cfg.Logging[0].Type = "std"
 		cfg.UserAPI.BCryptCost = bcrypt.MinCost
 		cfg.Global.JetStream.InMemory = true
+		cfg.ClientAPI.RegistrationDisabled = false
+		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 		cfg.ClientAPI.RegistrationSharedSecret = "complement"
 		cfg.Global.Presence = config.PresenceOptions{
 			EnableInbound:  true,
diff --git a/dendrite-config.yaml b/dendrite-config.yaml
index 47f08c4fd..1c11ef96d 100644
--- a/dendrite-config.yaml
+++ b/dendrite-config.yaml
@@ -159,7 +159,7 @@ client_api:
 
   # Prevents new users from being able to register on this homeserver, except when
   # using the registration shared secret below.
-  registration_disabled: false
+  registration_disabled: true
 
   # Prevents new guest accounts from being created. Guest registration is also
   # disabled implicitly by setting 'registration_disabled' above.
diff --git a/setup/base/base.go b/setup/base/base.go
index 7091c6ba5..4b771aa36 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -126,6 +126,10 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 
 	logrus.Infof("Dendrite version %s", internal.VersionString())
 
+	if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled {
+		logrus.Warn("Open registration is enabled")
+	}
+
 	closer, err := cfg.SetupTracing("Dendrite" + componentName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to start opentracing")
diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go
index 4590e752b..6104ed8b9 100644
--- a/setup/config/config_clientapi.go
+++ b/setup/config/config_clientapi.go
@@ -15,6 +15,12 @@ type ClientAPI struct {
 	// If set disables new users from registering (except via shared
 	// secrets)
 	RegistrationDisabled bool `yaml:"registration_disabled"`
+
+	// Enable registration without captcha verification or shared secret.
+	// This option is populated by the -really-enable-open-registration
+	// command line parameter as it is not recommended.
+	OpenRegistrationWithoutVerificationEnabled bool `yaml:"-"`
+
 	// If set, allows registration by anyone who also has the shared
 	// secret, even if registration is otherwise disabled.
 	RegistrationSharedSecret string `yaml:"registration_shared_secret"`
@@ -55,7 +61,8 @@ func (c *ClientAPI) Defaults(generate bool) {
 	c.RecaptchaEnabled = false
 	c.RecaptchaBypassSecret = ""
 	c.RecaptchaSiteVerifyAPI = ""
-	c.RegistrationDisabled = false
+	c.RegistrationDisabled = true
+	c.OpenRegistrationWithoutVerificationEnabled = false
 	c.RateLimiting.Defaults()
 }
 
@@ -72,6 +79,20 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	}
 	c.TURN.Verify(configErrs)
 	c.RateLimiting.Verify(configErrs)
+
+	// Ensure there is any spam counter measure when enabling registration
+	if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled {
+		if !c.RecaptchaEnabled {
+			configErrs.Add(
+				"You have tried to enable open registration without any secondary verification methods " +
+					"(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " +
+					"increasing the risk that your server will be used to send spam or abuse, and may result in " +
+					"your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " +
+					"start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " +
+					"should set the registration_disabled option in your Dendrite config.",
+			)
+		}
+	}
 }
 
 type TURN struct {
diff --git a/setup/flags.go b/setup/flags.go
index 281cf3392..a9dac61a1 100644
--- a/setup/flags.go
+++ b/setup/flags.go
@@ -25,8 +25,9 @@ import (
 )
 
 var (
-	configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
-	version    = flag.Bool("version", false, "Shows the current version and exits immediately.")
+	configPath                            = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
+	version                               = flag.Bool("version", false, "Shows the current version and exits immediately.")
+	enableRegistrationWithoutVerification = flag.Bool("really-enable-open-registration", false, "This allows open registration without secondary verification (reCAPTCHA). This is NOT RECOMMENDED and will SIGNIFICANTLY increase the risk that your server will be used to send spam or conduct attacks, which may result in your server being banned from rooms.")
 )
 
 // ParseFlags parses the commandline flags and uses them to create a config.
@@ -48,5 +49,9 @@ func ParseFlags(monolith bool) *config.Dendrite {
 		logrus.Fatalf("Invalid config file: %s", err)
 	}
 
+	if *enableRegistrationWithoutVerification {
+		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+	}
+
 	return cfg
 }

From 1e083794ef0d2968eae9c874e621d199a2b62238 Mon Sep 17 00:00:00 2001
From: Brian Meek <brian@hntlabs.com>
Date: Fri, 29 Apr 2022 00:55:35 -0700
Subject: [PATCH 056/391] Update golangci-lint, how it's installed, and added
 to the PATH (#2403)

* switch to dendrite server

* minor refactor to merge store code

* Fix issue where m.room.name is being filtered by the dendrite server

* refresh dendrite main

* refresh dendrite main

* missing merges from the last dendrite refresh

* revert unwanted changes in dendrite.yaml

* Update golangci-lint, how it's installed, and added to the PATH

Co-authored-by: Tak Wai Wong <takwaiw@gmail.com>
Co-authored-by: tak-slashtalk <64229756+tak-slashtalk@users.noreply.github.com>
---
 build/scripts/find-lint.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/scripts/find-lint.sh b/build/scripts/find-lint.sh
index e3564ae38..820b8cc46 100755
--- a/build/scripts/find-lint.sh
+++ b/build/scripts/find-lint.sh
@@ -25,7 +25,7 @@ echo "Installing golangci-lint..."
 
 # Make a backup of go.{mod,sum} first
 cp go.mod go.mod.bak && cp go.sum go.sum.bak
-go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.41.1
+go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2
 
 # Run linting
 echo "Looking for lint..."
@@ -33,7 +33,7 @@ echo "Looking for lint..."
 # Capture exit code to ensure go.{mod,sum} is restored before exiting
 exit_code=0
 
-PATH="$PATH:${GOPATH:-~/go}/bin" golangci-lint run $args || exit_code=1
+PATH="$PATH:$(go env GOPATH)/bin" golangci-lint run $args || exit_code=1
 
 # Restore go.{mod,sum}
 mv go.mod.bak go.mod && mv go.sum.bak go.sum

From 0d4b8eadaa45ff65cddd4b882a156de33c4bed7e Mon Sep 17 00:00:00 2001
From: Till Faelligen <tfaelligen@gmail.com>
Date: Fri, 29 Apr 2022 10:00:28 +0200
Subject: [PATCH 057/391] Add create-account to Getting started

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 75c827c33..2a8c36508 100644
--- a/README.md
+++ b/README.md
@@ -74,6 +74,10 @@ $ cp dendrite-config.yaml dendrite.yaml
 
 # Build and run the server:
 $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
+
+# Create an user account (add -admin for an admin user).
+# Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
+$ ./bin/create-account --config dendrite.yaml -username alice
 ```
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.

From 2a4517f8e6c7f68d8bd94f0b4640b142f2d3ac52 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 29 Apr 2022 09:10:08 +0100
Subject: [PATCH 058/391] Move admin functions into their own file in the
 client API

---
 clientapi/routing/admin.go   | 49 ++++++++++++++++++++++++++++++++++++
 clientapi/routing/routing.go | 35 +-------------------------
 2 files changed, 50 insertions(+), 34 deletions(-)
 create mode 100644 clientapi/routing/admin.go

diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
new file mode 100644
index 000000000..31e431c78
--- /dev/null
+++ b/clientapi/routing/admin.go
@@ -0,0 +1,49 @@
+package routing
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.RoomserverInternalAPI) util.JSONResponse {
+	if device.AccountType != userapi.AccountTypeAdmin {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+		}
+	}
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	roomID, ok := vars["roomID"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
+		}
+	}
+	res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
+	rsAPI.PerformAdminEvacuateRoom(
+		req.Context(),
+		&roomserverAPI.PerformAdminEvacuateRoomRequest{
+			RoomID: roomID,
+		},
+		res,
+	)
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"affected": res.Affected,
+		},
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index ec90b80db..ba1c76b81 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -122,40 +122,7 @@ func Setup(
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
 		httputil.MakeAuthAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if device.AccountType != userapi.AccountTypeAdmin {
-				return util.JSONResponse{
-					Code: http.StatusForbidden,
-					JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-				}
-			}
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			roomID, ok := vars["roomID"]
-			if !ok {
-				return util.JSONResponse{
-					Code: http.StatusBadRequest,
-					JSON: jsonerror.MissingArgument("Expecting room ID."),
-				}
-			}
-			res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
-			rsAPI.PerformAdminEvacuateRoom(
-				req.Context(),
-				&roomserverAPI.PerformAdminEvacuateRoomRequest{
-					RoomID: roomID,
-				},
-				res,
-			)
-			if err := res.Error; err != nil {
-				return err.JSONResponse()
-			}
-			return util.JSONResponse{
-				Code: 200,
-				JSON: map[string]interface{}{
-					"affected": res.Affected,
-				},
-			}
+			return AdminEvacuateRoom(req, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 

From d28d0ee66e22402bebd791a46de33c8bf3169e26 Mon Sep 17 00:00:00 2001
From: Brian Meek <brian@hntlabs.com>
Date: Fri, 29 Apr 2022 09:32:58 +0100
Subject: [PATCH 059/391] Fix `TestThumbnailsStorage` failing when media
 results come back in non-deterministic order; silence expected error when
 tests are run multiple times against the same postgres database
 (cherry-picked from #2395)

Signed-off-by: Brian Meek <brian@hntlabs.com>
---
 mediaapi/storage/storage_test.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/mediaapi/storage/storage_test.go b/mediaapi/storage/storage_test.go
index 8d3403045..fa88cd8e7 100644
--- a/mediaapi/storage/storage_test.go
+++ b/mediaapi/storage/storage_test.go
@@ -123,11 +123,19 @@ func TestThumbnailsStorage(t *testing.T) {
 				t.Fatalf("expected %d stored thumbnail metadata, got %d", len(thumbnails), len(gotMediadatas))
 			}
 			for i := range gotMediadatas {
-				if !reflect.DeepEqual(thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata) {
-					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata)
+				// metadata may be returned in a different order than it was stored, perform a search
+				metaDataMatches := func() bool {
+					for _, t := range thumbnails {
+						if reflect.DeepEqual(t.MediaMetadata, gotMediadatas[i].MediaMetadata) && reflect.DeepEqual(t.ThumbnailSize, gotMediadatas[i].ThumbnailSize) {
+							return true
+						}
+					}
+					return false
 				}
-				if !reflect.DeepEqual(thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize) {
-					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize)
+
+				if !metaDataMatches() {
+					t.Fatalf("expected metadata %+v, got %+v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata)
+
 				}
 			}
 		})

From 31799a3b2a733192a656cfa77662599bd1568bdc Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 29 Apr 2022 16:02:55 +0100
Subject: [PATCH 060/391] Device list display name fixes (#2405)

* Get device names from `unsigned` in `/user/devices`

* Fix display name updates

* Fix bug

* Fix another bug
---
 federationapi/routing/devices.go |  8 ++-
 keyserver/internal/internal.go   | 95 ++++++++++++++++----------------
 2 files changed, 54 insertions(+), 49 deletions(-)

diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go
index 8890eac4b..57286fa90 100644
--- a/federationapi/routing/devices.go
+++ b/federationapi/routing/devices.go
@@ -20,6 +20,7 @@ import (
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
 )
 
 // GetUserDevices for the given user id
@@ -69,9 +70,14 @@ func GetUserDevices(
 			continue
 		}
 
+		displayName := dev.DisplayName
+		if displayName == "" {
+			displayName = gjson.GetBytes(dev.DeviceKeys.KeyJSON, "unsigned.device_display_name").Str
+		}
+
 		device := gomatrixserverlib.RespUserDevice{
 			DeviceID:    dev.DeviceID,
-			DisplayName: dev.DisplayName,
+			DisplayName: displayName,
 			Keys:        key,
 		}
 
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 1677cf8e3..e556f44b0 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -632,43 +632,55 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per
 	}
 
 	var keysToStore []api.DeviceMessage
-	// assert that the user ID / device ID are not lying for each key
-	for _, key := range req.DeviceKeys {
-		var serverName gomatrixserverlib.ServerName
-		_, serverName, err = gomatrixserverlib.SplitID('@', key.UserID)
-		if err != nil {
-			continue // ignore invalid users
-		}
-		if serverName != a.ThisServer {
-			continue // ignore remote users
-		}
-		if len(key.KeyJSON) == 0 {
-			keysToStore = append(keysToStore, key.WithStreamID(0))
-			continue // deleted keys don't need sanity checking
-		}
-		// check that the device in question actually exists in the user
-		// API before we try and store a key for it
-		if _, ok := existingDeviceMap[key.DeviceID]; !ok {
-			continue
-		}
-		gotUserID := gjson.GetBytes(key.KeyJSON, "user_id").Str
-		gotDeviceID := gjson.GetBytes(key.KeyJSON, "device_id").Str
-		if gotUserID == key.UserID && gotDeviceID == key.DeviceID {
-			keysToStore = append(keysToStore, key.WithStreamID(0))
-			continue
-		}
-
-		res.KeyError(key.UserID, key.DeviceID, &api.KeyError{
-			Err: fmt.Sprintf(
-				"user_id or device_id mismatch: users: %s - %s, devices: %s - %s",
-				gotUserID, key.UserID, gotDeviceID, key.DeviceID,
-			),
-		})
-	}
 
 	if req.OnlyDisplayNameUpdates {
-		// add the display name field from keysToStore into existingKeys
-		keysToStore = appendDisplayNames(existingKeys, keysToStore)
+		for _, existingKey := range existingKeys {
+			for _, newKey := range req.DeviceKeys {
+				switch {
+				case existingKey.UserID != newKey.UserID:
+					continue
+				case existingKey.DeviceID != newKey.DeviceID:
+					continue
+				case existingKey.DisplayName != newKey.DisplayName:
+					existingKey.DisplayName = newKey.DisplayName
+				}
+			}
+			keysToStore = append(keysToStore, existingKey)
+		}
+	} else {
+		// assert that the user ID / device ID are not lying for each key
+		for _, key := range req.DeviceKeys {
+			var serverName gomatrixserverlib.ServerName
+			_, serverName, err = gomatrixserverlib.SplitID('@', key.UserID)
+			if err != nil {
+				continue // ignore invalid users
+			}
+			if serverName != a.ThisServer {
+				continue // ignore remote users
+			}
+			if len(key.KeyJSON) == 0 {
+				keysToStore = append(keysToStore, key.WithStreamID(0))
+				continue // deleted keys don't need sanity checking
+			}
+			// check that the device in question actually exists in the user
+			// API before we try and store a key for it
+			if _, ok := existingDeviceMap[key.DeviceID]; !ok {
+				continue
+			}
+			gotUserID := gjson.GetBytes(key.KeyJSON, "user_id").Str
+			gotDeviceID := gjson.GetBytes(key.KeyJSON, "device_id").Str
+			if gotUserID == key.UserID && gotDeviceID == key.DeviceID {
+				keysToStore = append(keysToStore, key.WithStreamID(0))
+				continue
+			}
+
+			res.KeyError(key.UserID, key.DeviceID, &api.KeyError{
+				Err: fmt.Sprintf(
+					"user_id or device_id mismatch: users: %s - %s, devices: %s - %s",
+					gotUserID, key.UserID, gotDeviceID, key.DeviceID,
+				),
+			})
+		}
 	}
 
 	// store the device keys and emit changes
@@ -764,16 +776,3 @@ func emitDeviceKeyChanges(producer KeyChangeProducer, existing, new []api.Device
 	}
 	return producer.ProduceKeyChanges(keysAdded)
 }
-
-func appendDisplayNames(existing, new []api.DeviceMessage) []api.DeviceMessage {
-	for i, existingDevice := range existing {
-		for _, newDevice := range new {
-			if existingDevice.DeviceID != newDevice.DeviceID {
-				continue
-			}
-			existingDevice.DisplayName = newDevice.DisplayName
-			existing[i] = existingDevice
-		}
-	}
-	return existing
-}

From 987d7adc5d4e34827759b206ac4cc17cce78e221 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Sat, 30 Apr 2022 00:07:50 +0200
Subject: [PATCH 061/391] Return "to", if we didn't return any presence events
 (#2407)

Return correct stream position, if we didn't return any presence events
---
 syncapi/streams/stream_presence.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index a84d19878..35ce53cb6 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -145,6 +145,10 @@ func (p *PresenceStreamProvider) IncrementalSync(
 		p.cache.Store(cacheKey, presence)
 	}
 
+	if len(req.Response.Presence.Events) == 0 {
+		return to
+	}
+
 	return lastPos
 }
 

From bfa344e83191c49bdc9917ab7a8ec31b93c202e9 Mon Sep 17 00:00:00 2001
From: Brian Meek <brian@hntlabs.com>
Date: Fri, 29 Apr 2022 15:23:11 -0700
Subject: [PATCH 062/391] Test_Devices, sqlite may return devices in different
 order, test should still pass (#2406)

---
 userapi/storage/storage_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index 2eb57d0bc..ef25b8000 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -168,7 +168,7 @@ func Test_Devices(t *testing.T) {
 
 		devices2, err := db.GetDevicesByID(ctx, deviceIDs)
 		assert.NoError(t, err, "unable to get devices by id")
-		assert.Equal(t, devices, devices2)
+		assert.ElementsMatch(t, devices, devices2)
 
 		// Update device
 		newName := "new display name"

From 979a551f1e2aeb9f3417df5e52a7279230b7a3ba Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 2 May 2022 10:47:16 +0200
Subject: [PATCH 063/391] Return `null` if MaxFileSizeBytes is 0 (#2409)

* Return "null" if MaxFileSizeBytes is 0

* Add comment and nil check (better save than sorry)

* Simplify config
---
 mediaapi/routing/download.go    |  2 +-
 mediaapi/routing/routing.go     |  8 ++++++--
 mediaapi/routing/upload.go      | 18 +++++++++---------
 mediaapi/routing/upload_test.go |  5 ++---
 setup/config/config_mediaapi.go |  6 +++---
 5 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go
index 5f22a9461..10b25a5cd 100644
--- a/mediaapi/routing/download.go
+++ b/mediaapi/routing/download.go
@@ -551,7 +551,7 @@ func (r *downloadRequest) getRemoteFile(
 			// If we do not have a record, we need to fetch the remote file first and then respond from the local file
 			err := r.fetchRemoteFileAndStoreMetadata(
 				ctx, client,
-				cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db,
+				cfg.AbsBasePath, cfg.MaxFileSizeBytes, db,
 				cfg.ThumbnailSizes, activeThumbnailGeneration,
 				cfg.MaxThumbnailGenerators,
 			)
diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go
index 0e1583991..97dfd3341 100644
--- a/mediaapi/routing/routing.go
+++ b/mediaapi/routing/routing.go
@@ -35,7 +35,7 @@ import (
 // configResponse is the response to GET /_matrix/media/r0/config
 // https://matrix.org/docs/spec/client_server/latest#get-matrix-media-r0-config
 type configResponse struct {
-	UploadSize config.FileSizeBytes `json:"m.upload.size"`
+	UploadSize *config.FileSizeBytes `json:"m.upload.size"`
 }
 
 // Setup registers the media API HTTP handlers
@@ -73,9 +73,13 @@ func Setup(
 		if r := rateLimits.Limit(req); r != nil {
 			return *r
 		}
+		respondSize := &cfg.MaxFileSizeBytes
+		if cfg.MaxFileSizeBytes == 0 {
+			respondSize = nil
+		}
 		return util.JSONResponse{
 			Code: http.StatusOK,
-			JSON: configResponse{UploadSize: *cfg.MaxFileSizeBytes},
+			JSON: configResponse{UploadSize: respondSize},
 		}
 	})
 
diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go
index 972c52af0..2175648ea 100644
--- a/mediaapi/routing/upload.go
+++ b/mediaapi/routing/upload.go
@@ -90,7 +90,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI, dev *usera
 		Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName),
 	}
 
-	if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil {
+	if resErr := r.Validate(cfg.MaxFileSizeBytes); resErr != nil {
 		return nil, resErr
 	}
 
@@ -148,20 +148,20 @@ func (r *uploadRequest) doUpload(
 	//   r.storeFileAndMetadata(ctx, tmpDir, ...)
 	// before you return from doUpload else we will leak a temp file. We could make this nicer with a `WithTransaction` style of
 	// nested function to guarantee either storage or cleanup.
-	if *cfg.MaxFileSizeBytes > 0 {
-		if *cfg.MaxFileSizeBytes+1 <= 0 {
+	if cfg.MaxFileSizeBytes > 0 {
+		if cfg.MaxFileSizeBytes+1 <= 0 {
 			r.Logger.WithFields(log.Fields{
-				"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
+				"MaxFileSizeBytes": cfg.MaxFileSizeBytes,
 			}).Warnf("Configured MaxFileSizeBytes overflows int64, defaulting to %d bytes", config.DefaultMaxFileSizeBytes)
-			cfg.MaxFileSizeBytes = &config.DefaultMaxFileSizeBytes
+			cfg.MaxFileSizeBytes = config.DefaultMaxFileSizeBytes
 		}
-		reqReader = io.LimitReader(reqReader, int64(*cfg.MaxFileSizeBytes)+1)
+		reqReader = io.LimitReader(reqReader, int64(cfg.MaxFileSizeBytes)+1)
 	}
 
 	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, reqReader, cfg.AbsBasePath)
 	if err != nil {
 		r.Logger.WithError(err).WithFields(log.Fields{
-			"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
+			"MaxFileSizeBytes": cfg.MaxFileSizeBytes,
 		}).Warn("Error while transferring file")
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -170,9 +170,9 @@ func (r *uploadRequest) doUpload(
 	}
 
 	// Check if temp file size exceeds max file size configuration
-	if *cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(*cfg.MaxFileSizeBytes) {
+	if cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(cfg.MaxFileSizeBytes) {
 		fileutils.RemoveDir(tmpDir, r.Logger) // delete temp file
-		return requestEntityTooLargeJSONResponse(*cfg.MaxFileSizeBytes)
+		return requestEntityTooLargeJSONResponse(cfg.MaxFileSizeBytes)
 	}
 
 	// Look up the media by the file hash. If we already have the file but under a
diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go
index b2c2f5a44..e04c010f8 100644
--- a/mediaapi/routing/upload_test.go
+++ b/mediaapi/routing/upload_test.go
@@ -36,12 +36,11 @@ func Test_uploadRequest_doUpload(t *testing.T) {
 	}
 
 	maxSize := config.FileSizeBytes(8)
-	unlimitedSize := config.FileSizeBytes(0)
 	logger := log.New().WithField("mediaapi", "test")
 	testdataPath := filepath.Join(wd, "./testdata")
 
 	cfg := &config.MediaAPI{
-		MaxFileSizeBytes:  &maxSize,
+		MaxFileSizeBytes:  maxSize,
 		BasePath:          config.Path(testdataPath),
 		AbsBasePath:       config.Path(testdataPath),
 		DynamicThumbnails: false,
@@ -124,7 +123,7 @@ func Test_uploadRequest_doUpload(t *testing.T) {
 				ctx:       context.Background(),
 				reqReader: strings.NewReader("test test test"),
 				cfg: &config.MediaAPI{
-					MaxFileSizeBytes:  &unlimitedSize,
+					MaxFileSizeBytes:  config.FileSizeBytes(0),
 					BasePath:          config.Path(testdataPath),
 					AbsBasePath:       config.Path(testdataPath),
 					DynamicThumbnails: false,
diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go
index 9a7d84969..c85020d2a 100644
--- a/setup/config/config_mediaapi.go
+++ b/setup/config/config_mediaapi.go
@@ -23,7 +23,7 @@ type MediaAPI struct {
 	// The maximum file size in bytes that is allowed to be stored on this server.
 	// Note: if max_file_size_bytes is set to 0, the size is unlimited.
 	// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
-	MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
+	MaxFileSizeBytes FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
 
 	// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
 	DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
@@ -48,7 +48,7 @@ func (c *MediaAPI) Defaults(generate bool) {
 		c.BasePath = "./media_store"
 	}
 
-	c.MaxFileSizeBytes = &DefaultMaxFileSizeBytes
+	c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
 	c.MaxThumbnailGenerators = 10
 }
 
@@ -61,7 +61,7 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
 
 	checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
-	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes))
+	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes))
 	checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
 
 	for i, size := range c.ThumbnailSizes {

From 4ad5f9c982fe5dc9e306a9269621ead8c31248cf Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 3 May 2022 16:35:06 +0100
Subject: [PATCH 064/391] Global database connection pool (for monolith mode)
 (#2411)

* Allow monolith components to share a single database pool

* Don't yell about missing connection strings

* Rename field

* Setup tweaks

* Fix panic

* Improve configuration checks

* Update config

* Fix lint errors

* Update comments
---
 appservice/appservice.go                      |  2 +-
 appservice/storage/postgres/storage.go        |  6 +-
 appservice/storage/sqlite3/storage.go         |  6 +-
 appservice/storage/storage.go                 |  7 +-
 appservice/storage/storage_wasm.go            |  5 +-
 build/gobind-pinecone/monolith.go             |  6 +-
 build/gobind-yggdrasil/monolith.go            |  6 +-
 cmd/create-account/main.go                    | 16 ++++-
 cmd/dendrite-demo-pinecone/main.go            |  6 +-
 cmd/dendrite-demo-yggdrasil/main.go           |  6 +-
 cmd/dendrite-monolith-server/main.go          |  6 +-
 .../personalities/mediaapi.go                 |  5 +-
 .../personalities/syncapi.go                  |  2 +-
 .../personalities/userapi.go                  |  4 +-
 cmd/dendritejs-pinecone/main.go               |  6 +-
 cmd/resolve-state/main.go                     |  2 +-
 dendrite-config.yaml                          | 10 +++
 federationapi/federationapi.go                |  2 +-
 federationapi/storage/postgres/storage.go     |  6 +-
 federationapi/storage/sqlite3/storage.go      |  6 +-
 federationapi/storage/storage.go              |  7 +-
 federationapi/storage/storage_wasm.go         |  5 +-
 internal/sqlutil/sqlutil.go                   | 51 ++++++++++++++
 internal/sqlutil/trace.go                     | 44 ------------
 keyserver/keyserver.go                        |  2 +-
 keyserver/storage/postgres/storage.go         |  7 +-
 keyserver/storage/sqlite3/storage.go          |  7 +-
 keyserver/storage/storage.go                  |  7 +-
 keyserver/storage/storage_test.go             |  2 +-
 keyserver/storage/storage_wasm.go             |  5 +-
 mediaapi/mediaapi.go                          |  4 +-
 mediaapi/routing/upload_test.go               |  2 +-
 mediaapi/storage/postgres/mediaapi.go         |  7 +-
 mediaapi/storage/sqlite3/mediaapi.go          |  7 +-
 mediaapi/storage/storage.go                   |  7 +-
 mediaapi/storage/storage_test.go              |  2 +-
 mediaapi/storage/storage_wasm.go              |  5 +-
 roomserver/internal/input/input_test.go       |  1 +
 roomserver/roomserver.go                      |  2 +-
 roomserver/storage/postgres/storage.go        | 13 ++--
 roomserver/storage/sqlite3/storage.go         | 18 ++---
 roomserver/storage/storage.go                 |  7 +-
 roomserver/storage/storage_wasm.go            |  5 +-
 setup/base/base.go                            | 69 +++++++++++++------
 setup/config/config_appservice.go             |  4 +-
 setup/config/config_federationapi.go          |  4 +-
 setup/config/config_global.go                 |  7 ++
 setup/config/config_keyserver.go              |  4 +-
 setup/config/config_mediaapi.go               |  4 +-
 setup/config/config_mscs.go                   |  4 +-
 setup/config/config_roomserver.go             |  4 +-
 setup/config/config_syncapi.go                |  4 +-
 setup/config/config_userapi.go                |  4 +-
 setup/monolith.go                             | 22 +++---
 setup/mscs/msc2836/msc2836.go                 |  2 +-
 setup/mscs/msc2836/storage.go                 | 23 +++----
 syncapi/storage/postgres/syncserver.go        |  6 +-
 syncapi/storage/sqlite3/syncserver.go         |  6 +-
 syncapi/storage/storage.go                    |  7 +-
 syncapi/storage/storage_test.go               |  2 +-
 syncapi/storage/storage_wasm.go               |  5 +-
 .../storage/tables/output_room_events_test.go |  2 +-
 syncapi/storage/tables/topology_test.go       |  2 +-
 syncapi/syncapi.go                            | 24 +++----
 userapi/storage/postgres/storage.go           |  7 +-
 userapi/storage/sqlite3/storage.go            |  7 +-
 userapi/storage/storage.go                    |  7 +-
 userapi/storage/storage_test.go               |  2 +-
 userapi/storage/storage_wasm.go               |  4 +-
 userapi/userapi.go                            | 15 +++-
 userapi/userapi_test.go                       |  2 +-
 71 files changed, 345 insertions(+), 240 deletions(-)
 create mode 100644 internal/sqlutil/sqlutil.go

diff --git a/appservice/appservice.go b/appservice/appservice.go
index b99091866..0db2c1009 100644
--- a/appservice/appservice.go
+++ b/appservice/appservice.go
@@ -62,7 +62,7 @@ func NewInternalAPI(
 	js, _ := jetstream.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 
 	// Create a connection to the appservice postgres DB
-	appserviceDB, err := storage.NewDatabase(&base.Cfg.AppServiceAPI.Database)
+	appserviceDB, err := storage.NewDatabase(base, &base.Cfg.AppServiceAPI.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to appservice db")
 	}
diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go
index eaf947ff3..a4c04b2cc 100644
--- a/appservice/storage/postgres/storage.go
+++ b/appservice/storage/postgres/storage.go
@@ -22,6 +22,7 @@ import (
 	// Import postgres database driver
 	_ "github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -35,13 +36,12 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open(dbProperties); err != nil {
+	if result.db, result.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
 		return nil, err
 	}
-	result.writer = sqlutil.NewDummyWriter()
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go
index 9260c7fe7..ad62b3628 100644
--- a/appservice/storage/sqlite3/storage.go
+++ b/appservice/storage/sqlite3/storage.go
@@ -21,6 +21,7 @@ import (
 
 	// Import SQLite database driver
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -34,13 +35,12 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open(dbProperties); err != nil {
+	if result.db, result.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
 		return nil, err
 	}
-	result.writer = sqlutil.NewExclusiveWriter()
 	if err = result.prepare(); err != nil {
 		return nil, err
 	}
diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go
index 97b8501e2..89d5e0cc2 100644
--- a/appservice/storage/storage.go
+++ b/appservice/storage/storage.go
@@ -22,17 +22,18 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage/postgres"
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets DB connection parameters
-func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(dbProperties)
+		return postgres.NewDatabase(base, dbProperties)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go
index 07d0e9ee1..230254598 100644
--- a/appservice/storage/storage_wasm.go
+++ b/appservice/storage/storage_wasm.go
@@ -18,13 +18,14 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index d047f3fff..8cf663d00 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -268,7 +268,6 @@ func (m *DendriteMonolith) Start() {
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
 	federation := conn.CreateFederationClient(base, m.PineconeQUIC)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
@@ -281,7 +280,7 @@ func (m *DendriteMonolith) Start() {
 	)
 
 	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	m.userAPI = userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	m.userAPI = userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(m.userAPI)
 
 	asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
@@ -295,7 +294,6 @@ func (m *DendriteMonolith) Start() {
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
-		AccountDB: accountDB,
 		Client:    conn.CreateClient(base, m.PineconeQUIC),
 		FedClient: federation,
 		KeyRing:   keyRing,
@@ -309,7 +307,7 @@ func (m *DendriteMonolith) Start() {
 		ExtUserDirectoryProvider: userProvider,
 	}
 	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
+		base,
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 4e95e3972..2c7d4e91b 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -107,7 +107,6 @@ func (m *DendriteMonolith) Start() {
 	m.processContext = base.ProcessContext
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
 	federation := ygg.CreateFederationClient(base)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
@@ -120,7 +119,7 @@ func (m *DendriteMonolith) Start() {
 	)
 
 	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
-	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
@@ -132,7 +131,6 @@ func (m *DendriteMonolith) Start() {
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
-		AccountDB: accountDB,
 		Client:    ygg.CreateClient(base),
 		FedClient: federation,
 		KeyRing:   keyRing,
@@ -147,7 +145,7 @@ func (m *DendriteMonolith) Start() {
 		),
 	}
 	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
+		base,
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 2719f8680..7a5660522 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -25,8 +25,8 @@ import (
 	"strings"
 
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/term"
 )
@@ -99,8 +99,18 @@ func main() {
 		}
 	}
 
-	b := base.NewBaseDendrite(cfg, "Monolith")
-	accountDB := b.CreateAccountsDB()
+	accountDB, err := storage.NewUserAPIDatabase(
+		nil,
+		&cfg.UserAPI.AccountDatabase,
+		cfg.Global.ServerName,
+		cfg.UserAPI.BCryptCost,
+		cfg.UserAPI.OpenIDTokenLifetimeMS,
+		0, // TODO
+		cfg.Global.ServerNotices.LocalPart,
+	)
+	if err != nil {
+		logrus.WithError(err).Fatalln("Failed to connect to the database")
+	}
 
 	accType := api.AccountTypeUser
 	if *isAdmin {
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 785e7b460..33487e64b 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -149,7 +149,6 @@ func main() {
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
 	federation := conn.CreateFederationClient(base, pQUIC)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
@@ -162,7 +161,7 @@ func main() {
 	)
 
 	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
@@ -174,7 +173,6 @@ func main() {
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
-		AccountDB: accountDB,
 		Client:    conn.CreateClient(base, pQUIC),
 		FedClient: federation,
 		KeyRing:   keyRing,
@@ -188,7 +186,7 @@ func main() {
 		ExtUserDirectoryProvider: userProvider,
 	}
 	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
+		base,
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index f9234319a..df9ba5121 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -104,7 +104,6 @@ func main() {
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
 	federation := ygg.CreateFederationClient(base)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
@@ -117,7 +116,7 @@ func main() {
 	)
 	rsAPI := rsComponent
 
-	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
@@ -130,7 +129,6 @@ func main() {
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
-		AccountDB: accountDB,
 		Client:    ygg.CreateClient(base),
 		FedClient: federation,
 		KeyRing:   keyRing,
@@ -145,7 +143,7 @@ func main() {
 		),
 	}
 	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
+		base,
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index 5fd5c0b5d..4c7c4297f 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -71,7 +71,6 @@ func main() {
 	base := basepkg.NewBaseDendrite(cfg, "Monolith", options...)
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
 	federation := base.CreateFederationClient()
 
 	rsImpl := roomserver.NewInternalAPI(base)
@@ -104,7 +103,7 @@ func main() {
 	}
 
 	pgClient := base.PushGatewayHTTPClient()
-	userImpl := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
+	userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
 	userAPI := userImpl
 	if base.UseHTTPAPIs {
 		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
@@ -135,7 +134,6 @@ func main() {
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
-		AccountDB: accountDB,
 		Client:    base.CreateClient(),
 		FedClient: federation,
 		KeyRing:   keyRing,
@@ -146,7 +144,7 @@ func main() {
 		KeyAPI:        keyAPI,
 	}
 	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
+		base,
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
diff --git a/cmd/dendrite-polylith-multi/personalities/mediaapi.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go
index fa9d36a38..8c0bfa195 100644
--- a/cmd/dendrite-polylith-multi/personalities/mediaapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go
@@ -24,7 +24,10 @@ func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	client := base.CreateClient()
 
-	mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, &base.Cfg.ClientAPI.RateLimiting, userAPI, client)
+	mediaapi.AddPublicRoutes(
+		base, base.PublicMediaAPIMux, &base.Cfg.MediaAPI, &base.Cfg.ClientAPI.RateLimiting,
+		userAPI, client,
+	)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.MediaAPI.InternalAPI.Listen,
diff --git a/cmd/dendrite-polylith-multi/personalities/syncapi.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go
index 6fee8419b..f9f1c5a09 100644
--- a/cmd/dendrite-polylith-multi/personalities/syncapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go
@@ -27,7 +27,7 @@ func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	rsAPI := base.RoomserverHTTPClient()
 
 	syncapi.AddPublicRoutes(
-		base.ProcessContext,
+		base,
 		base.PublicClientAPIMux, userAPI, rsAPI,
 		base.KeyServerHTTPClient(),
 		federation, &cfg.SyncAPI,
diff --git a/cmd/dendrite-polylith-multi/personalities/userapi.go b/cmd/dendrite-polylith-multi/personalities/userapi.go
index f1fa379c7..3fe5a43d7 100644
--- a/cmd/dendrite-polylith-multi/personalities/userapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/userapi.go
@@ -21,10 +21,8 @@ import (
 )
 
 func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	accountDB := base.CreateAccountsDB()
-
 	userAPI := userapi.NewInternalAPI(
-		base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices,
+		base, &cfg.UserAPI, cfg.Derived.ApplicationServices,
 		base.KeyServerHTTPClient(), base.RoomserverHTTPClient(),
 		base.PushGatewayHTTPClient(),
 	)
diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
index 5ecf1f2fb..ead381368 100644
--- a/cmd/dendritejs-pinecone/main.go
+++ b/cmd/dendritejs-pinecone/main.go
@@ -180,7 +180,6 @@ func startup() {
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck
 
-	accountDB := base.CreateAccountsDB()
 	federation := conn.CreateFederationClient(base, pSessions)
 	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 
@@ -189,7 +188,7 @@ func startup() {
 
 	rsAPI := roomserver.NewInternalAPI(base)
 
-	userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
+	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
 	asQuery := appservice.NewInternalAPI(
@@ -201,7 +200,6 @@ func startup() {
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
-		AccountDB: accountDB,
 		Client:    conn.CreateClient(base, pSessions),
 		FedClient: federation,
 		KeyRing:   keyRing,
@@ -215,7 +213,7 @@ func startup() {
 		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
 	}
 	monolith.AddAllPublicRoutes(
-		base.ProcessContext,
+		base,
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index 30331fbb3..c52fd6c42 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -45,7 +45,7 @@ func main() {
 		panic(err)
 	}
 
-	roomserverDB, err := storage.Open(&cfg.RoomServer.Database, cache)
+	roomserverDB, err := storage.Open(nil, &cfg.RoomServer.Database, cache)
 	if err != nil {
 		panic(err)
 	}
diff --git a/dendrite-config.yaml b/dendrite-config.yaml
index 1c11ef96d..1647af15d 100644
--- a/dendrite-config.yaml
+++ b/dendrite-config.yaml
@@ -54,6 +54,16 @@ global:
   # considered valid by other homeservers.
   key_validity_period: 168h0m0s
 
+  # Global database connection pool, for PostgreSQL monolith deployments only. If
+  # this section is populated then you can omit the "database" blocks in all other
+  # sections. For polylith deployments, or monolith deployments using SQLite databases,
+  # you must configure the "database" block for each component instead.
+  # database:
+  #   connection_string: postgres://user:pass@hostname/database?sslmode=disable
+  #   max_open_conns: 100
+  #   max_idle_conns: 5
+  #   conn_max_lifetime: -1
+
   # The server name to delegate server-server communications to, with optional port
   # e.g. localhost:443
   well_known_server_name: ""
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index 5bfe237a8..1848a242e 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -91,7 +91,7 @@ func NewInternalAPI(
 ) api.FederationInternalAPI {
 	cfg := &base.Cfg.FederationAPI
 
-	federationDB, err := storage.NewDatabase(&cfg.Database, base.Caches, base.Cfg.Global.ServerName)
+	federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
 	}
diff --git a/federationapi/storage/postgres/storage.go b/federationapi/storage/postgres/storage.go
index b2aea6929..9863afb2b 100644
--- a/federationapi/storage/postgres/storage.go
+++ b/federationapi/storage/postgres/storage.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -35,13 +36,12 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
+	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
 		return nil, err
 	}
-	d.writer = sqlutil.NewDummyWriter()
 	joinedHosts, err := NewPostgresJoinedHostsTable(d.db)
 	if err != nil {
 		return nil, err
diff --git a/federationapi/storage/sqlite3/storage.go b/federationapi/storage/sqlite3/storage.go
index c2e83211e..7d0cee90e 100644
--- a/federationapi/storage/sqlite3/storage.go
+++ b/federationapi/storage/sqlite3/storage.go
@@ -22,6 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -34,13 +35,12 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
+	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
 		return nil, err
 	}
-	d.writer = sqlutil.NewExclusiveWriter()
 	joinedHosts, err := NewSQLiteJoinedHostsTable(d.db)
 	if err != nil {
 		return nil, err
diff --git a/federationapi/storage/storage.go b/federationapi/storage/storage.go
index 4b52ca206..f246b9bc9 100644
--- a/federationapi/storage/storage.go
+++ b/federationapi/storage/storage.go
@@ -23,17 +23,18 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/storage/postgres"
 	"github.com/matrix-org/dendrite/federationapi/storage/sqlite3"
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties, cache, serverName)
+		return sqlite3.NewDatabase(base, dbProperties, cache, serverName)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(dbProperties, cache, serverName)
+		return postgres.NewDatabase(base, dbProperties, cache, serverName)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/federationapi/storage/storage_wasm.go b/federationapi/storage/storage_wasm.go
index 09abed63e..84d5a3a4c 100644
--- a/federationapi/storage/storage_wasm.go
+++ b/federationapi/storage/storage_wasm.go
@@ -19,15 +19,16 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/storage/sqlite3"
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties, cache, serverName)
+		return sqlite3.NewDatabase(base, dbProperties, cache, serverName)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go
new file mode 100644
index 000000000..0cdae6d30
--- /dev/null
+++ b/internal/sqlutil/sqlutil.go
@@ -0,0 +1,51 @@
+package sqlutil
+
+import (
+	"database/sql"
+	"fmt"
+	"regexp"
+
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/sirupsen/logrus"
+)
+
+// Open opens a database specified by its database driver name and a driver-specific data source name,
+// usually consisting of at least a database name and connection information. Includes tracing driver
+// if DENDRITE_TRACE_SQL=1
+func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) {
+	var err error
+	var driverName, dsn string
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		driverName = "sqlite3"
+		dsn, err = ParseFileURI(dbProperties.ConnectionString)
+		if err != nil {
+			return nil, fmt.Errorf("ParseFileURI: %w", err)
+		}
+	case dbProperties.ConnectionString.IsPostgres():
+		driverName = "postgres"
+		dsn = string(dbProperties.ConnectionString)
+	default:
+		return nil, fmt.Errorf("invalid database connection string %q", dbProperties.ConnectionString)
+	}
+	if tracingEnabled {
+		// install the wrapped driver
+		driverName += "-trace"
+	}
+	db, err := sql.Open(driverName, dsn)
+	if err != nil {
+		return nil, err
+	}
+	if driverName != "sqlite3" {
+		logrus.WithFields(logrus.Fields{
+			"MaxOpenConns":    dbProperties.MaxOpenConns(),
+			"MaxIdleConns":    dbProperties.MaxIdleConns(),
+			"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
+			"dataSourceName":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
+		}).Debug("Setting DB connection limits")
+		db.SetMaxOpenConns(dbProperties.MaxOpenConns())
+		db.SetMaxIdleConns(dbProperties.MaxIdleConns())
+		db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime())
+	}
+	return db, nil
+}
diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go
index 51eaa1b45..c16738616 100644
--- a/internal/sqlutil/trace.go
+++ b/internal/sqlutil/trace.go
@@ -16,19 +16,16 @@ package sqlutil
 
 import (
 	"context"
-	"database/sql"
 	"database/sql/driver"
 	"fmt"
 	"io"
 	"os"
-	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/ngrok/sqlmw"
 	"github.com/sirupsen/logrus"
 )
@@ -96,47 +93,6 @@ func trackGoID(query string) {
 	logrus.Warnf("unsafe goid %d: SQL executed not on an ExclusiveWriter: %s", thisGoID, q)
 }
 
-// Open opens a database specified by its database driver name and a driver-specific data source name,
-// usually consisting of at least a database name and connection information. Includes tracing driver
-// if DENDRITE_TRACE_SQL=1
-func Open(dbProperties *config.DatabaseOptions) (*sql.DB, error) {
-	var err error
-	var driverName, dsn string
-	switch {
-	case dbProperties.ConnectionString.IsSQLite():
-		driverName = "sqlite3"
-		dsn, err = ParseFileURI(dbProperties.ConnectionString)
-		if err != nil {
-			return nil, fmt.Errorf("ParseFileURI: %w", err)
-		}
-	case dbProperties.ConnectionString.IsPostgres():
-		driverName = "postgres"
-		dsn = string(dbProperties.ConnectionString)
-	default:
-		return nil, fmt.Errorf("invalid database connection string %q", dbProperties.ConnectionString)
-	}
-	if tracingEnabled {
-		// install the wrapped driver
-		driverName += "-trace"
-	}
-	db, err := sql.Open(driverName, dsn)
-	if err != nil {
-		return nil, err
-	}
-	if driverName != "sqlite3" {
-		logrus.WithFields(logrus.Fields{
-			"MaxOpenConns":    dbProperties.MaxOpenConns(),
-			"MaxIdleConns":    dbProperties.MaxIdleConns(),
-			"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
-			"dataSourceName":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
-		}).Debug("Setting DB connection limits")
-		db.SetMaxOpenConns(dbProperties.MaxOpenConns())
-		db.SetMaxIdleConns(dbProperties.MaxIdleConns())
-		db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime())
-	}
-	return db, nil
-}
-
 func init() {
 	registerDrivers()
 }
diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go
index c557dfbaa..007a48a55 100644
--- a/keyserver/keyserver.go
+++ b/keyserver/keyserver.go
@@ -41,7 +41,7 @@ func NewInternalAPI(
 ) api.KeyInternalAPI {
 	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
-	db, err := storage.NewDatabase(&cfg.Database)
+	db, err := storage.NewDatabase(base, &cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to key server database")
 	}
diff --git a/keyserver/storage/postgres/storage.go b/keyserver/storage/postgres/storage.go
index d4c7e2cc7..b8f70acf8 100644
--- a/keyserver/storage/postgres/storage.go
+++ b/keyserver/storage/postgres/storage.go
@@ -18,13 +18,14 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/keyserver/storage/shared"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewDatabase creates a new sync server database
-func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) {
 	var err error
-	db, err := sqlutil.Open(dbProperties)
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter())
 	if err != nil {
 		return nil, err
 	}
@@ -63,7 +64,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error)
 	}
 	d := &shared.Database{
 		DB:                    db,
-		Writer:                sqlutil.NewDummyWriter(),
+		Writer:                writer,
 		OneTimeKeysTable:      otk,
 		DeviceKeysTable:       dk,
 		KeyChangesTable:       kc,
diff --git a/keyserver/storage/sqlite3/storage.go b/keyserver/storage/sqlite3/storage.go
index 84d4cdf55..aeea9eac6 100644
--- a/keyserver/storage/sqlite3/storage.go
+++ b/keyserver/storage/sqlite3/storage.go
@@ -18,11 +18,12 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/storage/shared"
 	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3/deltas"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
-	db, err := sqlutil.Open(dbProperties)
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter())
 	if err != nil {
 		return nil, err
 	}
@@ -62,7 +63,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error)
 	}
 	d := &shared.Database{
 		DB:                    db,
-		Writer:                sqlutil.NewExclusiveWriter(),
+		Writer:                writer,
 		OneTimeKeysTable:      otk,
 		DeviceKeysTable:       dk,
 		KeyChangesTable:       kc,
diff --git a/keyserver/storage/storage.go b/keyserver/storage/storage.go
index 742e8463a..ab6a35401 100644
--- a/keyserver/storage/storage.go
+++ b/keyserver/storage/storage.go
@@ -22,17 +22,18 @@ import (
 
 	"github.com/matrix-org/dendrite/keyserver/storage/postgres"
 	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets postgres connection parameters
-func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(dbProperties)
+		return postgres.NewDatabase(base, dbProperties)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/keyserver/storage/storage_test.go b/keyserver/storage/storage_test.go
index 84d2098ad..9940eac60 100644
--- a/keyserver/storage/storage_test.go
+++ b/keyserver/storage/storage_test.go
@@ -22,7 +22,7 @@ func MustCreateDatabase(t *testing.T) (Database, func()) {
 		log.Fatal(err)
 	}
 	t.Logf("Database %s", tmpfile.Name())
-	db, err := NewDatabase(&config.DatabaseOptions{
+	db, err := NewDatabase(nil, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(fmt.Sprintf("file://%s", tmpfile.Name())),
 	})
 	if err != nil {
diff --git a/keyserver/storage/storage_wasm.go b/keyserver/storage/storage_wasm.go
index 8b31bfd01..75c9053e8 100644
--- a/keyserver/storage/storage_wasm.go
+++ b/keyserver/storage/storage_wasm.go
@@ -18,13 +18,14 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go
index e5daf480d..f2fa14384 100644
--- a/mediaapi/mediaapi.go
+++ b/mediaapi/mediaapi.go
@@ -18,6 +18,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/mediaapi/routing"
 	"github.com/matrix-org/dendrite/mediaapi/storage"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -26,13 +27,14 @@ import (
 
 // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.
 func AddPublicRoutes(
+	base *base.BaseDendrite,
 	router *mux.Router,
 	cfg *config.MediaAPI,
 	rateLimit *config.RateLimiting,
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
 ) {
-	mediaDB, err := storage.NewMediaAPIDatasource(&cfg.Database)
+	mediaDB, err := storage.NewMediaAPIDatasource(base, &cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to media db")
 	}
diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go
index e04c010f8..420d0eba9 100644
--- a/mediaapi/routing/upload_test.go
+++ b/mediaapi/routing/upload_test.go
@@ -50,7 +50,7 @@ func Test_uploadRequest_doUpload(t *testing.T) {
 	_ = os.Mkdir(testdataPath, os.ModePerm)
 	defer fileutils.RemoveDir(types.Path(testdataPath), nil)
 
-	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
+	db, err := storage.NewMediaAPIDatasource(nil, &config.DatabaseOptions{
 		ConnectionString:       "file::memory:?cache=shared",
 		MaxOpenConnections:     100,
 		MaxIdleConnections:     2,
diff --git a/mediaapi/storage/postgres/mediaapi.go b/mediaapi/storage/postgres/mediaapi.go
index ea70e575b..30ec64f84 100644
--- a/mediaapi/storage/postgres/mediaapi.go
+++ b/mediaapi/storage/postgres/mediaapi.go
@@ -20,12 +20,13 @@ import (
 	_ "github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewDatabase opens a postgres database.
-func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
-	db, err := sqlutil.Open(dbProperties)
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter())
 	if err != nil {
 		return nil, err
 	}
@@ -41,6 +42,6 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error)
 		MediaRepository: mediaRepo,
 		Thumbnails:      thumbnails,
 		DB:              db,
-		Writer:          sqlutil.NewExclusiveWriter(),
+		Writer:          writer,
 	}, nil
 }
diff --git a/mediaapi/storage/sqlite3/mediaapi.go b/mediaapi/storage/sqlite3/mediaapi.go
index abf329367..c0ab10e9f 100644
--- a/mediaapi/storage/sqlite3/mediaapi.go
+++ b/mediaapi/storage/sqlite3/mediaapi.go
@@ -19,12 +19,13 @@ import (
 	// Import the postgres database driver.
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewDatabase opens a SQLIte database.
-func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
-	db, err := sqlutil.Open(dbProperties)
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter())
 	if err != nil {
 		return nil, err
 	}
@@ -40,6 +41,6 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error)
 		MediaRepository: mediaRepo,
 		Thumbnails:      thumbnails,
 		DB:              db,
-		Writer:          sqlutil.NewExclusiveWriter(),
+		Writer:          writer,
 	}, nil
 }
diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go
index baa242e57..f673ae7e6 100644
--- a/mediaapi/storage/storage.go
+++ b/mediaapi/storage/storage.go
@@ -22,16 +22,17 @@ import (
 
 	"github.com/matrix-org/dendrite/mediaapi/storage/postgres"
 	"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewMediaAPIDatasource opens a database connection.
-func NewMediaAPIDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewMediaAPIDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(dbProperties)
+		return postgres.NewDatabase(base, dbProperties)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/mediaapi/storage/storage_test.go b/mediaapi/storage/storage_test.go
index fa88cd8e7..81f0a5d24 100644
--- a/mediaapi/storage/storage_test.go
+++ b/mediaapi/storage/storage_test.go
@@ -13,7 +13,7 @@ import (
 
 func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
-	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
+	db, err := storage.NewMediaAPIDatasource(nil, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	})
 	if err != nil {
diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go
index f67f9d5e1..41e4a28c0 100644
--- a/mediaapi/storage/storage_wasm.go
+++ b/mediaapi/storage/storage_wasm.go
@@ -18,14 +18,15 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // Open opens a postgres database.
-func NewMediaAPIDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewMediaAPIDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go
index 81c86ae38..5d34842bf 100644
--- a/roomserver/internal/input/input_test.go
+++ b/roomserver/internal/input/input_test.go
@@ -53,6 +53,7 @@ func TestSingleTransactionOnInput(t *testing.T) {
 		t.Fatal(err)
 	}
 	db, err := storage.Open(
+		nil,
 		&config.DatabaseOptions{
 			ConnectionString:   "",
 			MaxOpenConnections: 1,
diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go
index 36e3c5269..46261eb3e 100644
--- a/roomserver/roomserver.go
+++ b/roomserver/roomserver.go
@@ -45,7 +45,7 @@ func NewInternalAPI(
 		perspectiveServerNames = append(perspectiveServerNames, kp.ServerName)
 	}
 
-	roomserverDB, err := storage.Open(&cfg.Database, base.Caches)
+	roomserverDB, err := storage.Open(base, &cfg.Database, base.Caches)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to room server db")
 	}
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index b5e05c982..da8d25848 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
@@ -35,11 +36,11 @@ type Database struct {
 }
 
 // Open a postgres database.
-func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
+func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
 	var d Database
-	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open(dbProperties); err != nil {
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter())
+	if err != nil {
 		return nil, fmt.Errorf("sqlutil.Open: %w", err)
 	}
 
@@ -59,7 +60,7 @@ func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches)
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
 	// to in the database code should now exist.
-	if err := d.prepare(db, cache); err != nil {
+	if err := d.prepare(db, writer, cache); err != nil {
 		return nil, err
 	}
 
@@ -110,7 +111,7 @@ func (d *Database) create(db *sql.DB) error {
 	return nil
 }
 
-func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error {
+func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.RoomServerCaches) error {
 	eventStateKeys, err := prepareEventStateKeysTable(db)
 	if err != nil {
 		return err
@@ -166,7 +167,7 @@ func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error {
 	d.Database = shared.Database{
 		DB:                  db,
 		Cache:               cache,
-		Writer:              sqlutil.NewDummyWriter(),
+		Writer:              writer,
 		EventTypesTable:     eventTypes,
 		EventStateKeysTable: eventStateKeys,
 		EventJSONTable:      eventJSON,
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 325c253b5..e6cf1a53f 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -18,12 +18,14 @@ package sqlite3
 import (
 	"context"
 	"database/sql"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -34,12 +36,12 @@ type Database struct {
 }
 
 // Open a sqlite database.
-func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
+func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
 	var d Database
-	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open(dbProperties); err != nil {
-		return nil, err
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter())
+	if err != nil {
+		return nil, fmt.Errorf("sqlutil.Open: %w", err)
 	}
 
 	//db.Exec("PRAGMA journal_mode=WAL;")
@@ -49,7 +51,7 @@ func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches)
 	// cause the roomserver to be unresponsive to new events because something will
 	// acquire the global mutex and never unlock it because it is waiting for a connection
 	// which it will never obtain.
-	db.SetMaxOpenConns(20)
+	// db.SetMaxOpenConns(20)
 
 	// Create the tables.
 	if err := d.create(db); err != nil {
@@ -67,7 +69,7 @@ func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches)
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
 	// to in the database code should now exist.
-	if err := d.prepare(db, cache); err != nil {
+	if err := d.prepare(db, writer, cache); err != nil {
 		return nil, err
 	}
 
@@ -118,7 +120,7 @@ func (d *Database) create(db *sql.DB) error {
 	return nil
 }
 
-func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error {
+func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.RoomServerCaches) error {
 	eventStateKeys, err := prepareEventStateKeysTable(db)
 	if err != nil {
 		return err
@@ -174,7 +176,7 @@ func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error {
 	d.Database = shared.Database{
 		DB:                  db,
 		Cache:               cache,
-		Writer:              sqlutil.NewExclusiveWriter(),
+		Writer:              writer,
 		EventsTable:         events,
 		EventTypesTable:     eventTypes,
 		EventStateKeysTable: eventStateKeys,
diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go
index 9f98ea3ed..8a87b7d7c 100644
--- a/roomserver/storage/storage.go
+++ b/roomserver/storage/storage.go
@@ -23,16 +23,17 @@ import (
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // Open opens a database connection.
-func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
+func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.Open(dbProperties, cache)
+		return sqlite3.Open(base, dbProperties, cache)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.Open(dbProperties, cache)
+		return postgres.Open(base, dbProperties, cache)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go
index dfc374e6e..df5a56ac3 100644
--- a/roomserver/storage/storage_wasm.go
+++ b/roomserver/storage/storage_wasm.go
@@ -19,14 +19,15 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
+func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.Open(dbProperties, cache)
+		return sqlite3.Open(base, dbProperties, cache)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/setup/base/base.go b/setup/base/base.go
index 4b771aa36..9b227b70b 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -17,6 +17,7 @@ package base
 import (
 	"context"
 	"crypto/tls"
+	"database/sql"
 	"fmt"
 	"io"
 	"net"
@@ -32,6 +33,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushgateway"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/atomic"
@@ -40,7 +42,6 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/process"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 
 	"github.com/gorilla/mux"
 	"github.com/kardianos/minwinsvc"
@@ -81,6 +82,8 @@ type BaseDendrite struct {
 	Cfg                    *config.Dendrite
 	Caches                 *caching.Caches
 	DNSCache               *gomatrixserverlib.DNSCache
+	Database               *sql.DB
+	DatabaseWriter         sqlutil.Writer
 }
 
 const NoListener = ""
@@ -112,7 +115,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 	}
 
 	configErrors := &config.ConfigErrors{}
-	cfg.Verify(configErrors, componentName == "Monolith") // TODO: better way?
+	isMonolith := componentName == "Monolith" // TODO: better way?
+	cfg.Verify(configErrors, isMonolith)
 	if len(*configErrors) > 0 {
 		for _, err := range *configErrors {
 			logrus.Errorf("Configuration error: %s", err)
@@ -185,6 +189,24 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		},
 	}
 
+	// If we're in monolith mode, we'll set up a global pool of database
+	// connections. A component is welcome to use this pool if they don't
+	// have a separate database config of their own.
+	var db *sql.DB
+	var writer sqlutil.Writer
+	if cfg.Global.DatabaseOptions.ConnectionString != "" {
+		if !isMonolith {
+			logrus.Panic("Using a global database connection pool is not supported in polylith deployments")
+		}
+		if cfg.Global.DatabaseOptions.ConnectionString.IsSQLite() {
+			logrus.Panic("Using a global database connection pool is not supported with SQLite databases")
+		}
+		if db, err = sqlutil.Open(&cfg.Global.DatabaseOptions, sqlutil.NewDummyWriter()); err != nil {
+			logrus.WithError(err).Panic("Failed to set up global database connections")
+		}
+		logrus.Debug("Using global database connection pool")
+	}
+
 	// Ideally we would only use SkipClean on routes which we know can allow '/' but due to
 	// https://github.com/gorilla/mux/issues/460 we have to attach this at the top router.
 	// When used in conjunction with UseEncodedPath() we get the behaviour we want when parsing
@@ -214,6 +236,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		DendriteAdminMux:       mux.NewRouter().SkipClean(true).PathPrefix(httputil.DendriteAdminPathPrefix).Subrouter().UseEncodedPath(),
 		SynapseAdminMux:        mux.NewRouter().SkipClean(true).PathPrefix(httputil.SynapseAdminPathPrefix).Subrouter().UseEncodedPath(),
 		apiHttpClient:          &apiClient,
+		Database:               db,     // set if monolith with global connection pool only
+		DatabaseWriter:         writer, // set if monolith with global connection pool only
 	}
 }
 
@@ -222,6 +246,29 @@ func (b *BaseDendrite) Close() error {
 	return b.tracerCloser.Close()
 }
 
+// DatabaseConnection assists in setting up a database connection. It accepts
+// the database properties and a new writer for the given component. If we're
+// running in monolith mode with a global connection pool configured then we
+// will return that connection, along with the global writer, effectively
+// ignoring the options provided. Otherwise we'll open a new database connection
+// using the supplied options and writer. Note that it's possible for the pointer
+// receiver to be nil here – that's deliberate as some of the unit tests don't
+// have a BaseDendrite and just want a connection with the supplied config
+// without any pooling stuff.
+func (b *BaseDendrite) DatabaseConnection(dbProperties *config.DatabaseOptions, writer sqlutil.Writer) (*sql.DB, sqlutil.Writer, error) {
+	if dbProperties.ConnectionString != "" || b == nil {
+		// Open a new database connection using the supplied config.
+		db, err := sqlutil.Open(dbProperties, writer)
+		return db, writer, err
+	}
+	if b.Database != nil && b.DatabaseWriter != nil {
+		// Ignore the supplied config and return the global pool and
+		// writer.
+		return b.Database, b.DatabaseWriter, nil
+	}
+	return nil, nil, fmt.Errorf("no database connections configured")
+}
+
 // AppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP.
 func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI {
 	a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.apiHttpClient)
@@ -273,24 +320,6 @@ func (b *BaseDendrite) PushGatewayHTTPClient() pushgateway.Client {
 	return pushgateway.NewHTTPClient(b.Cfg.UserAPI.PushGatewayDisableTLSValidation)
 }
 
-// CreateAccountsDB creates a new instance of the accounts database. Should only
-// be called once per component.
-func (b *BaseDendrite) CreateAccountsDB() userdb.Database {
-	db, err := userdb.NewUserAPIDatabase(
-		&b.Cfg.UserAPI.AccountDatabase,
-		b.Cfg.Global.ServerName,
-		b.Cfg.UserAPI.BCryptCost,
-		b.Cfg.UserAPI.OpenIDTokenLifetimeMS,
-		userapi.DefaultLoginTokenLifetime,
-		b.Cfg.Global.ServerNotices.LocalPart,
-	)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to accounts db")
-	}
-
-	return db
-}
-
 // CreateClient creates a new client (normally used for media fetch requests).
 // Should only be called once per component.
 func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client {
diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go
index 3f4e1c917..d93b6ebe0 100644
--- a/setup/config/config_appservice.go
+++ b/setup/config/config_appservice.go
@@ -52,7 +52,9 @@ func (c *AppServiceAPI) Defaults(generate bool) {
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect))
-	checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
+	}
 }
 
 // ApplicationServiceNamespace is the namespace that a specific application
diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go
index 176334dd8..f62a23e1f 100644
--- a/setup/config/config_federationapi.go
+++ b/setup/config/config_federationapi.go
@@ -49,7 +49,9 @@ func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	if !isMonolith {
 		checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
 	}
-	checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
+	}
 }
 
 // The config for setting a proxy to use for server->server requests
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index c1650f077..d609e2460 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -34,6 +34,13 @@ type Global struct {
 	// Defaults to 24 hours.
 	KeyValidityPeriod time.Duration `yaml:"key_validity_period"`
 
+	// Global pool of database connections, which is used only in monolith mode. If a
+	// component does not specify any database options of its own, then this pool of
+	// connections will be used instead. This way we don't have to manage connection
+	// counts on a per-component basis, but can instead do it for the entire monolith.
+	// In a polylith deployment, this will be ignored.
+	DatabaseOptions DatabaseOptions `yaml:"database"`
+
 	// The server name to delegate server-server communications to, with optional port
 	WellKnownServerName string `yaml:"well_known_server_name"`
 
diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go
index 6180ccbc8..9e2d54cdc 100644
--- a/setup/config/config_keyserver.go
+++ b/setup/config/config_keyserver.go
@@ -20,5 +20,7 @@ func (c *KeyServer) Defaults(generate bool) {
 func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect))
-	checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
+	}
 }
diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go
index c85020d2a..273de322a 100644
--- a/setup/config/config_mediaapi.go
+++ b/setup/config/config_mediaapi.go
@@ -58,7 +58,9 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	if !isMonolith {
 		checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
 	}
-	checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
+	}
 
 	checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
 	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes))
diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go
index 66a4c80c9..b992f7152 100644
--- a/setup/config/config_mscs.go
+++ b/setup/config/config_mscs.go
@@ -31,5 +31,7 @@ func (c *MSCs) Enabled(msc string) bool {
 }
 
 func (c *MSCs) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkNotEmpty(configErrs, "mscs.database.connection_string", string(c.Database.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "mscs.database.connection_string", string(c.Database.ConnectionString))
+	}
 }
diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go
index 73abb4f47..8a3227349 100644
--- a/setup/config/config_roomserver.go
+++ b/setup/config/config_roomserver.go
@@ -20,5 +20,7 @@ func (c *RoomServer) Defaults(generate bool) {
 func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect))
-	checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
+	}
 }
diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go
index dc813cb7d..48fd9f506 100644
--- a/setup/config/config_syncapi.go
+++ b/setup/config/config_syncapi.go
@@ -27,5 +27,7 @@ func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	if !isMonolith {
 		checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
 	}
-	checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
+	}
 }
diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go
index 570dc6030..4aa3b57bb 100644
--- a/setup/config/config_userapi.go
+++ b/setup/config/config_userapi.go
@@ -37,6 +37,8 @@ func (c *UserAPI) Defaults(generate bool) {
 func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
-	checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
+	}
 	checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
 }
diff --git a/setup/monolith.go b/setup/monolith.go
index c86ec7b69..a414172ce 100644
--- a/setup/monolith.go
+++ b/setup/monolith.go
@@ -25,11 +25,10 @@ import (
 	keyAPI "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/mediaapi"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -37,7 +36,6 @@ import (
 // all components of Dendrite, for use in monolith mode.
 type Monolith struct {
 	Config    *config.Dendrite
-	AccountDB userdb.Database
 	KeyRing   *gomatrixserverlib.KeyRing
 	Client    *gomatrixserverlib.Client
 	FedClient *gomatrixserverlib.FederationClient
@@ -54,26 +52,28 @@ type Monolith struct {
 }
 
 // AddAllPublicRoutes attaches all public paths to the given router
-func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux, dendriteMux *mux.Router) {
+func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux, dendriteMux *mux.Router) {
 	userDirectoryProvider := m.ExtUserDirectoryProvider
 	if userDirectoryProvider == nil {
 		userDirectoryProvider = m.UserAPI
 	}
 	clientapi.AddPublicRoutes(
-		process, csMux, synapseMux, dendriteMux, &m.Config.ClientAPI,
-		m.FedClient, m.RoomserverAPI,
-		m.AppserviceAPI, transactions.New(),
+		base.ProcessContext, csMux, synapseMux, dendriteMux, &m.Config.ClientAPI,
+		m.FedClient, m.RoomserverAPI, m.AppserviceAPI, transactions.New(),
 		m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI,
 		m.ExtPublicRoomsProvider, &m.Config.MSCs,
 	)
 	federationapi.AddPublicRoutes(
-		process, ssMux, keyMux, wkMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient,
-		m.KeyRing, m.RoomserverAPI, m.FederationAPI,
+		base.ProcessContext, ssMux, keyMux, wkMux, &m.Config.FederationAPI,
+		m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI,
 		m.KeyAPI, &m.Config.MSCs, nil,
 	)
-	mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting, m.UserAPI, m.Client)
+	mediaapi.AddPublicRoutes(
+		base, mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting,
+		m.UserAPI, m.Client,
+	)
 	syncapi.AddPublicRoutes(
-		process, csMux, m.UserAPI, m.RoomserverAPI,
+		base, csMux, m.UserAPI, m.RoomserverAPI,
 		m.KeyAPI, m.FedClient, &m.Config.SyncAPI,
 	)
 }
diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go
index 29c781a88..452b14580 100644
--- a/setup/mscs/msc2836/msc2836.go
+++ b/setup/mscs/msc2836/msc2836.go
@@ -102,7 +102,7 @@ func Enable(
 	base *base.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI,
 	userAPI userapi.UserInternalAPI, keyRing gomatrixserverlib.JSONVerifier,
 ) error {
-	db, err := NewDatabase(&base.Cfg.MSCs.Database)
+	db, err := NewDatabase(base, &base.Cfg.MSCs.Database)
 	if err != nil {
 		return fmt.Errorf("cannot enable MSC2836: %w", err)
 	}
diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go
index 72523916b..827e82f70 100644
--- a/setup/mscs/msc2836/storage.go
+++ b/setup/mscs/msc2836/storage.go
@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -58,19 +59,17 @@ type DB struct {
 }
 
 // NewDatabase loads the database for msc2836
-func NewDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
+func NewDatabase(base *base.BaseDendrite, dbOpts *config.DatabaseOptions) (Database, error) {
 	if dbOpts.ConnectionString.IsPostgres() {
-		return newPostgresDatabase(dbOpts)
+		return newPostgresDatabase(base, dbOpts)
 	}
-	return newSQLiteDatabase(dbOpts)
+	return newSQLiteDatabase(base, dbOpts)
 }
 
-func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
-	d := DB{
-		writer: sqlutil.NewDummyWriter(),
-	}
+func newPostgresDatabase(base *base.BaseDendrite, dbOpts *config.DatabaseOptions) (Database, error) {
+	d := DB{}
 	var err error
-	if d.db, err = sqlutil.Open(dbOpts); err != nil {
+	if d.db, d.writer, err = base.DatabaseConnection(dbOpts, sqlutil.NewDummyWriter()); err != nil {
 		return nil, err
 	}
 	_, err = d.db.Exec(`
@@ -145,12 +144,10 @@ func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
 	return &d, err
 }
 
-func newSQLiteDatabase(dbOpts *config.DatabaseOptions) (Database, error) {
-	d := DB{
-		writer: sqlutil.NewExclusiveWriter(),
-	}
+func newSQLiteDatabase(base *base.BaseDendrite, dbOpts *config.DatabaseOptions) (Database, error) {
+	d := DB{}
 	var err error
-	if d.db, err = sqlutil.Open(dbOpts); err != nil {
+	if d.db, d.writer, err = base.DatabaseConnection(dbOpts, sqlutil.NewExclusiveWriter()); err != nil {
 		return nil, err
 	}
 	_, err = d.db.Exec(`
diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go
index b0382512a..9cfe7c070 100644
--- a/syncapi/storage/postgres/syncserver.go
+++ b/syncapi/storage/postgres/syncserver.go
@@ -21,6 +21,7 @@ import (
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
@@ -35,13 +36,12 @@ type SyncServerDatasource struct {
 }
 
 // NewDatabase creates a new sync server database
-func NewDatabase(dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) {
 	var d SyncServerDatasource
 	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
+	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
 		return nil, err
 	}
-	d.writer = sqlutil.NewDummyWriter()
 	accountData, err := NewPostgresAccountDataTable(d.db)
 	if err != nil {
 		return nil, err
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index dfc289482..e08a0ba82 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -19,6 +19,7 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
 	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas"
@@ -35,13 +36,12 @@ type SyncServerDatasource struct {
 
 // NewDatabase creates a new sync server database
 // nolint: gocyclo
-func NewDatabase(dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) {
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) {
 	var d SyncServerDatasource
 	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
+	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
 		return nil, err
 	}
-	d.writer = sqlutil.NewExclusiveWriter()
 	if err = d.prepare(dbProperties); err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go
index 7f9c28e9d..5b20c6cc2 100644
--- a/syncapi/storage/storage.go
+++ b/syncapi/storage/storage.go
@@ -20,18 +20,19 @@ package storage
 import (
 	"fmt"
 
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage/postgres"
 	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
 )
 
 // NewSyncServerDatasource opens a database connection.
-func NewSyncServerDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(dbProperties)
+		return postgres.NewDatabase(base, dbProperties)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index 15bb769a2..1150c2f3d 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -17,7 +17,7 @@ var ctx = context.Background()
 
 func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
-	db, err := storage.NewSyncServerDatasource(&config.DatabaseOptions{
+	db, err := storage.NewSyncServerDatasource(nil, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	})
 	if err != nil {
diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go
index f7fef962b..c15444743 100644
--- a/syncapi/storage/storage_wasm.go
+++ b/syncapi/storage/storage_wasm.go
@@ -17,15 +17,16 @@ package storage
 import (
 	"fmt"
 
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
 )
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewSyncServerDatasource(dbProperties *config.DatabaseOptions) (Database, error) {
+func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties)
+		return sqlite3.NewDatabase(base, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go
index a143e5ecd..8bbf879d4 100644
--- a/syncapi/storage/tables/output_room_events_test.go
+++ b/syncapi/storage/tables/output_room_events_test.go
@@ -21,7 +21,7 @@ func newOutputRoomEventsTable(t *testing.T, dbType test.DBType) (tables.Events,
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
 	db, err := sqlutil.Open(&config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
-	})
+	}, sqlutil.NewExclusiveWriter())
 	if err != nil {
 		t.Fatalf("failed to open db: %s", err)
 	}
diff --git a/syncapi/storage/tables/topology_test.go b/syncapi/storage/tables/topology_test.go
index b6ece0b0d..2334aae2e 100644
--- a/syncapi/storage/tables/topology_test.go
+++ b/syncapi/storage/tables/topology_test.go
@@ -20,7 +20,7 @@ func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.D
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
 	db, err := sqlutil.Open(&config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
-	})
+	}, sqlutil.NewExclusiveWriter())
 	if err != nil {
 		t.Fatalf("failed to open db: %s", err)
 	}
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index b2d333f74..a2b8859cd 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -23,9 +23,9 @@ import (
 
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -41,7 +41,7 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers for the SyncAPI
 // component.
 func AddPublicRoutes(
-	process *process.ProcessContext,
+	base *base.BaseDendrite,
 	router *mux.Router,
 	userAPI userapi.UserInternalAPI,
 	rsAPI api.RoomserverInternalAPI,
@@ -49,9 +49,9 @@ func AddPublicRoutes(
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.SyncAPI,
 ) {
-	js, natsClient := jetstream.Prepare(process, &cfg.Matrix.JetStream)
+	js, natsClient := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
-	syncDB, err := storage.NewSyncServerDatasource(&cfg.Database)
+	syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to sync db")
 	}
@@ -86,7 +86,7 @@ func AddPublicRoutes(
 	}
 
 	keyChangeConsumer := consumers.NewOutputKeyChangeEventConsumer(
-		process, cfg, cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
+		base.ProcessContext, cfg, cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
 		js, keyAPI, rsAPI, syncDB, notifier,
 		streams.DeviceListStreamProvider,
 	)
@@ -95,7 +95,7 @@ func AddPublicRoutes(
 	}
 
 	roomConsumer := consumers.NewOutputRoomEventConsumer(
-		process, cfg, js, syncDB, notifier, streams.PDUStreamProvider,
+		base.ProcessContext, cfg, js, syncDB, notifier, streams.PDUStreamProvider,
 		streams.InviteStreamProvider, rsAPI, userAPIStreamEventProducer,
 	)
 	if err = roomConsumer.Start(); err != nil {
@@ -103,7 +103,7 @@ func AddPublicRoutes(
 	}
 
 	clientConsumer := consumers.NewOutputClientDataConsumer(
-		process, cfg, js, syncDB, notifier, streams.AccountDataStreamProvider,
+		base.ProcessContext, cfg, js, syncDB, notifier, streams.AccountDataStreamProvider,
 		userAPIReadUpdateProducer,
 	)
 	if err = clientConsumer.Start(); err != nil {
@@ -111,28 +111,28 @@ func AddPublicRoutes(
 	}
 
 	notificationConsumer := consumers.NewOutputNotificationDataConsumer(
-		process, cfg, js, syncDB, notifier, streams.NotificationDataStreamProvider,
+		base.ProcessContext, cfg, js, syncDB, notifier, streams.NotificationDataStreamProvider,
 	)
 	if err = notificationConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start notification data consumer")
 	}
 
 	typingConsumer := consumers.NewOutputTypingEventConsumer(
-		process, cfg, js, eduCache, notifier, streams.TypingStreamProvider,
+		base.ProcessContext, cfg, js, eduCache, notifier, streams.TypingStreamProvider,
 	)
 	if err = typingConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start typing consumer")
 	}
 
 	sendToDeviceConsumer := consumers.NewOutputSendToDeviceEventConsumer(
-		process, cfg, js, syncDB, notifier, streams.SendToDeviceStreamProvider,
+		base.ProcessContext, cfg, js, syncDB, notifier, streams.SendToDeviceStreamProvider,
 	)
 	if err = sendToDeviceConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start send-to-device consumer")
 	}
 
 	receiptConsumer := consumers.NewOutputReceiptEventConsumer(
-		process, cfg, js, syncDB, notifier, streams.ReceiptStreamProvider,
+		base.ProcessContext, cfg, js, syncDB, notifier, streams.ReceiptStreamProvider,
 		userAPIReadUpdateProducer,
 	)
 	if err = receiptConsumer.Start(); err != nil {
@@ -140,7 +140,7 @@ func AddPublicRoutes(
 	}
 
 	presenceConsumer := consumers.NewPresenceConsumer(
-		process, cfg, js, natsClient, syncDB,
+		base.ProcessContext, cfg, js, natsClient, syncDB,
 		notifier, streams.PresenceStreamProvider,
 		userAPI,
 	)
diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go
index b2a517605..74100a728 100644
--- a/userapi/storage/postgres/storage.go
+++ b/userapi/storage/postgres/storage.go
@@ -21,6 +21,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
@@ -30,8 +31,8 @@ import (
 )
 
 // NewDatabase creates a new accounts and profiles database
-func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
-	db, err := sqlutil.Open(dbProperties)
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter())
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +108,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 		Notifications:         notificationsTable,
 		ServerName:            serverName,
 		DB:                    db,
-		Writer:                sqlutil.NewDummyWriter(),
+		Writer:                writer,
 		LoginTokenLifetime:    loginTokenLifetime,
 		BcryptCost:            bcryptCost,
 		OpenIDTokenLifetimeMS: openIDTokenLifetimeMS,
diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go
index 03c013f00..6858d3d15 100644
--- a/userapi/storage/sqlite3/storage.go
+++ b/userapi/storage/sqlite3/storage.go
@@ -21,6 +21,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
@@ -31,8 +32,8 @@ import (
 )
 
 // NewDatabase creates a new accounts and profiles database
-func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
-	db, err := sqlutil.Open(dbProperties)
+func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
+	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter())
 	if err != nil {
 		return nil, err
 	}
@@ -108,7 +109,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 		Notifications:         notificationsTable,
 		ServerName:            serverName,
 		DB:                    db,
-		Writer:                sqlutil.NewExclusiveWriter(),
+		Writer:                writer,
 		LoginTokenLifetime:    loginTokenLifetime,
 		BcryptCost:            bcryptCost,
 		OpenIDTokenLifetimeMS: openIDTokenLifetimeMS,
diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go
index faf1ce75c..42221e752 100644
--- a/userapi/storage/storage.go
+++ b/userapi/storage/storage.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/postgres"
 	"github.com/matrix-org/dendrite/userapi/storage/sqlite3"
@@ -30,12 +31,12 @@ import (
 
 // NewUserAPIDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets postgres connection parameters
-func NewUserAPIDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) {
+func NewUserAPIDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
+		return sqlite3.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
+		return postgres.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index ef25b8000..79d5a8dae 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -29,7 +29,7 @@ var (
 
 func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
-	db, err := storage.NewUserAPIDatabase(&config.DatabaseOptions{
+	db, err := storage.NewUserAPIDatabase(nil, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	}, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server")
 	if err != nil {
diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go
index a8e6f031c..5d5d292e6 100644
--- a/userapi/storage/storage_wasm.go
+++ b/userapi/storage/storage_wasm.go
@@ -18,12 +18,14 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func NewUserAPIDatabase(
+	base *base.BaseDendrite,
 	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 	bcryptCost int,
@@ -33,7 +35,7 @@ func NewUserAPIDatabase(
 ) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
+		return sqlite3.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/userapi/userapi.go b/userapi/userapi.go
index e91ce3a7a..9174119e1 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -42,12 +42,25 @@ func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI) {
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite, db storage.Database, cfg *config.UserAPI,
+	base *base.BaseDendrite, cfg *config.UserAPI,
 	appServices []config.ApplicationService, keyAPI keyapi.KeyInternalAPI,
 	rsAPI rsapi.RoomserverInternalAPI, pgClient pushgateway.Client,
 ) api.UserInternalAPI {
 	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
+	db, err := storage.NewUserAPIDatabase(
+		base,
+		&cfg.AccountDatabase,
+		cfg.Matrix.ServerName,
+		cfg.BCryptCost,
+		cfg.OpenIDTokenLifetimeMS,
+		api.DefaultLoginTokenLifetime,
+		cfg.Matrix.ServerNotices.LocalPart,
+	)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to accounts db")
+	}
+
 	syncProducer := producers.NewSyncAPI(
 		db, js,
 		// TODO: user API should handle syncs for account data. Right now,
diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index 076b4f3c6..64e23909a 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -52,7 +52,7 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts) (api.UserInternalAPI, s
 		MaxOpenConnections: 1,
 		MaxIdleConnections: 1,
 	}
-	accountDB, err := storage.NewUserAPIDatabase(dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
+	accountDB, err := storage.NewUserAPIDatabase(nil, dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
 	if err != nil {
 		t.Fatalf("failed to create account DB: %s", err)
 	}

From dd061a172e97005a4a7a4c37db6caf3f77c10d51 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 3 May 2022 17:17:02 +0100
Subject: [PATCH 065/391] Tidy up `AddPublicRoutes` (#2412)

* Simplify federation API `AddPublicRoutes`

* Simplify client API `AddPublicRoutes`

* Simplify media API `AddPublicRoutes`

* Simplify sync API `AddPublicRoutes`

* Simplify `AddAllPublicRoutes`
---
 build/gobind-pinecone/monolith.go             | 11 +----------
 build/gobind-yggdrasil/monolith.go            | 11 +----------
 clientapi/clientapi.go                        | 19 ++++++++-----------
 cmd/dendrite-demo-pinecone/main.go            | 11 +----------
 cmd/dendrite-demo-yggdrasil/main.go           | 11 +----------
 cmd/dendrite-monolith-server/main.go          | 11 +----------
 .../personalities/clientapi.go                |  6 ++----
 .../personalities/federationapi.go            |  7 +++----
 .../personalities/mediaapi.go                 |  3 +--
 .../personalities/syncapi.go                  |  4 ++--
 cmd/dendritejs-pinecone/main.go               | 11 +----------
 federationapi/federationapi.go                | 19 +++++++++----------
 federationapi/federationapi_test.go           |  2 +-
 mediaapi/mediaapi.go                          | 10 ++++------
 setup/monolith.go                             | 19 +++++++------------
 syncapi/syncapi.go                            | 11 ++++++-----
 16 files changed, 49 insertions(+), 117 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 8cf663d00..310ac7dda 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -306,16 +306,7 @@ func (m *DendriteMonolith) Start() {
 		ExtPublicRoomsProvider:   roomProvider,
 		ExtUserDirectoryProvider: userProvider,
 	}
-	monolith.AddAllPublicRoutes(
-		base,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
-	)
+	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 2c7d4e91b..991bc462f 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -144,16 +144,7 @@ func (m *DendriteMonolith) Start() {
 			ygg, fsAPI, federation,
 		),
 	}
-	monolith.AddAllPublicRoutes(
-		base,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
-	)
+	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter()
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index ad277056c..0d16e4c14 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -15,7 +15,6 @@
 package clientapi
 
 import (
-	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/producers"
@@ -24,20 +23,15 @@ import (
 	"github.com/matrix-org/dendrite/internal/transactions"
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
-	process *process.ProcessContext,
-	router *mux.Router,
-	synapseAdminRouter *mux.Router,
-	dendriteAdminRouter *mux.Router,
-	cfg *config.ClientAPI,
+	base *base.BaseDendrite,
 	federation *gomatrixserverlib.FederationClient,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
@@ -47,9 +41,10 @@ func AddPublicRoutes(
 	userDirectoryProvider userapi.UserDirectoryProvider,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
-	mscCfg *config.MSCs,
 ) {
-	js, natsClient := jetstream.Prepare(process, &cfg.Matrix.JetStream)
+	cfg := &base.Cfg.ClientAPI
+	mscCfg := &base.Cfg.MSCs
+	js, natsClient := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
@@ -63,7 +58,9 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		router, synapseAdminRouter, dendriteAdminRouter,
+		base.PublicClientAPIMux,
+		base.SynapseAdminMux,
+		base.DendriteAdminMux,
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI,
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 33487e64b..703436051 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -185,16 +185,7 @@ func main() {
 		ExtPublicRoomsProvider:   roomProvider,
 		ExtUserDirectoryProvider: userProvider,
 	}
-	monolith.AddAllPublicRoutes(
-		base,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
-	)
+	monolith.AddAllPublicRoutes(base)
 
 	wsUpgrader := websocket.Upgrader{
 		CheckOrigin: func(_ *http.Request) bool {
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index df9ba5121..619720d6c 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -142,16 +142,7 @@ func main() {
 			ygg, fsAPI, federation,
 		),
 	}
-	monolith.AddAllPublicRoutes(
-		base,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
-	)
+	monolith.AddAllPublicRoutes(base)
 	if err := mscs.Enable(base, &monolith); err != nil {
 		logrus.WithError(err).Fatalf("Failed to enable MSCs")
 	}
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index 4c7c4297f..2fa4675a4 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -143,16 +143,7 @@ func main() {
 		UserAPI:       userAPI,
 		KeyAPI:        keyAPI,
 	}
-	monolith.AddAllPublicRoutes(
-		base,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
-	)
+	monolith.AddAllPublicRoutes(base)
 
 	if len(base.Cfg.MSCs.MSCs) > 0 {
 		if err := mscs.Enable(base, &monolith); err != nil {
diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go
index 7ed2075aa..a5d69d07c 100644
--- a/cmd/dendrite-polylith-multi/personalities/clientapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go
@@ -31,11 +31,9 @@ func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	keyAPI := base.KeyServerHTTPClient()
 
 	clientapi.AddPublicRoutes(
-		base.ProcessContext, base.PublicClientAPIMux,
-		base.SynapseAdminMux, base.DendriteAdminMux,
-		&base.Cfg.ClientAPI, federation, rsAPI, asQuery,
+		base, federation, rsAPI, asQuery,
 		transactions.New(), fsAPI, userAPI, userAPI,
-		keyAPI, nil, &cfg.MSCs,
+		keyAPI, nil,
 	)
 
 	base.SetupAndServeHTTP(
diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go
index b82577ce3..6377ce9e3 100644
--- a/cmd/dendrite-polylith-multi/personalities/federationapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go
@@ -29,10 +29,9 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	keyRing := fsAPI.KeyRing()
 
 	federationapi.AddPublicRoutes(
-		base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux,
-		&base.Cfg.FederationAPI, userAPI, federation, keyRing,
-		rsAPI, fsAPI, keyAPI,
-		&base.Cfg.MSCs, nil,
+		base,
+		userAPI, federation, keyRing,
+		rsAPI, fsAPI, keyAPI, nil,
 	)
 
 	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
diff --git a/cmd/dendrite-polylith-multi/personalities/mediaapi.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go
index 8c0bfa195..69d5fd5a8 100644
--- a/cmd/dendrite-polylith-multi/personalities/mediaapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go
@@ -25,8 +25,7 @@ func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	client := base.CreateClient()
 
 	mediaapi.AddPublicRoutes(
-		base, base.PublicMediaAPIMux, &base.Cfg.MediaAPI, &base.Cfg.ClientAPI.RateLimiting,
-		userAPI, client,
+		base, userAPI, client,
 	)
 
 	base.SetupAndServeHTTP(
diff --git a/cmd/dendrite-polylith-multi/personalities/syncapi.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go
index f9f1c5a09..2245b9b54 100644
--- a/cmd/dendrite-polylith-multi/personalities/syncapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go
@@ -28,9 +28,9 @@ func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 
 	syncapi.AddPublicRoutes(
 		base,
-		base.PublicClientAPIMux, userAPI, rsAPI,
+		userAPI, rsAPI,
 		base.KeyServerHTTPClient(),
-		federation, &cfg.SyncAPI,
+		federation,
 	)
 
 	base.SetupAndServeHTTP(
diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
index ead381368..e070173aa 100644
--- a/cmd/dendritejs-pinecone/main.go
+++ b/cmd/dendritejs-pinecone/main.go
@@ -212,16 +212,7 @@ func startup() {
 		//ServerKeyAPI:        serverKeyAPI,
 		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
 	}
-	monolith.AddAllPublicRoutes(
-		base,
-		base.PublicClientAPIMux,
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		base.PublicMediaAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
-	)
+	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index 1848a242e..c627aab5c 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -29,9 +29,7 @@ import (
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 
@@ -47,20 +45,18 @@ func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) {
 
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
 func AddPublicRoutes(
-	process *process.ProcessContext,
-	fedRouter, keyRouter, wellKnownRouter *mux.Router,
-	cfg *config.FederationAPI,
+	base *base.BaseDendrite,
 	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	federationAPI federationAPI.FederationInternalAPI,
 	keyAPI keyserverAPI.KeyInternalAPI,
-	mscCfg *config.MSCs,
 	servers federationAPI.ServersInRoomProvider,
 ) {
-
-	js, _ := jetstream.Prepare(process, &cfg.Matrix.JetStream)
+	cfg := &base.Cfg.FederationAPI
+	mscCfg := &base.Cfg.MSCs
+	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 	producer := &producers.SyncAPIProducer{
 		JetStream:              js,
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
@@ -72,8 +68,11 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		fedRouter, keyRouter, wellKnownRouter, cfg, rsAPI,
-		federationAPI, keyRing,
+		base.PublicFederationAPIMux,
+		base.PublicKeyAPIMux,
+		base.PublicWellKnownAPIMux,
+		cfg,
+		rsAPI, federationAPI, keyRing,
 		federation, userAPI, keyAPI, mscCfg,
 		servers, producer,
 	)
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index 833359c11..687241646 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -30,7 +30,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	fsAPI := base.FederationAPIHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base.ProcessContext, base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, &cfg.MSCs, nil)
+	federationapi.AddPublicRoutes(base, nil, nil, keyRing, nil, fsAPI, nil, nil)
 	baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go
index f2fa14384..5976957ca 100644
--- a/mediaapi/mediaapi.go
+++ b/mediaapi/mediaapi.go
@@ -15,11 +15,9 @@
 package mediaapi
 
 import (
-	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/mediaapi/routing"
 	"github.com/matrix-org/dendrite/mediaapi/storage"
 	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -28,18 +26,18 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.
 func AddPublicRoutes(
 	base *base.BaseDendrite,
-	router *mux.Router,
-	cfg *config.MediaAPI,
-	rateLimit *config.RateLimiting,
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
 ) {
+	cfg := &base.Cfg.MediaAPI
+	rateCfg := &base.Cfg.ClientAPI.RateLimiting
+
 	mediaDB, err := storage.NewMediaAPIDatasource(base, &cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to media db")
 	}
 
 	routing.Setup(
-		router, cfg, rateLimit, mediaDB, userAPI, client,
+		base.PublicMediaAPIMux, cfg, rateCfg, mediaDB, userAPI, client,
 	)
 }
diff --git a/setup/monolith.go b/setup/monolith.go
index a414172ce..23bd2fb52 100644
--- a/setup/monolith.go
+++ b/setup/monolith.go
@@ -15,7 +15,6 @@
 package setup
 
 import (
-	"github.com/gorilla/mux"
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi"
 	"github.com/matrix-org/dendrite/clientapi/api"
@@ -52,28 +51,24 @@ type Monolith struct {
 }
 
 // AddAllPublicRoutes attaches all public paths to the given router
-func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux, dendriteMux *mux.Router) {
+func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite) {
 	userDirectoryProvider := m.ExtUserDirectoryProvider
 	if userDirectoryProvider == nil {
 		userDirectoryProvider = m.UserAPI
 	}
 	clientapi.AddPublicRoutes(
-		base.ProcessContext, csMux, synapseMux, dendriteMux, &m.Config.ClientAPI,
-		m.FedClient, m.RoomserverAPI, m.AppserviceAPI, transactions.New(),
+		base, m.FedClient, m.RoomserverAPI, m.AppserviceAPI, transactions.New(),
 		m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI,
-		m.ExtPublicRoomsProvider, &m.Config.MSCs,
+		m.ExtPublicRoomsProvider,
 	)
 	federationapi.AddPublicRoutes(
-		base.ProcessContext, ssMux, keyMux, wkMux, &m.Config.FederationAPI,
-		m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI,
-		m.KeyAPI, &m.Config.MSCs, nil,
+		base, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI,
+		m.KeyAPI, nil,
 	)
 	mediaapi.AddPublicRoutes(
-		base, mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting,
-		m.UserAPI, m.Client,
+		base, m.UserAPI, m.Client,
 	)
 	syncapi.AddPublicRoutes(
-		base, csMux, m.UserAPI, m.RoomserverAPI,
-		m.KeyAPI, m.FedClient, &m.Config.SyncAPI,
+		base, m.UserAPI, m.RoomserverAPI, m.KeyAPI, m.FedClient,
 	)
 }
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index a2b8859cd..d8becb6ed 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -17,14 +17,12 @@ package syncapi
 import (
 	"context"
 
-	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/sirupsen/logrus"
 
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -42,13 +40,13 @@ import (
 // component.
 func AddPublicRoutes(
 	base *base.BaseDendrite,
-	router *mux.Router,
 	userAPI userapi.UserInternalAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keyAPI keyapi.KeyInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
-	cfg *config.SyncAPI,
 ) {
+	cfg := &base.Cfg.SyncAPI
+
 	js, natsClient := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database)
@@ -148,5 +146,8 @@ func AddPublicRoutes(
 		logrus.WithError(err).Panicf("failed to start presence consumer")
 	}
 
-	routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg, lazyLoadCache)
+	routing.Setup(
+		base.PublicClientAPIMux, requestPool, syncDB, userAPI,
+		federation, rsAPI, cfg, lazyLoadCache,
+	)
 }

From e01d1e1f5b39677e8817bda7cc946b3c8fcf9a4a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 3 May 2022 17:38:54 +0100
Subject: [PATCH 066/391] Skip tests that require a database if we can't
 connect to one (#2413)

* Skip tests that require a database if we can't connect to one

* Add `DENDRITE_SKIP_DB_TESTS` environment variable to bring @kegsay joy

* Call it `DENDRITE_TEST_SKIP_NODB` intead
---
 test/db.go | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/test/db.go b/test/db.go
index 6412feaa6..fecae5d48 100644
--- a/test/db.go
+++ b/test/db.go
@@ -33,10 +33,19 @@ var DBTypeSQLite DBType = 1
 var DBTypePostgres DBType = 2
 
 var Quiet = false
+var Required = os.Getenv("DENDRITE_TEST_SKIP_NODB") == ""
 
-func createLocalDB(dbName string) {
+func fatalError(t *testing.T, format string, args ...interface{}) {
+	if Required {
+		t.Fatalf(format, args...)
+	} else {
+		t.Skipf(format, args...)
+	}
+}
+
+func createLocalDB(t *testing.T, dbName string) {
 	if !Quiet {
-		fmt.Println("Note: tests require a postgres install accessible to the current user")
+		t.Log("Note: tests require a postgres install accessible to the current user")
 	}
 	createDB := exec.Command("createdb", dbName)
 	if !Quiet {
@@ -44,15 +53,15 @@ func createLocalDB(dbName string) {
 		createDB.Stderr = os.Stderr
 	}
 	err := createDB.Run()
-	if err != nil && !Quiet {
-		fmt.Println("createLocalDB returned error:", err)
+	if err != nil {
+		fatalError(t, "createLocalDB returned error: %s", err)
 	}
 }
 
 func createRemoteDB(t *testing.T, dbName, user, connStr string) {
 	db, err := sql.Open("postgres", connStr+" dbname=postgres")
 	if err != nil {
-		t.Fatalf("failed to open postgres conn with connstr=%s : %s", connStr, err)
+		fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err)
 	}
 	_, err = db.Exec(fmt.Sprintf(`CREATE DATABASE %s;`, dbName))
 	if err != nil {
@@ -133,7 +142,7 @@ func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, clo
 	hash := sha256.Sum256([]byte(wd))
 	dbName := fmt.Sprintf("dendrite_test_%s", hex.EncodeToString(hash[:16]))
 	if postgresDB == "" { // local server, use createdb
-		createLocalDB(dbName)
+		createLocalDB(t, dbName)
 	} else { // remote server, shell into the postgres user and CREATE DATABASE
 		createRemoteDB(t, dbName, user, connStr)
 	}

From b0a9e85c4a02f39880682d9d682f9cc7af13a93c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 3 May 2022 17:40:56 +0100
Subject: [PATCH 067/391] Fix bug in database global setup

---
 setup/base/base.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup/base/base.go b/setup/base/base.go
index 9b227b70b..3641ad780 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -201,7 +201,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		if cfg.Global.DatabaseOptions.ConnectionString.IsSQLite() {
 			logrus.Panic("Using a global database connection pool is not supported with SQLite databases")
 		}
-		if db, err = sqlutil.Open(&cfg.Global.DatabaseOptions, sqlutil.NewDummyWriter()); err != nil {
+		writer = sqlutil.NewDummyWriter()
+		if db, err = sqlutil.Open(&cfg.Global.DatabaseOptions, writer); err != nil {
 			logrus.WithError(err).Panic("Failed to set up global database connections")
 		}
 		logrus.Debug("Using global database connection pool")

From 3c940c428d529476b6fa2cbf1ba28d53ec011584 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 4 May 2022 19:04:28 +0200
Subject: [PATCH 068/391] Add opt-in anonymous stats reporting (#2249)

* Initial phone home stats queries

* Add userAgent to UpdateDeviceLastSeen
Add new Table for tracking daily user vists

* Add user_daily_visits table

* Fix queries

* userapi stats tables & queries

* userapi interface and internal api

* sycnapi stats queries

* testing phone home stats

* Add complete config to syncapi

* add missing files

* Fix queries

* Send empty request

* Add version & monolith stats

* Add configuration for phone home stats

* Move WASM to its own file, add config and comments

* Add tracing methods

* Add total rooms

* Add more fields, actually send data somewhere

* Move stats to the userapi

* Move phone home stats to util package

* Cleanup

* Linter & parts of GH comments

* More GH comments changes
- Move comments to SQL statements
- Shrink interface, add struct for stats
- No fatal errors, use defaults

* Be more explicit when querying

* Fix wrong calculation & wrong query params
Add tests

* Add Windows stats

* ADd build constraint

* Use new testing structure
Fix issues with getting values when using SQLite
Fix wrong AddDate value
Export UpdateUserDailyVisits

* Fix query params

* Fix test

* Add comment about countR30UsersSQL and countR30UsersV2SQL; fix test

* Update config

* Also update example config file

* Use OS level proxy, update logging

Co-authored-by: kegsay <kegan@matrix.org>
---
 cmd/dendrite-polylith-multi/main.go        |   2 +-
 dendrite-config.yaml                       |   9 +
 setup/config/config.go                     |   3 +
 setup/config/config_global.go              |  25 ++
 syncapi/sync/requestpool.go                |   1 +
 userapi/api/api.go                         |   1 +
 userapi/internal/api.go                    |   2 +-
 userapi/storage/interface.go               |   8 +-
 userapi/storage/postgres/devices_table.go  |   6 +-
 userapi/storage/postgres/stats_table.go    | 437 ++++++++++++++++++++
 userapi/storage/postgres/storage.go        |   5 +
 userapi/storage/shared/storage.go          |  13 +-
 userapi/storage/sqlite3/devices_table.go   |   6 +-
 userapi/storage/sqlite3/stats_table.go     | 452 +++++++++++++++++++++
 userapi/storage/sqlite3/storage.go         |   5 +
 userapi/storage/storage_test.go            |   2 +-
 userapi/storage/tables/interface.go        |   9 +-
 userapi/storage/tables/stats_table_test.go | 319 +++++++++++++++
 userapi/types/statistics.go                |  30 ++
 userapi/userapi.go                         |   5 +
 userapi/util/phonehomestats.go             | 160 ++++++++
 userapi/util/stats.go                      |  47 +++
 userapi/util/stats_wasm.go                 |  20 +
 userapi/util/stats_windows.go              |  29 ++
 24 files changed, 1582 insertions(+), 14 deletions(-)
 create mode 100644 userapi/storage/postgres/stats_table.go
 create mode 100644 userapi/storage/sqlite3/stats_table.go
 create mode 100644 userapi/storage/tables/stats_table_test.go
 create mode 100644 userapi/types/statistics.go
 create mode 100644 userapi/util/phonehomestats.go
 create mode 100644 userapi/util/stats.go
 create mode 100644 userapi/util/stats_wasm.go
 create mode 100644 userapi/util/stats_windows.go

diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go
index 6226cc328..4fccaa922 100644
--- a/cmd/dendrite-polylith-multi/main.go
+++ b/cmd/dendrite-polylith-multi/main.go
@@ -31,7 +31,7 @@ import (
 type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)
 
 func main() {
-	cfg := setup.ParseFlags(true)
+	cfg := setup.ParseFlags(false)
 
 	component := ""
 	if flag.NFlag() > 0 {
diff --git a/dendrite-config.yaml b/dendrite-config.yaml
index 1647af15d..7709e0c87 100644
--- a/dendrite-config.yaml
+++ b/dendrite-config.yaml
@@ -85,6 +85,15 @@ global:
     # Whether outbound presence events are allowed, e.g. sending presence events to other servers
     enable_outbound: false
 
+  # Configures opt-in anonymous stats reporting.
+  report_stats:
+    # Whether this instance sends anonymous usage stats
+    enabled: false
+
+    # The endpoint to report the anonymized homeserver usage statistics to.
+    # Defaults to https://matrix.org/report-usage-stats/push
+    endpoint: https://matrix.org/report-usage-stats/push
+
   # Server notices allows server admins to send messages to all users.
   server_notices:
     enabled: false
diff --git a/setup/config/config.go b/setup/config/config.go
index e03518e24..9b9000a62 100644
--- a/setup/config/config.go
+++ b/setup/config/config.go
@@ -78,6 +78,8 @@ type Dendrite struct {
 
 	// Any information derived from the configuration options for later use.
 	Derived Derived `yaml:"-"`
+
+	IsMonolith bool `yaml:"-"`
 }
 
 // TODO: Kill Derived
@@ -210,6 +212,7 @@ func loadConfig(
 ) (*Dendrite, error) {
 	var c Dendrite
 	c.Defaults(false)
+	c.IsMonolith = monolithic
 
 	var err error
 	if err = yaml.Unmarshal(configData, &c); err != nil {
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index d609e2460..9d4c1485e 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -70,6 +70,9 @@ type Global struct {
 
 	// ServerNotices configuration used for sending server notices
 	ServerNotices ServerNotices `yaml:"server_notices"`
+
+	// ReportStats configures opt-in anonymous stats reporting.
+	ReportStats ReportStats `yaml:"report_stats"`
 }
 
 func (c *Global) Defaults(generate bool) {
@@ -86,6 +89,7 @@ func (c *Global) Defaults(generate bool) {
 	c.DNSCache.Defaults()
 	c.Sentry.Defaults()
 	c.ServerNotices.Defaults(generate)
+	c.ReportStats.Defaults()
 }
 
 func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
@@ -97,6 +101,7 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	c.Sentry.Verify(configErrs, isMonolith)
 	c.DNSCache.Verify(configErrs, isMonolith)
 	c.ServerNotices.Verify(configErrs, isMonolith)
+	c.ReportStats.Verify(configErrs, isMonolith)
 }
 
 type OldVerifyKeys struct {
@@ -163,6 +168,26 @@ func (c *ServerNotices) Defaults(generate bool) {
 
 func (c *ServerNotices) Verify(errors *ConfigErrors, isMonolith bool) {}
 
+// ReportStats configures opt-in anonymous stats reporting.
+type ReportStats struct {
+	// Enabled configures anonymous usage stats of the server
+	Enabled bool `yaml:"enabled"`
+
+	// Endpoint the endpoint to report stats to
+	Endpoint string `yaml:"endpoint"`
+}
+
+func (c *ReportStats) Defaults() {
+	c.Enabled = false
+	c.Endpoint = "https://matrix.org/report-usage-stats/push"
+}
+
+func (c *ReportStats) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	if c.Enabled {
+		checkNotEmpty(configErrs, "global.report_stats.endpoint", c.Endpoint)
+	}
+}
+
 // The configuration to use for Sentry error reporting
 type Sentry struct {
 	Enabled bool `yaml:"enabled"`
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 76d550a65..f8e502d2c 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -182,6 +182,7 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device)
 		UserID:     device.UserID,
 		DeviceID:   device.ID,
 		RemoteAddr: remoteAddr,
+		UserAgent:  req.UserAgent(),
 	}
 	lsres := &userapi.PerformLastSeenUpdateResponse{}
 	go rp.userAPI.PerformLastSeenUpdate(req.Context(), lsreq, lsres) // nolint:errcheck
diff --git a/userapi/api/api.go b/userapi/api/api.go
index 6aa6a6842..6ab68fa08 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -320,6 +320,7 @@ type PerformLastSeenUpdateRequest struct {
 	UserID     string
 	DeviceID   string
 	RemoteAddr string
+	UserAgent  string
 }
 
 // PerformLastSeenUpdateResponse is the response for PerformLastSeenUpdate.
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index be58e2d8d..394bfa224 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -210,7 +210,7 @@ func (a *UserInternalAPI) PerformLastSeenUpdate(
 	if err != nil {
 		return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 	}
-	if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr); err != nil {
+	if err := a.DB.UpdateDeviceLastSeen(ctx, localpart, req.DeviceID, req.RemoteAddr, req.UserAgent); err != nil {
 		return fmt.Errorf("a.DeviceDB.UpdateDeviceLastSeen: %w", err)
 	}
 	return nil
diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go
index a4562cf19..f7cd1810a 100644
--- a/userapi/storage/interface.go
+++ b/userapi/storage/interface.go
@@ -22,6 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
+	"github.com/matrix-org/dendrite/userapi/types"
 )
 
 type Profile interface {
@@ -67,7 +68,7 @@ type Device interface {
 	// Returns the device on success.
 	CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error)
 	UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error
-	UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error
+	UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error
 	RemoveDevices(ctx context.Context, localpart string, devices []string) error
 	// RemoveAllDevices deleted all devices for this user. Returns the devices deleted.
 	RemoveAllDevices(ctx context.Context, localpart, exceptDeviceID string) (devices []api.Device, err error)
@@ -135,9 +136,14 @@ type Database interface {
 	OpenID
 	Profile
 	Pusher
+	Statistics
 	ThreePID
 }
 
+type Statistics interface {
+	UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error)
+}
+
 // Err3PIDInUse is the error returned when trying to save an association involving
 // a third-party identifier which is already associated to a local user.
 var Err3PIDInUse = errors.New("this third-party identifier is already in use")
diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go
index 6c777982f..ccb776672 100644
--- a/userapi/storage/postgres/devices_table.go
+++ b/userapi/storage/postgres/devices_table.go
@@ -96,7 +96,7 @@ const selectDevicesByIDSQL = "" +
 	"SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id = ANY($1) ORDER BY last_seen_ts DESC"
 
 const updateDeviceLastSeen = "" +
-	"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
+	"UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5"
 
 type devicesStatements struct {
 	insertDeviceStmt             *sql.Stmt
@@ -304,9 +304,9 @@ func (s *devicesStatements) SelectDevicesByLocalpart(
 	return devices, rows.Err()
 }
 
-func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
+func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error {
 	lastSeenTs := time.Now().UnixNano() / 1000000
 	stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
-	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
+	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID)
 	return err
 }
diff --git a/userapi/storage/postgres/stats_table.go b/userapi/storage/postgres/stats_table.go
new file mode 100644
index 000000000..f71900015
--- /dev/null
+++ b/userapi/storage/postgres/stats_table.go
@@ -0,0 +1,437 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/tables"
+	"github.com/matrix-org/dendrite/userapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+const userDailyVisitsSchema = `
+CREATE TABLE IF NOT EXISTS user_daily_visits (
+    localpart TEXT NOT NULL,
+	device_id TEXT NOT NULL,
+	timestamp BIGINT NOT NULL,
+	user_agent TEXT
+);
+
+-- Device IDs and timestamp must be unique for a given user per day
+CREATE UNIQUE INDEX IF NOT EXISTS localpart_device_timestamp_idx ON user_daily_visits(localpart, device_id, timestamp);
+CREATE INDEX IF NOT EXISTS timestamp_idx ON user_daily_visits(timestamp);
+CREATE INDEX IF NOT EXISTS localpart_timestamp_idx ON user_daily_visits(localpart, timestamp);
+`
+
+const countUsersLastSeenAfterSQL = "" +
+	"SELECT COUNT(*) FROM (" +
+	" SELECT localpart FROM device_devices WHERE last_seen_ts > $1 " +
+	" GROUP BY localpart" +
+	" ) u"
+
+// Note on the following countR30UsersSQL and countR30UsersV2SQL: The different checks are intentional.
+// This is to ensure the values reported by Dendrite are the same as by Synapse.
+// Queries are taken from: https://github.com/matrix-org/synapse/blob/9ce51a47f6e37abd0a1275281806399d874eb026/synapse/storage/databases/main/stats.py
+
+/*
+R30Users counts the number of 30 day retained users, defined as:
+- Users who have created their accounts more than 30 days ago
+- Where last seen at most 30 days ago
+- Where account creation and last_seen are > 30 days apart
+*/
+const countR30UsersSQL = `
+SELECT platform, COUNT(*) FROM (
+	SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts)
+	FROM account_accounts users
+	INNER JOIN
+	(SELECT 
+		localpart, last_seen_ts,
+		CASE
+	    	WHEN user_agent LIKE '%%Android%%' THEN 'android'
+    	    WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
+        	WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
+        	WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
+        	WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
+        	ELSE 'unknown'
+		END
+    	AS platform
+		FROM device_devices
+	) uip
+	ON users.localpart = uip.localpart
+	AND users.account_type <> 4
+	AND users.created_ts < $1
+	AND uip.last_seen_ts > $1
+	AND (uip.last_seen_ts) - users.created_ts > $2
+	GROUP BY users.localpart, platform, users.created_ts
+	) u GROUP BY PLATFORM
+`
+
+/*
+R30UsersV2 counts the number of 30 day retained users, defined as users that:
+- Appear more than once in the past 60 days
+- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
+*/
+const countR30UsersV2SQL = `
+SELECT
+	client_type,
+    count(client_type)
+FROM 
+	(
+    	SELECT
+        	localpart,
+            CASE
+            	WHEN
+                LOWER(user_agent) LIKE '%%riot%%' OR
+				LOWER(user_agent) LIKE '%%element%%'
+                THEN CASE
+                	WHEN LOWER(user_agent) LIKE '%%electron%%' THEN 'electron'
+					WHEN LOWER(user_agent) LIKE '%%android%%' THEN 'android'
+					WHEN LOWER(user_agent) LIKE '%%ios%%' THEN 'ios'
+					ELSE 'unknown'
+				END
+				WHEN LOWER(user_agent) LIKE '%%mozilla%%' OR LOWER(user_agent) LIKE '%%gecko%%' THEN 'web'
+				ELSE 'unknown'
+			END as client_type
+		FROM user_daily_visits
+		WHERE timestamp > $1 AND timestamp < $2
+		GROUP BY localpart, client_type
+		HAVING max(timestamp) - min(timestamp) > $3
+	) AS temp
+GROUP BY client_type
+`
+
+const countUserByAccountTypeSQL = `
+SELECT COUNT(*) FROM account_accounts WHERE account_type = ANY($1)
+`
+
+// $1 = All non guest AccountType IDs
+// $2 = Guest AccountType
+const countRegisteredUserByTypeStmt = `
+SELECT user_type, COUNT(*) AS count FROM (
+	SELECT
+    CASE
+    	WHEN account_type = ANY($1) AND appservice_id IS NULL THEN 'native'
+        WHEN account_type = $2 AND appservice_id IS NULL THEN 'guest'
+        WHEN account_type = ANY($1) AND appservice_id IS NOT NULL THEN 'bridged'
+	END AS user_type
+    FROM account_accounts
+    WHERE created_ts > $3
+) AS t GROUP BY user_type
+`
+
+// account_type 1 = users; 3 = admins
+const updateUserDailyVisitsSQL = `
+INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent)
+	SELECT u.localpart, u.device_id, $1, MAX(u.user_agent)
+	FROM device_devices AS u
+	LEFT JOIN (
+		SELECT localpart, device_id, timestamp FROM user_daily_visits
+		WHERE timestamp = $1
+	) udv
+	ON u.localpart = udv.localpart AND u.device_id = udv.device_id
+	INNER JOIN device_devices d ON d.localpart = u.localpart
+	INNER JOIN account_accounts a ON a.localpart = u.localpart
+	WHERE $2 <= d.last_seen_ts AND d.last_seen_ts < $3
+	AND a.account_type in (1, 3)
+	GROUP BY u.localpart, u.device_id
+ON CONFLICT (localpart, device_id, timestamp) DO NOTHING
+;
+`
+
+const queryDBEngineVersion = "SHOW server_version;"
+
+type statsStatements struct {
+	serverName                    gomatrixserverlib.ServerName
+	lastUpdate                    time.Time
+	countUsersLastSeenAfterStmt   *sql.Stmt
+	countR30UsersStmt             *sql.Stmt
+	countR30UsersV2Stmt           *sql.Stmt
+	updateUserDailyVisitsStmt     *sql.Stmt
+	countUserByAccountTypeStmt    *sql.Stmt
+	countRegisteredUserByTypeStmt *sql.Stmt
+	dbEngineVersionStmt           *sql.Stmt
+}
+
+func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
+	s := &statsStatements{
+		serverName: serverName,
+		lastUpdate: time.Now(),
+	}
+
+	_, err := db.Exec(userDailyVisitsSchema)
+	if err != nil {
+		return nil, err
+	}
+	go s.startTimers()
+	return s, sqlutil.StatementList{
+		{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
+		{&s.countR30UsersStmt, countR30UsersSQL},
+		{&s.countR30UsersV2Stmt, countR30UsersV2SQL},
+		{&s.updateUserDailyVisitsStmt, updateUserDailyVisitsSQL},
+		{&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL},
+		{&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeStmt},
+		{&s.dbEngineVersionStmt, queryDBEngineVersion},
+	}.Prepare(db)
+}
+
+func (s *statsStatements) startTimers() {
+	var updateStatsFunc func()
+	updateStatsFunc = func() {
+		logrus.Infof("Executing UpdateUserDailyVisits")
+		if err := s.UpdateUserDailyVisits(context.Background(), nil, time.Now(), s.lastUpdate); err != nil {
+			logrus.WithError(err).Error("failed to update daily user visits")
+		}
+		time.AfterFunc(time.Hour*3, updateStatsFunc)
+	}
+	time.AfterFunc(time.Minute*5, updateStatsFunc)
+}
+
+func (s *statsStatements) allUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUserByAccountTypeStmt)
+	err = stmt.QueryRowContext(ctx,
+		pq.Int64Array{
+			int64(api.AccountTypeUser),
+			int64(api.AccountTypeGuest),
+			int64(api.AccountTypeAdmin),
+			int64(api.AccountTypeAppService),
+		},
+	).Scan(&result)
+	return
+}
+
+func (s *statsStatements) nonBridgedUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUserByAccountTypeStmt)
+	err = stmt.QueryRowContext(ctx,
+		pq.Int64Array{
+			int64(api.AccountTypeUser),
+			int64(api.AccountTypeGuest),
+			int64(api.AccountTypeAdmin),
+		},
+	).Scan(&result)
+	return
+}
+
+func (s *statsStatements) registeredUserByType(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.countRegisteredUserByTypeStmt)
+	registeredAfter := time.Now().AddDate(0, 0, -30)
+
+	rows, err := stmt.QueryContext(ctx,
+		pq.Int64Array{
+			int64(api.AccountTypeUser),
+			int64(api.AccountTypeAdmin),
+			int64(api.AccountTypeAppService),
+		},
+		api.AccountTypeGuest,
+		gomatrixserverlib.AsTimestamp(registeredAfter),
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "RegisteredUserByType: failed to close rows")
+
+	var userType string
+	var count int64
+	var result = make(map[string]int64)
+	for rows.Next() {
+		if err = rows.Scan(&userType, &count); err != nil {
+			return nil, err
+		}
+		result[userType] = count
+	}
+
+	return result, rows.Err()
+}
+
+func (s *statsStatements) dailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -1)
+	err = stmt.QueryRowContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+	).Scan(&result)
+	return
+}
+
+func (s *statsStatements) monthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -30)
+	err = stmt.QueryRowContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+	).Scan(&result)
+	return
+}
+
+/*
+R30Users counts the number of 30 day retained users, defined as:
+- Users who have created their accounts more than 30 days ago
+- Where last seen at most 30 days ago
+- Where account creation and last_seen are > 30 days apart
+*/
+func (s *statsStatements) r30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -30)
+	diff := time.Hour * 24 * 30
+
+	rows, err := stmt.QueryContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+		diff.Milliseconds(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "R30Users: failed to close rows")
+
+	var platform string
+	var count int64
+	var result = make(map[string]int64)
+	for rows.Next() {
+		if err = rows.Scan(&platform, &count); err != nil {
+			return nil, err
+		}
+		if platform == "unknown" {
+			continue
+		}
+		result["all"] += count
+		result[platform] = count
+	}
+
+	return result, rows.Err()
+}
+
+/*
+R30UsersV2 counts the number of 30 day retained users, defined as users that:
+- Appear more than once in the past 60 days
+- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
+*/
+func (s *statsStatements) r30UsersV2(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.countR30UsersV2Stmt)
+	sixtyDaysAgo := time.Now().AddDate(0, 0, -60)
+	diff := time.Hour * 24 * 30
+	tomorrow := time.Now().Add(time.Hour * 24)
+
+	rows, err := stmt.QueryContext(ctx,
+		gomatrixserverlib.AsTimestamp(sixtyDaysAgo),
+		gomatrixserverlib.AsTimestamp(tomorrow),
+		diff.Milliseconds(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "R30UsersV2: failed to close rows")
+
+	var platform string
+	var count int64
+	var result = map[string]int64{
+		"ios":      0,
+		"android":  0,
+		"web":      0,
+		"electron": 0,
+		"all":      0,
+	}
+	for rows.Next() {
+		if err = rows.Scan(&platform, &count); err != nil {
+			return nil, err
+		}
+		if _, ok := result[platform]; !ok {
+			continue
+		}
+		result["all"] += count
+		result[platform] = count
+	}
+
+	return result, rows.Err()
+}
+
+// UserStatistics collects some information about users on this instance.
+// Returns the stats itself as well as the database engine version and type.
+// On error, returns the stats collected up to the error.
+func (s *statsStatements) UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error) {
+	var (
+		stats = &types.UserStatistics{
+			R30UsersV2: map[string]int64{
+				"ios":      0,
+				"android":  0,
+				"web":      0,
+				"electron": 0,
+				"all":      0,
+			},
+			R30Users:              map[string]int64{},
+			RegisteredUsersByType: map[string]int64{},
+		}
+		dbEngine = &types.DatabaseEngine{Engine: "Postgres", Version: "unknown"}
+		err      error
+	)
+	stats.AllUsers, err = s.allUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.DailyUsers, err = s.dailyUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.MonthlyUsers, err = s.monthlyUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.R30Users, err = s.r30Users(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.R30UsersV2, err = s.r30UsersV2(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.NonBridgedUsers, err = s.nonBridgedUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.RegisteredUsersByType, err = s.registeredUserByType(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+
+	stmt := sqlutil.TxStmt(txn, s.dbEngineVersionStmt)
+	err = stmt.QueryRowContext(ctx).Scan(&dbEngine.Version)
+	return stats, dbEngine, err
+}
+
+func (s *statsStatements) UpdateUserDailyVisits(
+	ctx context.Context, txn *sql.Tx,
+	startTime, lastUpdate time.Time,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.updateUserDailyVisitsStmt)
+	startTime = startTime.Truncate(time.Hour * 24)
+
+	// edge case
+	if startTime.After(s.lastUpdate) {
+		startTime = startTime.AddDate(0, 0, -1)
+	}
+	_, err := stmt.ExecContext(ctx,
+		gomatrixserverlib.AsTimestamp(startTime),
+		gomatrixserverlib.AsTimestamp(lastUpdate),
+		gomatrixserverlib.AsTimestamp(time.Now()),
+	)
+	if err == nil {
+		s.lastUpdate = time.Now()
+	}
+	return err
+}
diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go
index 74100a728..b9afb5a56 100644
--- a/userapi/storage/postgres/storage.go
+++ b/userapi/storage/postgres/storage.go
@@ -94,6 +94,10 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, fmt.Errorf("NewPostgresNotificationTable: %w", err)
 	}
+	statsTable, err := NewPostgresStatsTable(db, serverName)
+	if err != nil {
+		return nil, fmt.Errorf("NewPostgresStatsTable: %w", err)
+	}
 	return &shared.Database{
 		AccountDatas:          accountDataTable,
 		Accounts:              accountsTable,
@@ -106,6 +110,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		ThreePIDs:             threePIDTable,
 		Pushers:               pusherTable,
 		Notifications:         notificationsTable,
+		Stats:                 statsTable,
 		ServerName:            serverName,
 		DB:                    db,
 		Writer:                writer,
diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index f7212e030..0cf713dac 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/matrix-org/dendrite/userapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 
@@ -51,6 +52,7 @@ type Database struct {
 	LoginTokens           tables.LoginTokenTable
 	Notifications         tables.NotificationTable
 	Pushers               tables.PusherTable
+	Stats                 tables.StatsTable
 	LoginTokenLifetime    time.Duration
 	ServerName            gomatrixserverlib.ServerName
 	BcryptCost            int
@@ -611,10 +613,10 @@ func (d *Database) RemoveAllDevices(
 	return
 }
 
-// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address
-func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr string) error {
+// UpdateDeviceLastSeen updates a last seen timestamp and the ip address.
+func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID, ipAddr, userAgent string) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr)
+		return d.Devices.UpdateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr, userAgent)
 	})
 }
 
@@ -756,3 +758,8 @@ func (d *Database) RemovePushers(
 		return d.Pushers.DeletePushers(ctx, txn, appid, pushkey)
 	})
 }
+
+// UserStatistics populates types.UserStatistics, used in reports.
+func (d *Database) UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) {
+	return d.Stats.UserStatistics(ctx, nil)
+}
diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go
index b86ed1cc2..93291e6ad 100644
--- a/userapi/storage/sqlite3/devices_table.go
+++ b/userapi/storage/sqlite3/devices_table.go
@@ -81,7 +81,7 @@ const selectDevicesByIDSQL = "" +
 	"SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id IN ($1) ORDER BY last_seen_ts DESC"
 
 const updateDeviceLastSeen = "" +
-	"UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE localpart = $3 AND device_id = $4"
+	"UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5"
 
 type devicesStatements struct {
 	db                           *sql.DB
@@ -306,9 +306,9 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s
 	return devices, rows.Err()
 }
 
-func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error {
+func (s *devicesStatements) UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error {
 	lastSeenTs := time.Now().UnixNano() / 1000000
 	stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt)
-	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, localpart, deviceID)
+	_, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, userAgent, localpart, deviceID)
 	return err
 }
diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go
new file mode 100644
index 000000000..af4c7ff98
--- /dev/null
+++ b/userapi/storage/sqlite3/stats_table.go
@@ -0,0 +1,452 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/tables"
+	"github.com/matrix-org/dendrite/userapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+const userDailyVisitsSchema = `
+CREATE TABLE IF NOT EXISTS user_daily_visits (
+    localpart TEXT NOT NULL,
+	device_id TEXT NOT NULL,
+	timestamp BIGINT NOT NULL,
+	user_agent TEXT
+);
+
+-- Device IDs and timestamp must be unique for a given user per day
+CREATE UNIQUE INDEX IF NOT EXISTS localpart_device_timestamp_idx ON user_daily_visits(localpart, device_id, timestamp);
+CREATE INDEX IF NOT EXISTS timestamp_idx ON user_daily_visits(timestamp);
+CREATE INDEX IF NOT EXISTS localpart_timestamp_idx ON user_daily_visits(localpart, timestamp);
+`
+
+const countUsersLastSeenAfterSQL = "" +
+	"SELECT COUNT(*) FROM (" +
+	" SELECT localpart FROM device_devices WHERE last_seen_ts > $1 " +
+	" GROUP BY localpart" +
+	" ) u"
+
+// Note on the following countR30UsersSQL and countR30UsersV2SQL: The different checks are intentional.
+// This is to ensure the values reported by Dendrite are the same as by Synapse.
+// Queries are taken from: https://github.com/matrix-org/synapse/blob/9ce51a47f6e37abd0a1275281806399d874eb026/synapse/storage/databases/main/stats.py
+
+/*
+R30Users counts the number of 30 day retained users, defined as:
+- Users who have created their accounts more than 30 days ago
+- Where last seen at most 30 days ago
+- Where account creation and last_seen are > 30 days apart
+*/
+const countR30UsersSQL = `
+SELECT platform, COUNT(*) FROM (
+	SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts)
+	FROM account_accounts users
+	INNER JOIN
+	(SELECT 
+		localpart, last_seen_ts,
+		CASE
+	    	WHEN user_agent LIKE '%%Android%%' THEN 'android'
+    	    WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
+        	WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
+        	WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
+        	WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
+        	ELSE 'unknown'
+		END
+    	AS platform
+		FROM device_devices
+	) uip
+	ON users.localpart = uip.localpart
+	AND users.account_type <> 4
+	AND users.created_ts < $1
+	AND uip.last_seen_ts > $2
+	AND (uip.last_seen_ts) - users.created_ts > $3
+	GROUP BY users.localpart, platform, users.created_ts
+	) u GROUP BY PLATFORM
+`
+
+// Note on the following countR30UsersSQL and countR30UsersV2SQL: The different checks are intentional.
+// This is to ensure the values reported are the same as Synapse reports.
+// Queries are taken from: https://github.com/matrix-org/synapse/blob/9ce51a47f6e37abd0a1275281806399d874eb026/synapse/storage/databases/main/stats.py
+
+/*
+R30UsersV2 counts the number of 30 day retained users, defined as users that:
+- Appear more than once in the past 60 days
+- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
+*/
+const countR30UsersV2SQL = `
+SELECT
+	client_type,
+    count(client_type)
+FROM 
+	(
+    	SELECT
+        	localpart,
+            CASE
+            	WHEN
+                LOWER(user_agent) LIKE '%%riot%%' OR
+				LOWER(user_agent) LIKE '%%element%%'
+                THEN CASE
+                	WHEN LOWER(user_agent) LIKE '%%electron%%' THEN 'electron'
+					WHEN LOWER(user_agent) LIKE '%%android%%' THEN 'android'
+					WHEN LOWER(user_agent) LIKE '%%ios%%' THEN 'ios'
+					ELSE 'unknown'
+				END
+				WHEN LOWER(user_agent) LIKE '%%mozilla%%' OR LOWER(user_agent) LIKE '%%gecko%%' THEN 'web'
+				ELSE 'unknown'
+			END as client_type
+		FROM user_daily_visits
+		WHERE timestamp > $1 AND timestamp < $2
+		GROUP BY localpart, client_type
+		HAVING max(timestamp) - min(timestamp) > $3
+	) AS temp
+GROUP BY client_type
+`
+
+const countUserByAccountTypeSQL = `
+SELECT COUNT(*) FROM account_accounts WHERE account_type IN ($1)
+`
+
+// $1 = Guest AccountType
+// $3 & $4 = All non guest AccountType IDs
+const countRegisteredUserByTypeSQL = `
+SELECT user_type, COUNT(*) AS count FROM (
+	SELECT
+    CASE
+    	WHEN account_type IN ($1) AND appservice_id IS NULL THEN 'native'
+        WHEN account_type = $4 AND appservice_id IS NULL THEN 'guest'
+		WHEN account_type IN ($5) AND appservice_id IS NOT NULL THEN 'bridged'
+	END AS user_type
+    FROM account_accounts
+    WHERE created_ts > $8
+) AS t GROUP BY user_type
+`
+
+// account_type 1 = users; 3 = admins
+const updateUserDailyVisitsSQL = `
+INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent)
+	SELECT u.localpart, u.device_id, $1, MAX(u.user_agent)
+	FROM device_devices AS u
+	LEFT JOIN (
+		SELECT localpart, device_id, timestamp FROM user_daily_visits
+		WHERE timestamp = $1
+	) udv
+	ON u.localpart = udv.localpart AND u.device_id = udv.device_id
+	INNER JOIN device_devices d ON d.localpart = u.localpart
+	INNER JOIN account_accounts a ON a.localpart = u.localpart
+	WHERE $2 <= d.last_seen_ts AND d.last_seen_ts < $3
+	AND a.account_type in (1, 3)
+	GROUP BY u.localpart, u.device_id
+ON CONFLICT (localpart, device_id, timestamp) DO NOTHING
+;
+`
+
+const queryDBEngineVersion = "select sqlite_version();"
+
+type statsStatements struct {
+	serverName                    gomatrixserverlib.ServerName
+	db                            *sql.DB
+	lastUpdate                    time.Time
+	countUsersLastSeenAfterStmt   *sql.Stmt
+	countR30UsersStmt             *sql.Stmt
+	countR30UsersV2Stmt           *sql.Stmt
+	updateUserDailyVisitsStmt     *sql.Stmt
+	countUserByAccountTypeStmt    *sql.Stmt
+	countRegisteredUserByTypeStmt *sql.Stmt
+	dbEngineVersionStmt           *sql.Stmt
+}
+
+func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
+	s := &statsStatements{
+		serverName: serverName,
+		lastUpdate: time.Now(),
+		db:         db,
+	}
+
+	_, err := db.Exec(userDailyVisitsSchema)
+	if err != nil {
+		return nil, err
+	}
+	go s.startTimers()
+	return s, sqlutil.StatementList{
+		{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
+		{&s.countR30UsersStmt, countR30UsersSQL},
+		{&s.countR30UsersV2Stmt, countR30UsersV2SQL},
+		{&s.updateUserDailyVisitsStmt, updateUserDailyVisitsSQL},
+		{&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL},
+		{&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeSQL},
+		{&s.dbEngineVersionStmt, queryDBEngineVersion},
+	}.Prepare(db)
+}
+
+func (s *statsStatements) startTimers() {
+	var updateStatsFunc func()
+	updateStatsFunc = func() {
+		logrus.Infof("Executing UpdateUserDailyVisits")
+		if err := s.UpdateUserDailyVisits(context.Background(), nil, time.Now(), s.lastUpdate); err != nil {
+			logrus.WithError(err).Error("failed to update daily user visits")
+		}
+		time.AfterFunc(time.Hour*3, updateStatsFunc)
+	}
+	time.AfterFunc(time.Minute*5, updateStatsFunc)
+}
+
+func (s *statsStatements) allUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	query := strings.Replace(countUserByAccountTypeSQL, "($1)", sqlutil.QueryVariadic(4), 1)
+	queryStmt, err := s.db.Prepare(query)
+	if err != nil {
+		return 0, err
+	}
+	stmt := sqlutil.TxStmt(txn, queryStmt)
+	err = stmt.QueryRowContext(ctx,
+		1, 2, 3, 4,
+	).Scan(&result)
+	return
+}
+
+func (s *statsStatements) nonBridgedUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	query := strings.Replace(countUserByAccountTypeSQL, "($1)", sqlutil.QueryVariadic(3), 1)
+	queryStmt, err := s.db.Prepare(query)
+	if err != nil {
+		return 0, err
+	}
+	stmt := sqlutil.TxStmt(txn, queryStmt)
+	err = stmt.QueryRowContext(ctx,
+		1, 2, 3,
+	).Scan(&result)
+	return
+}
+
+func (s *statsStatements) registeredUserByType(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	// $1 = Guest AccountType; $2 = timestamp
+	// $3 & $4 = All non guest AccountType IDs
+	nonGuests := []api.AccountType{api.AccountTypeUser, api.AccountTypeAdmin, api.AccountTypeAppService}
+	countSQL := strings.Replace(countRegisteredUserByTypeSQL, "($1)", sqlutil.QueryVariadicOffset(len(nonGuests), 0), 1)
+	countSQL = strings.Replace(countSQL, "($5)", sqlutil.QueryVariadicOffset(len(nonGuests), 1+len(nonGuests)), 1)
+	queryStmt, err := s.db.Prepare(countSQL)
+	if err != nil {
+		return nil, err
+	}
+	stmt := sqlutil.TxStmt(txn, queryStmt)
+	registeredAfter := time.Now().AddDate(0, 0, -30)
+
+	params := make([]interface{}, len(nonGuests)*2+2)
+	// nonGuests is used twice
+	for i, v := range nonGuests {
+		params[i] = v                  // i: 0 1 2 => ($1, $2, $3)
+		params[i+1+len(nonGuests)] = v // i: 4 5 6 => ($5, $6, $7)
+	}
+	params[3] = api.AccountTypeGuest                           // $4
+	params[7] = gomatrixserverlib.AsTimestamp(registeredAfter) // $8
+
+	rows, err := stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "RegisteredUserByType: failed to close rows")
+
+	var userType string
+	var count int64
+	var result = make(map[string]int64)
+	for rows.Next() {
+		if err = rows.Scan(&userType, &count); err != nil {
+			return nil, err
+		}
+		result[userType] = count
+	}
+
+	return result, rows.Err()
+}
+
+func (s *statsStatements) dailyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -1)
+	err = stmt.QueryRowContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+	).Scan(&result)
+	return
+}
+
+func (s *statsStatements) monthlyUsers(ctx context.Context, txn *sql.Tx) (result int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.countUsersLastSeenAfterStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -30)
+	err = stmt.QueryRowContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+	).Scan(&result)
+	return
+}
+
+/* R30Users counts the number of 30 day retained users, defined as:
+- Users who have created their accounts more than 30 days ago
+- Where last seen at most 30 days ago
+- Where account creation and last_seen are > 30 days apart
+*/
+func (s *statsStatements) r30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt)
+	lastSeenAfter := time.Now().AddDate(0, 0, -30)
+	diff := time.Hour * 24 * 30
+
+	rows, err := stmt.QueryContext(ctx,
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+		gomatrixserverlib.AsTimestamp(lastSeenAfter),
+		diff.Milliseconds(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "R30Users: failed to close rows")
+
+	var platform string
+	var count int64
+	var result = make(map[string]int64)
+	for rows.Next() {
+		if err = rows.Scan(&platform, &count); err != nil {
+			return nil, err
+		}
+		if platform == "unknown" {
+			continue
+		}
+		result["all"] += count
+		result[platform] = count
+	}
+
+	return result, rows.Err()
+}
+
+/* R30UsersV2 counts the number of 30 day retained users, defined as users that:
+- Appear more than once in the past 60 days
+- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
+*/
+func (s *statsStatements) r30UsersV2(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.countR30UsersV2Stmt)
+	sixtyDaysAgo := time.Now().AddDate(0, 0, -60)
+	diff := time.Hour * 24 * 30
+	tomorrow := time.Now().Add(time.Hour * 24)
+
+	rows, err := stmt.QueryContext(ctx,
+		gomatrixserverlib.AsTimestamp(sixtyDaysAgo),
+		gomatrixserverlib.AsTimestamp(tomorrow),
+		diff.Milliseconds(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "R30UsersV2: failed to close rows")
+
+	var platform string
+	var count int64
+	var result = map[string]int64{
+		"ios":      0,
+		"android":  0,
+		"web":      0,
+		"electron": 0,
+		"all":      0,
+	}
+	for rows.Next() {
+		if err = rows.Scan(&platform, &count); err != nil {
+			return nil, err
+		}
+		if _, ok := result[platform]; !ok {
+			continue
+		}
+		result["all"] += count
+		result[platform] = count
+	}
+	return result, rows.Err()
+}
+
+// UserStatistics collects some information about users on this instance.
+// Returns the stats itself as well as the database engine version and type.
+// On error, returns the stats collected up to the error.
+func (s *statsStatements) UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error) {
+	var (
+		stats = &types.UserStatistics{
+			R30UsersV2: map[string]int64{
+				"ios":      0,
+				"android":  0,
+				"web":      0,
+				"electron": 0,
+				"all":      0,
+			},
+			R30Users:              map[string]int64{},
+			RegisteredUsersByType: map[string]int64{},
+		}
+		dbEngine = &types.DatabaseEngine{Engine: "SQLite", Version: "unknown"}
+		err      error
+	)
+	stats.AllUsers, err = s.allUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.DailyUsers, err = s.dailyUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.MonthlyUsers, err = s.monthlyUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.R30Users, err = s.r30Users(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.R30UsersV2, err = s.r30UsersV2(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.NonBridgedUsers, err = s.nonBridgedUsers(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+	stats.RegisteredUsersByType, err = s.registeredUserByType(ctx, txn)
+	if err != nil {
+		return stats, dbEngine, err
+	}
+
+	stmt := sqlutil.TxStmt(txn, s.dbEngineVersionStmt)
+	err = stmt.QueryRowContext(ctx).Scan(&dbEngine.Version)
+	return stats, dbEngine, err
+}
+
+func (s *statsStatements) UpdateUserDailyVisits(
+	ctx context.Context, txn *sql.Tx,
+	startTime, lastUpdate time.Time,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.updateUserDailyVisitsStmt)
+	startTime = startTime.Truncate(time.Hour * 24)
+
+	// edge case
+	if startTime.After(s.lastUpdate) {
+		startTime = startTime.AddDate(0, 0, -1)
+	}
+	_, err := stmt.ExecContext(ctx,
+		gomatrixserverlib.AsTimestamp(startTime),
+		gomatrixserverlib.AsTimestamp(lastUpdate),
+		gomatrixserverlib.AsTimestamp(time.Now()),
+	)
+	if err == nil {
+		s.lastUpdate = time.Now()
+	}
+	return err
+}
diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go
index 6858d3d15..a822f687d 100644
--- a/userapi/storage/sqlite3/storage.go
+++ b/userapi/storage/sqlite3/storage.go
@@ -95,6 +95,10 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, fmt.Errorf("NewPostgresNotificationTable: %w", err)
 	}
+	statsTable, err := NewSQLiteStatsTable(db, serverName)
+	if err != nil {
+		return nil, fmt.Errorf("NewSQLiteStatsTable: %w", err)
+	}
 	return &shared.Database{
 		AccountDatas:          accountDataTable,
 		Accounts:              accountsTable,
@@ -107,6 +111,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		ThreePIDs:             threePIDTable,
 		Pushers:               pusherTable,
 		Notifications:         notificationsTable,
+		Stats:                 statsTable,
 		ServerName:            serverName,
 		DB:                    db,
 		Writer:                writer,
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index 79d5a8dae..5683fe067 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -174,7 +174,7 @@ func Test_Devices(t *testing.T) {
 		newName := "new display name"
 		err = db.UpdateDevice(ctx, localpart, deviceWithID.ID, &newName)
 		assert.NoError(t, err, "unable to update device displayname")
-		err = db.UpdateDeviceLastSeen(ctx, localpart, deviceWithID.ID, "127.0.0.1")
+		err = db.UpdateDeviceLastSeen(ctx, localpart, deviceWithID.ID, "127.0.0.1", "Element Web")
 		assert.NoError(t, err, "unable to update device last seen")
 
 		deviceWithID.DisplayName = newName
diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go
index eb0cae314..2fe955670 100644
--- a/userapi/storage/tables/interface.go
+++ b/userapi/storage/tables/interface.go
@@ -18,9 +18,11 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/types"
 )
 
 type AccountDataTable interface {
@@ -48,7 +50,7 @@ type DevicesTable interface {
 	SelectDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error)
 	SelectDevicesByLocalpart(ctx context.Context, txn *sql.Tx, localpart, exceptDeviceID string) ([]api.Device, error)
 	SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error)
-	UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr string) error
+	UpdateDeviceLastSeen(ctx context.Context, txn *sql.Tx, localpart, deviceID, ipAddr, userAgent string) error
 }
 
 type KeyBackupTable interface {
@@ -111,6 +113,11 @@ type NotificationTable interface {
 	SelectRoomCounts(ctx context.Context, txn *sql.Tx, localpart, roomID string) (total int64, highlight int64, _ error)
 }
 
+type StatsTable interface {
+	UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error)
+	UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx, startTime, lastUpdate time.Time) error
+}
+
 type NotificationFilter uint32
 
 const (
diff --git a/userapi/storage/tables/stats_table_test.go b/userapi/storage/tables/stats_table_test.go
new file mode 100644
index 000000000..11521c8b0
--- /dev/null
+++ b/userapi/storage/tables/stats_table_test.go
@@ -0,0 +1,319 @@
+package tables_test
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/postgres"
+	"github.com/matrix-org/dendrite/userapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/userapi/storage/tables"
+	"github.com/matrix-org/dendrite/userapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+func mustMakeDBs(t *testing.T, dbType test.DBType) (
+	*sql.DB, tables.AccountsTable, tables.DevicesTable, tables.StatsTable, func(),
+) {
+	t.Helper()
+
+	var (
+		accTable   tables.AccountsTable
+		devTable   tables.DevicesTable
+		statsTable tables.StatsTable
+		err        error
+	)
+
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, nil)
+	if err != nil {
+		t.Fatalf("failed to open db: %s", err)
+	}
+
+	switch dbType {
+	case test.DBTypeSQLite:
+		accTable, err = sqlite3.NewSQLiteAccountsTable(db, "localhost")
+		if err != nil {
+			t.Fatalf("unable to create acc db: %v", err)
+		}
+		devTable, err = sqlite3.NewSQLiteDevicesTable(db, "localhost")
+		if err != nil {
+			t.Fatalf("unable to open device db: %v", err)
+		}
+		statsTable, err = sqlite3.NewSQLiteStatsTable(db, "localhost")
+		if err != nil {
+			t.Fatalf("unable to open stats db: %v", err)
+		}
+	case test.DBTypePostgres:
+		accTable, err = postgres.NewPostgresAccountsTable(db, "localhost")
+		if err != nil {
+			t.Fatalf("unable to create acc db: %v", err)
+		}
+		devTable, err = postgres.NewPostgresDevicesTable(db, "localhost")
+		if err != nil {
+			t.Fatalf("unable to open device db: %v", err)
+		}
+		statsTable, err = postgres.NewPostgresStatsTable(db, "localhost")
+		if err != nil {
+			t.Fatalf("unable to open stats db: %v", err)
+		}
+	}
+
+	return db, accTable, devTable, statsTable, close
+}
+
+func mustMakeAccountAndDevice(
+	t *testing.T,
+	ctx context.Context,
+	accDB tables.AccountsTable,
+	devDB tables.DevicesTable,
+	localpart string,
+	accType api.AccountType,
+	userAgent string,
+) {
+	t.Helper()
+
+	appServiceID := ""
+	if accType == api.AccountTypeAppService {
+		appServiceID = util.RandomString(16)
+	}
+
+	_, err := accDB.InsertAccount(ctx, nil, localpart, "", appServiceID, accType)
+	if err != nil {
+		t.Fatalf("unable to create account: %v", err)
+	}
+	_, err = devDB.InsertDevice(ctx, nil, "deviceID", localpart, util.RandomString(16), nil, "", userAgent)
+	if err != nil {
+		t.Fatalf("unable to create device: %v", err)
+	}
+}
+
+func mustUpdateDeviceLastSeen(
+	t *testing.T,
+	ctx context.Context,
+	db *sql.DB,
+	localpart string,
+	timestamp time.Time,
+) {
+	t.Helper()
+	_, err := db.ExecContext(ctx, "UPDATE device_devices SET last_seen_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart)
+	if err != nil {
+		t.Fatalf("unable to update device last seen")
+	}
+}
+
+func mustUserUpdateRegistered(
+	t *testing.T,
+	ctx context.Context,
+	db *sql.DB,
+	localpart string,
+	timestamp time.Time,
+) {
+	_, err := db.ExecContext(ctx, "UPDATE account_accounts SET created_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart)
+	if err != nil {
+		t.Fatalf("unable to update device last seen")
+	}
+}
+
+// These tests must run sequentially, as they build up on each other
+func Test_UserStatistics(t *testing.T) {
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, accDB, devDB, statsDB, close := mustMakeDBs(t, dbType)
+		defer close()
+		wantType := "SQLite"
+		if dbType == test.DBTypePostgres {
+			wantType = "Postgres"
+		}
+
+		t.Run(fmt.Sprintf("want %s database engine", wantType), func(t *testing.T) {
+			_, gotDB, err := statsDB.UserStatistics(ctx, nil)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if wantType != gotDB.Engine { // can't use DeepEqual, as the Version might differ
+				t.Errorf("UserStatistics() got DB engine = %+v, want %s", gotDB.Engine, wantType)
+			}
+		})
+
+		t.Run("Want Users", func(t *testing.T) {
+			mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user1", api.AccountTypeUser, "Element Android")
+			mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user2", api.AccountTypeUser, "Element iOS")
+			mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user3", api.AccountTypeUser, "Element web")
+			mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user4", api.AccountTypeGuest, "Element Electron")
+			mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user5", api.AccountTypeAdmin, "gecko")
+			mustMakeAccountAndDevice(t, ctx, accDB, devDB, "user6", api.AccountTypeAppService, "gecko")
+			gotStats, _, err := statsDB.UserStatistics(ctx, nil)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			wantStats := &types.UserStatistics{
+				RegisteredUsersByType: map[string]int64{
+					"native":  4,
+					"guest":   1,
+					"bridged": 1,
+				},
+				R30Users: map[string]int64{},
+				R30UsersV2: map[string]int64{
+					"ios":      0,
+					"android":  0,
+					"web":      0,
+					"electron": 0,
+					"all":      0,
+				},
+				AllUsers:        6,
+				NonBridgedUsers: 5,
+				DailyUsers:      6,
+				MonthlyUsers:    6,
+			}
+			if !reflect.DeepEqual(gotStats, wantStats) {
+				t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats)
+			}
+		})
+
+		t.Run("Users not active for one/two month", func(t *testing.T) {
+			mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, -2, 0))
+			mustUpdateDeviceLastSeen(t, ctx, db, "user2", time.Now().AddDate(0, -1, 0))
+			gotStats, _, err := statsDB.UserStatistics(ctx, nil)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			wantStats := &types.UserStatistics{
+				RegisteredUsersByType: map[string]int64{
+					"native":  4,
+					"guest":   1,
+					"bridged": 1,
+				},
+				R30Users: map[string]int64{},
+				R30UsersV2: map[string]int64{
+					"ios":      0,
+					"android":  0,
+					"web":      0,
+					"electron": 0,
+					"all":      0,
+				},
+				AllUsers:        6,
+				NonBridgedUsers: 5,
+				DailyUsers:      4,
+				MonthlyUsers:    4,
+			}
+			if !reflect.DeepEqual(gotStats, wantStats) {
+				t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats)
+			}
+		})
+
+		/* R30Users counts the number of 30 day retained users, defined as:
+		- Users who have created their accounts more than 30 days ago
+		- Where last seen at most 30 days ago
+		- Where account creation and last_seen are > 30 days apart
+		*/
+		t.Run("R30Users tests", func(t *testing.T) {
+			mustUserUpdateRegistered(t, ctx, db, "user1", time.Now().AddDate(0, -2, 0))
+			mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now())
+			mustUserUpdateRegistered(t, ctx, db, "user4", time.Now().AddDate(0, -2, 0))
+			mustUpdateDeviceLastSeen(t, ctx, db, "user4", time.Now())
+			startTime := time.Now().AddDate(0, 0, -2)
+			err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour))
+			if err != nil {
+				t.Fatalf("unable to update daily visits stats: %v", err)
+			}
+
+			gotStats, _, err := statsDB.UserStatistics(ctx, nil)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			wantStats := &types.UserStatistics{
+				RegisteredUsersByType: map[string]int64{
+					"native":  3,
+					"bridged": 1,
+				},
+				R30Users: map[string]int64{
+					"all":      2,
+					"android":  1,
+					"electron": 1,
+				},
+				R30UsersV2: map[string]int64{
+					"ios":      0,
+					"android":  0,
+					"web":      0,
+					"electron": 0,
+					"all":      0,
+				},
+				AllUsers:        6,
+				NonBridgedUsers: 5,
+				DailyUsers:      5,
+				MonthlyUsers:    5,
+			}
+			if !reflect.DeepEqual(gotStats, wantStats) {
+				t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats)
+			}
+		})
+
+		/*
+			R30UsersV2 counts the number of 30 day retained users, defined as users that:
+			- Appear more than once in the past 60 days
+			- Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
+			most recent -> neueste
+			least recent -> älteste
+
+		*/
+		t.Run("R30UsersV2 tests", func(t *testing.T) {
+			// generate some data
+			for i := 100; i > 0; i-- {
+				mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, 0, -i))
+				mustUpdateDeviceLastSeen(t, ctx, db, "user5", time.Now().AddDate(0, 0, -i))
+				startTime := time.Now().AddDate(0, 0, -i)
+				err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour))
+				if err != nil {
+					t.Fatalf("unable to update daily visits stats: %v", err)
+				}
+			}
+			gotStats, _, err := statsDB.UserStatistics(ctx, nil)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			wantStats := &types.UserStatistics{
+				RegisteredUsersByType: map[string]int64{
+					"native":  3,
+					"bridged": 1,
+				},
+				R30Users: map[string]int64{
+					"all":      2,
+					"android":  1,
+					"electron": 1,
+				},
+				R30UsersV2: map[string]int64{
+					"ios":      0,
+					"android":  1,
+					"web":      1,
+					"electron": 0,
+					"all":      2,
+				},
+				AllUsers:        6,
+				NonBridgedUsers: 5,
+				DailyUsers:      3,
+				MonthlyUsers:    5,
+			}
+			if !reflect.DeepEqual(gotStats, wantStats) {
+				t.Errorf("UserStatistics() gotStats = \n%+v\nwant\n%+v", gotStats, wantStats)
+			}
+		})
+	})
+
+}
diff --git a/userapi/types/statistics.go b/userapi/types/statistics.go
new file mode 100644
index 000000000..09564f78f
--- /dev/null
+++ b/userapi/types/statistics.go
@@ -0,0 +1,30 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+type UserStatistics struct {
+	RegisteredUsersByType map[string]int64
+	R30Users              map[string]int64
+	R30UsersV2            map[string]int64
+	AllUsers              int64
+	NonBridgedUsers       int64
+	DailyUsers            int64
+	MonthlyUsers          int64
+}
+
+type DatabaseEngine struct {
+	Engine  string
+	Version string
+}
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 9174119e1..5b11665db 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -30,6 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/userapi/inthttp"
 	"github.com/matrix-org/dendrite/userapi/producers"
 	"github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/dendrite/userapi/util"
 	"github.com/sirupsen/logrus"
 )
 
@@ -104,5 +105,9 @@ func NewInternalAPI(
 	}
 	time.AfterFunc(time.Minute, cleanOldNotifs)
 
+	if base.Cfg.Global.ReportStats.Enabled {
+		go util.StartPhoneHomeCollector(time.Now(), base.Cfg, db)
+	}
+
 	return userAPI
 }
diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go
new file mode 100644
index 000000000..ad93a50e3
--- /dev/null
+++ b/userapi/util/phonehomestats.go
@@ -0,0 +1,160 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"math"
+	"net/http"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+type phoneHomeStats struct {
+	prevData   timestampToRUUsage
+	stats      map[string]interface{}
+	serverName gomatrixserverlib.ServerName
+	startTime  time.Time
+	cfg        *config.Dendrite
+	db         storage.Statistics
+	isMonolith bool
+	client     *http.Client
+}
+
+type timestampToRUUsage struct {
+	timestamp int64
+	usage     syscall.Rusage
+}
+
+func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB storage.Statistics) {
+
+	p := phoneHomeStats{
+		startTime:  startTime,
+		serverName: cfg.Global.ServerName,
+		cfg:        cfg,
+		db:         statsDB,
+		isMonolith: cfg.IsMonolith,
+		client: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: http.DefaultTransport,
+		},
+	}
+
+	// start initial run after 5min
+	time.AfterFunc(time.Minute*5, p.collect)
+
+	// run every 3 hours
+	ticker := time.NewTicker(time.Hour * 3)
+	for range ticker.C {
+		p.collect()
+	}
+}
+
+func (p *phoneHomeStats) collect() {
+	p.stats = make(map[string]interface{})
+	// general information
+	p.stats["homeserver"] = p.serverName
+	p.stats["monolith"] = p.isMonolith
+	p.stats["version"] = internal.VersionString()
+	p.stats["timestamp"] = time.Now().Unix()
+	p.stats["go_version"] = runtime.Version()
+	p.stats["go_arch"] = runtime.GOARCH
+	p.stats["go_os"] = runtime.GOOS
+	p.stats["num_cpu"] = runtime.NumCPU()
+	p.stats["num_go_routine"] = runtime.NumGoroutine()
+	p.stats["uptime_seconds"] = math.Floor(time.Since(p.startTime).Seconds())
+
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1)
+	defer cancel()
+
+	// cpu and memory usage information
+	err := getMemoryStats(p)
+	if err != nil {
+		logrus.WithError(err).Warn("unable to get memory/cpu stats, using defaults")
+	}
+
+	// configuration information
+	p.stats["federation_disabled"] = p.cfg.Global.DisableFederation
+	p.stats["nats_embedded"] = true
+	p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory
+	if len(p.cfg.Global.JetStream.Addresses) > 0 {
+		p.stats["nats_embedded"] = false
+		p.stats["nats_in_memory"] = false // probably
+	}
+	if len(p.cfg.Logging) > 0 {
+		p.stats["log_level"] = p.cfg.Logging[0].Level
+	} else {
+		p.stats["log_level"] = "info"
+	}
+
+	// message and room stats
+	// TODO: Find a solution to actually set these values
+	p.stats["total_room_count"] = 0
+	p.stats["daily_messages"] = 0
+	p.stats["daily_sent_messages"] = 0
+	p.stats["daily_e2ee_messages"] = 0
+	p.stats["daily_sent_e2ee_messages"] = 0
+
+	// user stats and DB engine
+	userStats, db, err := p.db.UserStatistics(ctx)
+	if err != nil {
+		logrus.WithError(err).Warn("unable to query userstats, using default values")
+	}
+	p.stats["database_engine"] = db.Engine
+	p.stats["database_server_version"] = db.Version
+	p.stats["total_users"] = userStats.AllUsers
+	p.stats["total_nonbridged_users"] = userStats.NonBridgedUsers
+	p.stats["daily_active_users"] = userStats.DailyUsers
+	p.stats["monthly_active_users"] = userStats.MonthlyUsers
+	for t, c := range userStats.RegisteredUsersByType {
+		p.stats["daily_user_type_"+t] = c
+	}
+	for t, c := range userStats.R30Users {
+		p.stats["r30_users_"+t] = c
+	}
+	for t, c := range userStats.R30UsersV2 {
+		p.stats["r30v2_users_"+t] = c
+	}
+
+	output := bytes.Buffer{}
+	if err = json.NewEncoder(&output).Encode(p.stats); err != nil {
+		logrus.WithError(err).Error("unable to encode anonymous stats")
+		return
+	}
+
+	logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStats.Endpoint, output.String())
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, p.cfg.Global.ReportStats.Endpoint, &output)
+	if err != nil {
+		logrus.WithError(err).Error("unable to create anonymous stats request")
+		return
+	}
+	request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
+
+	_, err = p.client.Do(request)
+	if err != nil {
+		logrus.WithError(err).Error("unable to send anonymous stats")
+		return
+	}
+}
diff --git a/userapi/util/stats.go b/userapi/util/stats.go
new file mode 100644
index 000000000..22ef12aad
--- /dev/null
+++ b/userapi/util/stats.go
@@ -0,0 +1,47 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !wasm && !windows
+// +build !wasm,!windows
+
+package util
+
+import (
+	"syscall"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+func getMemoryStats(p *phoneHomeStats) error {
+	oldUsage := p.prevData
+	newUsage := syscall.Rusage{}
+	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &newUsage); err != nil {
+		logrus.WithError(err).Error("unable to get usage")
+		return err
+	}
+	newData := timestampToRUUsage{timestamp: time.Now().Unix(), usage: newUsage}
+	p.prevData = newData
+
+	usedCPUTime := (newUsage.Utime.Sec + newUsage.Stime.Sec) - (oldUsage.usage.Utime.Sec + oldUsage.usage.Stime.Sec)
+
+	if usedCPUTime == 0 || newData.timestamp == oldUsage.timestamp {
+		p.stats["cpu_average"] = 0
+	} else {
+		// conversion to int64 required for GOARCH=386
+		p.stats["cpu_average"] = int64(usedCPUTime) / (newData.timestamp - oldUsage.timestamp) * 100
+	}
+	p.stats["memory_rss"] = newUsage.Maxrss
+	return nil
+}
diff --git a/userapi/util/stats_wasm.go b/userapi/util/stats_wasm.go
new file mode 100644
index 000000000..a182e4e6e
--- /dev/null
+++ b/userapi/util/stats_wasm.go
@@ -0,0 +1,20 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+// stub, since WASM doesn't support syscall.Getrusage
+func getMemoryStats(p *phoneHomeStats) error {
+	return nil
+}
diff --git a/userapi/util/stats_windows.go b/userapi/util/stats_windows.go
new file mode 100644
index 000000000..0b3f8d013
--- /dev/null
+++ b/userapi/util/stats_windows.go
@@ -0,0 +1,29 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !wasm
+// +build !wasm
+
+package util
+
+import (
+	"runtime"
+)
+
+func getMemoryStats(p *phoneHomeStats) error {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	p.stats["memory_rss"] = memStats.Alloc
+	return nil
+}

From d86dcbef66dad344bc38c58762a9634ff126d5c7 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 5 May 2022 09:56:03 +0100
Subject: [PATCH 069/391] syncapi: define specific interfaces for internal HTTP
 communications (#2416)

* syncapi: use finer-grained interfaces when making the syncapi

* Use specific interfaces for syncapi-roomserver interactions

* Define query access token api for shared http auth code
---
 clientapi/auth/auth.go                        |  2 +-
 .../personalities/syncapi.go                  |  2 -
 internal/httputil/httpapi.go                  |  2 +-
 keyserver/api/api.go                          |  9 ++-
 roomserver/api/api.go                         | 81 ++++++++++---------
 setup/monolith.go                             |  2 +-
 syncapi/consumers/keychange.go                |  7 +-
 syncapi/consumers/presence.go                 |  4 +-
 syncapi/consumers/roomserver.go               |  4 +-
 syncapi/internal/keychange.go                 |  8 +-
 syncapi/routing/context.go                    |  2 +-
 syncapi/routing/messages.go                   |  9 +--
 syncapi/routing/routing.go                    |  6 +-
 syncapi/streams/stream_accountdata.go         |  2 +-
 syncapi/streams/stream_devicelist.go          |  4 +-
 syncapi/streams/stream_pdu.go                 |  2 +-
 syncapi/streams/streams.go                    |  4 +-
 syncapi/sync/requestpool.go                   | 10 +--
 syncapi/syncapi.go                            | 12 ++-
 userapi/api/api.go                            | 14 +++-
 20 files changed, 95 insertions(+), 91 deletions(-)

diff --git a/clientapi/auth/auth.go b/clientapi/auth/auth.go
index 575c5377f..93345f4b9 100644
--- a/clientapi/auth/auth.go
+++ b/clientapi/auth/auth.go
@@ -51,7 +51,7 @@ type AccountDatabase interface {
 // Note: For an AS user, AS dummy device is returned.
 // On failure returns an JSON error response which can be sent to the client.
 func VerifyUserFromRequest(
-	req *http.Request, userAPI api.UserInternalAPI,
+	req *http.Request, userAPI api.QueryAcccessTokenAPI,
 ) (*api.Device, *util.JSONResponse) {
 	// Try to find the Application Service user
 	token, err := ExtractAccessToken(req)
diff --git a/cmd/dendrite-polylith-multi/personalities/syncapi.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go
index 2245b9b54..41637fe1d 100644
--- a/cmd/dendrite-polylith-multi/personalities/syncapi.go
+++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go
@@ -22,7 +22,6 @@ import (
 
 func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
-	federation := base.CreateFederationClient()
 
 	rsAPI := base.RoomserverHTTPClient()
 
@@ -30,7 +29,6 @@ func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 		base,
 		userAPI, rsAPI,
 		base.KeyServerHTTPClient(),
-		federation,
 	)
 
 	base.SetupAndServeHTTP(
diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index 5fcacd2ad..3a818cc5e 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -49,7 +49,7 @@ type BasicAuth struct {
 
 // MakeAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request.
 func MakeAuthAPI(
-	metricsName string, userAPI userapi.UserInternalAPI,
+	metricsName string, userAPI userapi.QueryAcccessTokenAPI,
 	f func(*http.Request, *userapi.Device) util.JSONResponse,
 ) http.Handler {
 	h := func(req *http.Request) util.JSONResponse {
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index 429617b10..ce651ba4e 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -27,6 +27,7 @@ import (
 )
 
 type KeyInternalAPI interface {
+	SyncKeyAPI
 	// SetUserAPI assigns a user API to query when extracting device names.
 	SetUserAPI(i userapi.UserInternalAPI)
 	// InputDeviceListUpdate from a federated server EDU
@@ -38,12 +39,16 @@ type KeyInternalAPI interface {
 	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse)
 	PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse)
 	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse)
-	QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse)
-	QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse)
 	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse)
 	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse)
 }
 
+// API functions required by the syncapi
+type SyncKeyAPI interface {
+	QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse)
+	QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse)
+}
+
 // KeyError is returned if there was a problem performing/querying the server
 type KeyError struct {
 	Err                string `json:"error"`
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index f0ca8a615..2e4ec3ffd 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -12,6 +12,8 @@ import (
 
 // RoomserverInputAPI is used to write events to the room server.
 type RoomserverInternalAPI interface {
+	SyncRoomserverAPI
+
 	// needed to avoid chicken and egg scenario when setting up the
 	// interdependencies between the roomserver and other input APIs
 	SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing)
@@ -78,34 +80,6 @@ type RoomserverInternalAPI interface {
 		res *QueryPublishedRoomsResponse,
 	) error
 
-	// Query the latest events and state for a room from the room server.
-	QueryLatestEventsAndState(
-		ctx context.Context,
-		request *QueryLatestEventsAndStateRequest,
-		response *QueryLatestEventsAndStateResponse,
-	) error
-
-	// Query the state after a list of events in a room from the room server.
-	QueryStateAfterEvents(
-		ctx context.Context,
-		request *QueryStateAfterEventsRequest,
-		response *QueryStateAfterEventsResponse,
-	) error
-
-	// Query a list of events by event ID.
-	QueryEventsByID(
-		ctx context.Context,
-		request *QueryEventsByIDRequest,
-		response *QueryEventsByIDResponse,
-	) error
-
-	// Query the membership event for an user for a room.
-	QueryMembershipForUser(
-		ctx context.Context,
-		request *QueryMembershipForUserRequest,
-		response *QueryMembershipForUserResponse,
-	) error
-
 	// Query a list of membership events for a room
 	QueryMembershipsForRoom(
 		ctx context.Context,
@@ -157,22 +131,11 @@ type RoomserverInternalAPI interface {
 	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
 	// QueryRoomsForUser retrieves a list of room IDs matching the given query.
 	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
-	// QueryBulkStateContent does a bulk query for state event content in the given rooms.
-	QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
-	// QuerySharedUsers returns a list of users who share at least 1 room in common with the given user.
-	QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error
 	// QueryKnownUsers returns a list of users that we know about from our joined rooms.
 	QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error
 	// QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs.
 	QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error
 
-	// Query a given amount (or less) of events prior to a given set of events.
-	PerformBackfill(
-		ctx context.Context,
-		request *PerformBackfillRequest,
-		response *PerformBackfillResponse,
-	) error
-
 	// PerformForget forgets a rooms history for a specific user
 	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
 
@@ -228,3 +191,43 @@ type RoomserverInternalAPI interface {
 		response *RemoveRoomAliasResponse,
 	) error
 }
+
+// API functions required by the syncapi
+type SyncRoomserverAPI interface {
+	// Query the latest events and state for a room from the room server.
+	QueryLatestEventsAndState(
+		ctx context.Context,
+		request *QueryLatestEventsAndStateRequest,
+		response *QueryLatestEventsAndStateResponse,
+	) error
+	// QueryBulkStateContent does a bulk query for state event content in the given rooms.
+	QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
+	// QuerySharedUsers returns a list of users who share at least 1 room in common with the given user.
+	QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error
+	// Query a list of events by event ID.
+	QueryEventsByID(
+		ctx context.Context,
+		request *QueryEventsByIDRequest,
+		response *QueryEventsByIDResponse,
+	) error
+	// Query the membership event for an user for a room.
+	QueryMembershipForUser(
+		ctx context.Context,
+		request *QueryMembershipForUserRequest,
+		response *QueryMembershipForUserResponse,
+	) error
+
+	// Query the state after a list of events in a room from the room server.
+	QueryStateAfterEvents(
+		ctx context.Context,
+		request *QueryStateAfterEventsRequest,
+		response *QueryStateAfterEventsResponse,
+	) error
+
+	// Query a given amount (or less) of events prior to a given set of events.
+	PerformBackfill(
+		ctx context.Context,
+		request *PerformBackfillRequest,
+		response *PerformBackfillResponse,
+	) error
+}
diff --git a/setup/monolith.go b/setup/monolith.go
index 23bd2fb52..e033c14d7 100644
--- a/setup/monolith.go
+++ b/setup/monolith.go
@@ -69,6 +69,6 @@ func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite) {
 		base, m.UserAPI, m.Client,
 	)
 	syncapi.AddPublicRoutes(
-		base, m.UserAPI, m.RoomserverAPI, m.KeyAPI, m.FedClient,
+		base, m.UserAPI, m.RoomserverAPI, m.KeyAPI,
 	)
 }
diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go
index e806f76e6..c8d88ddac 100644
--- a/syncapi/consumers/keychange.go
+++ b/syncapi/consumers/keychange.go
@@ -42,8 +42,7 @@ type OutputKeyChangeEventConsumer struct {
 	notifier   *notifier.Notifier
 	stream     types.StreamProvider
 	serverName gomatrixserverlib.ServerName // our server name
-	rsAPI      roomserverAPI.RoomserverInternalAPI
-	keyAPI     api.KeyInternalAPI
+	rsAPI      roomserverAPI.SyncRoomserverAPI
 }
 
 // NewOutputKeyChangeEventConsumer creates a new OutputKeyChangeEventConsumer.
@@ -53,8 +52,7 @@ func NewOutputKeyChangeEventConsumer(
 	cfg *config.SyncAPI,
 	topic string,
 	js nats.JetStreamContext,
-	keyAPI api.KeyInternalAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.SyncRoomserverAPI,
 	store storage.Database,
 	notifier *notifier.Notifier,
 	stream types.StreamProvider,
@@ -66,7 +64,6 @@ func NewOutputKeyChangeEventConsumer(
 		topic:      topic,
 		db:         store,
 		serverName: cfg.Matrix.ServerName,
-		keyAPI:     keyAPI,
 		rsAPI:      rsAPI,
 		notifier:   notifier,
 		stream:     stream,
diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go
index 6bcca48f4..388c08ff4 100644
--- a/syncapi/consumers/presence.go
+++ b/syncapi/consumers/presence.go
@@ -41,7 +41,7 @@ type PresenceConsumer struct {
 	db            storage.Database
 	stream        types.StreamProvider
 	notifier      *notifier.Notifier
-	deviceAPI     api.UserDeviceAPI
+	deviceAPI     api.SyncUserAPI
 	cfg           *config.SyncAPI
 }
 
@@ -55,7 +55,7 @@ func NewPresenceConsumer(
 	db storage.Database,
 	notifier *notifier.Notifier,
 	stream types.StreamProvider,
-	deviceAPI api.UserDeviceAPI,
+	deviceAPI api.SyncUserAPI,
 ) *PresenceConsumer {
 	return &PresenceConsumer{
 		ctx:           process.Context(),
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 5bdc0fad7..7712c8403 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -38,7 +38,7 @@ import (
 type OutputRoomEventConsumer struct {
 	ctx          context.Context
 	cfg          *config.SyncAPI
-	rsAPI        api.RoomserverInternalAPI
+	rsAPI        api.SyncRoomserverAPI
 	jetstream    nats.JetStreamContext
 	durable      string
 	topic        string
@@ -58,7 +58,7 @@ func NewOutputRoomEventConsumer(
 	notifier *notifier.Notifier,
 	pduStream types.StreamProvider,
 	inviteStream types.StreamProvider,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.SyncRoomserverAPI,
 	producer *producers.UserAPIStreamEventProducer,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go
index dc4acd8da..d96718d20 100644
--- a/syncapi/internal/keychange.go
+++ b/syncapi/internal/keychange.go
@@ -29,7 +29,7 @@ import (
 const DeviceListLogName = "dl"
 
 // DeviceOTKCounts adds one-time key counts to the /sync response
-func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.KeyInternalAPI, userID, deviceID string, res *types.Response) error {
+func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, deviceID string, res *types.Response) error {
 	var queryRes keyapi.QueryOneTimeKeysResponse
 	keyAPI.QueryOneTimeKeys(ctx, &keyapi.QueryOneTimeKeysRequest{
 		UserID:   userID,
@@ -46,7 +46,7 @@ func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.KeyInternalAPI, userID,
 // was filled in, else false if there are no new device list changes because there is nothing to catch up on. The response MUST
 // be already filled in with join/leave information.
 func DeviceListCatchup(
-	ctx context.Context, keyAPI keyapi.KeyInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
+	ctx context.Context, keyAPI keyapi.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI,
 	userID string, res *types.Response, from, to types.StreamPosition,
 ) (newPos types.StreamPosition, hasNew bool, err error) {
 
@@ -130,7 +130,7 @@ func DeviceListCatchup(
 
 // TrackChangedUsers calculates the values of device_lists.changed|left in the /sync response.
 func TrackChangedUsers(
-	ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, userID string, newlyJoinedRooms, newlyLeftRooms []string,
+	ctx context.Context, rsAPI roomserverAPI.SyncRoomserverAPI, userID string, newlyJoinedRooms, newlyLeftRooms []string,
 ) (changed, left []string, err error) {
 	// process leaves first, then joins afterwards so if we join/leave/join/leave we err on the side of including users.
 
@@ -216,7 +216,7 @@ func TrackChangedUsers(
 }
 
 func filterSharedUsers(
-	ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, userID string, usersWithChangedKeys []string,
+	ctx context.Context, rsAPI roomserverAPI.SyncRoomserverAPI, userID string, usersWithChangedKeys []string,
 ) (map[string]int, []string) {
 	var result []string
 	var sharedUsersRes roomserverAPI.QuerySharedUsersResponse
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 17215b669..f5f4b2dd0 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -42,7 +42,7 @@ type ContextRespsonse struct {
 
 func Context(
 	req *http.Request, device *userapi.Device,
-	rsAPI roomserver.RoomserverInternalAPI,
+	rsAPI roomserver.SyncRoomserverAPI,
 	syncDB storage.Database,
 	roomID, eventID string,
 	lazyLoadCache *caching.LazyLoadCache,
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index f34901bf2..f19dfaed3 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -36,8 +36,7 @@ import (
 type messagesReq struct {
 	ctx              context.Context
 	db               storage.Database
-	rsAPI            api.RoomserverInternalAPI
-	federation       *gomatrixserverlib.FederationClient
+	rsAPI            api.SyncRoomserverAPI
 	cfg              *config.SyncAPI
 	roomID           string
 	from             *types.TopologyToken
@@ -61,8 +60,7 @@ type messagesResp struct {
 // See: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-client-r0-rooms-roomid-messages
 func OnIncomingMessagesRequest(
 	req *http.Request, db storage.Database, roomID string, device *userapi.Device,
-	federation *gomatrixserverlib.FederationClient,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.SyncRoomserverAPI,
 	cfg *config.SyncAPI,
 	srp *sync.RequestPool,
 	lazyLoadCache *caching.LazyLoadCache,
@@ -180,7 +178,6 @@ func OnIncomingMessagesRequest(
 		ctx:              req.Context(),
 		db:               db,
 		rsAPI:            rsAPI,
-		federation:       federation,
 		cfg:              cfg,
 		roomID:           roomID,
 		from:             &from,
@@ -247,7 +244,7 @@ func OnIncomingMessagesRequest(
 	}
 }
 
-func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.RoomserverInternalAPI) (bool, error) {
+func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (bool, error) {
 	req := api.QueryMembershipForUserRequest{
 		RoomID: roomID,
 		UserID: userID,
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 4102cf073..245ee5b66 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -36,8 +36,8 @@ import (
 // nolint: gocyclo
 func Setup(
 	csMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database,
-	userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient,
-	rsAPI api.RoomserverInternalAPI,
+	userAPI userapi.SyncUserAPI,
+	rsAPI api.SyncRoomserverAPI,
 	cfg *config.SyncAPI,
 	lazyLoadCache *caching.LazyLoadCache,
 ) {
@@ -53,7 +53,7 @@ func Setup(
 		if err != nil {
 			return util.ErrorResponse(err)
 		}
-		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg, srp, lazyLoadCache)
+		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, rsAPI, cfg, srp, lazyLoadCache)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/user/{userId}/filter",
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 2cddbcf04..9c19b846b 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -10,7 +10,7 @@ import (
 
 type AccountDataStreamProvider struct {
 	StreamProvider
-	userAPI userapi.UserInternalAPI
+	userAPI userapi.SyncUserAPI
 }
 
 func (p *AccountDataStreamProvider) Setup() {
diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go
index 6ff8a7fd5..f42099510 100644
--- a/syncapi/streams/stream_devicelist.go
+++ b/syncapi/streams/stream_devicelist.go
@@ -11,8 +11,8 @@ import (
 
 type DeviceListStreamProvider struct {
 	StreamProvider
-	rsAPI  api.RoomserverInternalAPI
-	keyAPI keyapi.KeyInternalAPI
+	rsAPI  api.SyncRoomserverAPI
+	keyAPI keyapi.SyncKeyAPI
 }
 
 func (p *DeviceListStreamProvider) CompleteSync(
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 0d033095d..f774a1af8 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -33,7 +33,7 @@ type PDUStreamProvider struct {
 	workers atomic.Int32
 	// userID+deviceID -> lazy loading cache
 	lazyLoadCache *caching.LazyLoadCache
-	rsAPI         roomserverAPI.RoomserverInternalAPI
+	rsAPI         roomserverAPI.SyncRoomserverAPI
 }
 
 func (p *PDUStreamProvider) worker() {
diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go
index a18a0cc41..af2a0387e 100644
--- a/syncapi/streams/streams.go
+++ b/syncapi/streams/streams.go
@@ -25,8 +25,8 @@ type Streams struct {
 }
 
 func NewSyncStreamProviders(
-	d storage.Database, userAPI userapi.UserInternalAPI,
-	rsAPI rsapi.RoomserverInternalAPI, keyAPI keyapi.KeyInternalAPI,
+	d storage.Database, userAPI userapi.SyncUserAPI,
+	rsAPI rsapi.SyncRoomserverAPI, keyAPI keyapi.SyncKeyAPI,
 	eduCache *caching.EDUCache, lazyLoadCache *caching.LazyLoadCache, notifier *notifier.Notifier,
 ) *Streams {
 	streams := &Streams{
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index f8e502d2c..99d1e40c3 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -45,9 +45,9 @@ import (
 type RequestPool struct {
 	db       storage.Database
 	cfg      *config.SyncAPI
-	userAPI  userapi.UserInternalAPI
-	keyAPI   keyapi.KeyInternalAPI
-	rsAPI    roomserverAPI.RoomserverInternalAPI
+	userAPI  userapi.SyncUserAPI
+	keyAPI   keyapi.SyncKeyAPI
+	rsAPI    roomserverAPI.SyncRoomserverAPI
 	lastseen *sync.Map
 	presence *sync.Map
 	streams  *streams.Streams
@@ -62,8 +62,8 @@ type PresencePublisher interface {
 // NewRequestPool makes a new RequestPool
 func NewRequestPool(
 	db storage.Database, cfg *config.SyncAPI,
-	userAPI userapi.UserInternalAPI, keyAPI keyapi.KeyInternalAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	userAPI userapi.SyncUserAPI, keyAPI keyapi.SyncKeyAPI,
+	rsAPI roomserverAPI.SyncRoomserverAPI,
 	streams *streams.Streams, notifier *notifier.Notifier,
 	producer PresencePublisher,
 ) *RequestPool {
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index d8becb6ed..686e2044f 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -25,7 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/syncapi/consumers"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
@@ -40,10 +39,9 @@ import (
 // component.
 func AddPublicRoutes(
 	base *base.BaseDendrite,
-	userAPI userapi.UserInternalAPI,
-	rsAPI api.RoomserverInternalAPI,
-	keyAPI keyapi.KeyInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	userAPI userapi.SyncUserAPI,
+	rsAPI api.SyncRoomserverAPI,
+	keyAPI keyapi.SyncKeyAPI,
 ) {
 	cfg := &base.Cfg.SyncAPI
 
@@ -85,7 +83,7 @@ func AddPublicRoutes(
 
 	keyChangeConsumer := consumers.NewOutputKeyChangeEventConsumer(
 		base.ProcessContext, cfg, cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
-		js, keyAPI, rsAPI, syncDB, notifier,
+		js, rsAPI, syncDB, notifier,
 		streams.DeviceListStreamProvider,
 	)
 	if err = keyChangeConsumer.Start(); err != nil {
@@ -148,6 +146,6 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		base.PublicClientAPIMux, requestPool, syncDB, userAPI,
-		federation, rsAPI, cfg, lazyLoadCache,
+		rsAPI, cfg, lazyLoadCache,
 	)
 }
diff --git a/userapi/api/api.go b/userapi/api/api.go
index 6ab68fa08..6f00fe44f 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -31,7 +31,8 @@ type UserInternalAPI interface {
 	UserRegisterAPI
 	UserAccountAPI
 	UserThreePIDAPI
-	UserDeviceAPI
+	QueryAcccessTokenAPI
+	SyncUserAPI
 
 	InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error
 
@@ -42,15 +43,20 @@ type UserInternalAPI interface {
 	PerformPushRulesPut(ctx context.Context, req *PerformPushRulesPutRequest, res *struct{}) error
 
 	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse)
-	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
-	QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error
+
 	QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error
 	QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error
 	QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error
 	QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error
 }
 
-type UserDeviceAPI interface {
+type QueryAcccessTokenAPI interface {
+	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
+}
+
+type SyncUserAPI interface {
+	QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error
+	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
 	PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error
 	PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error
 	PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error

From 5c37f165ae5301ee1f41c1491bed7fcc7c439c40 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 5 May 2022 10:53:52 +0100
Subject: [PATCH 070/391] Errors from createdb are non-fatal (#2420)

As they are expected if the database already exists.
---
 test/db.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/db.go b/test/db.go
index fecae5d48..a1754cd08 100644
--- a/test/db.go
+++ b/test/db.go
@@ -53,8 +53,8 @@ func createLocalDB(t *testing.T, dbName string) {
 		createDB.Stderr = os.Stderr
 	}
 	err := createDB.Run()
-	if err != nil {
-		fatalError(t, "createLocalDB returned error: %s", err)
+	if err != nil && !Quiet {
+		fmt.Println("createLocalDB returned error:", err)
 	}
 }
 

From 658e82f8bcfc78d8489f9687e88fb712af0ea75f Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 5 May 2022 12:00:18 +0200
Subject: [PATCH 071/391] Don't use in-memory db for userapi tests (#2417)

* Don't use in-memory db

* Use WithAllDatabases where possible
---
 userapi/userapi_test.go | 205 +++++++++++++++++++++-------------------
 1 file changed, 109 insertions(+), 96 deletions(-)

diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index 64e23909a..e614765a2 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package userapi
+package userapi_test
 
 import (
 	"context"
@@ -23,15 +23,17 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	internalTest "github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/dendrite/userapi/inthttp"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/internal"
-	"github.com/matrix-org/dendrite/userapi/inthttp"
 	"github.com/matrix-org/dendrite/userapi/storage"
 )
 
@@ -43,16 +45,15 @@ type apiTestOpts struct {
 	loginTokenLifetime time.Duration
 }
 
-func MustMakeInternalAPI(t *testing.T, opts apiTestOpts) (api.UserInternalAPI, storage.Database) {
+func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType) (api.UserInternalAPI, storage.Database, func()) {
 	if opts.loginTokenLifetime == 0 {
 		opts.loginTokenLifetime = api.DefaultLoginTokenLifetime * time.Millisecond
 	}
-	dbopts := &config.DatabaseOptions{
-		ConnectionString:   "file::memory:",
-		MaxOpenConnections: 1,
-		MaxIdleConnections: 1,
-	}
-	accountDB, err := storage.NewUserAPIDatabase(nil, dbopts, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+
+	accountDB, err := storage.NewUserAPIDatabase(nil, &config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
 	if err != nil {
 		t.Fatalf("failed to create account DB: %s", err)
 	}
@@ -66,13 +67,15 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts) (api.UserInternalAPI, s
 	return &internal.UserInternalAPI{
 		DB:         accountDB,
 		ServerName: cfg.Matrix.ServerName,
-	}, accountDB
+	}, accountDB, close
 }
 
 func TestQueryProfile(t *testing.T) {
 	aliceAvatarURL := "mxc://example.com/alice"
 	aliceDisplayName := "Alice"
-	userAPI, accountDB := MustMakeInternalAPI(t, apiTestOpts{})
+	// only one DBType, since userapi.AddInternalRoutes complains about multiple prometheus counters added
+	userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, test.DBTypeSQLite)
+	defer close()
 	_, err := accountDB.CreateAccount(context.TODO(), "alice", "foobar", "", api.AccountTypeUser)
 	if err != nil {
 		t.Fatalf("failed to make account: %s", err)
@@ -131,8 +134,8 @@ func TestQueryProfile(t *testing.T) {
 
 	t.Run("HTTP API", func(t *testing.T) {
 		router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
-		AddInternalRoutes(router, userAPI)
-		apiURL, cancel := test.ListenAndServe(t, router, false)
+		userapi.AddInternalRoutes(router, userAPI)
+		apiURL, cancel := internalTest.ListenAndServe(t, router, false)
 		defer cancel()
 		httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{})
 		if err != nil {
@@ -149,110 +152,120 @@ func TestLoginToken(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("tokenLoginFlow", func(t *testing.T) {
-		userAPI, accountDB := MustMakeInternalAPI(t, apiTestOpts{})
+		test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+			userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType)
+			defer close()
+			_, err := accountDB.CreateAccount(ctx, "auser", "apassword", "", api.AccountTypeUser)
+			if err != nil {
+				t.Fatalf("failed to make account: %s", err)
+			}
 
-		_, err := accountDB.CreateAccount(ctx, "auser", "apassword", "", api.AccountTypeUser)
-		if err != nil {
-			t.Fatalf("failed to make account: %s", err)
-		}
+			t.Log("Creating a login token like the SSO callback would...")
 
-		t.Log("Creating a login token like the SSO callback would...")
+			creq := api.PerformLoginTokenCreationRequest{
+				Data: api.LoginTokenData{UserID: "@auser:example.com"},
+			}
+			var cresp api.PerformLoginTokenCreationResponse
+			if err := userAPI.PerformLoginTokenCreation(ctx, &creq, &cresp); err != nil {
+				t.Fatalf("PerformLoginTokenCreation failed: %v", err)
+			}
 
-		creq := api.PerformLoginTokenCreationRequest{
-			Data: api.LoginTokenData{UserID: "@auser:example.com"},
-		}
-		var cresp api.PerformLoginTokenCreationResponse
-		if err := userAPI.PerformLoginTokenCreation(ctx, &creq, &cresp); err != nil {
-			t.Fatalf("PerformLoginTokenCreation failed: %v", err)
-		}
+			if cresp.Metadata.Token == "" {
+				t.Errorf("PerformLoginTokenCreation Token: got %q, want non-empty", cresp.Metadata.Token)
+			}
+			if cresp.Metadata.Expiration.Before(time.Now()) {
+				t.Errorf("PerformLoginTokenCreation Expiration: got %v, want non-expired", cresp.Metadata.Expiration)
+			}
 
-		if cresp.Metadata.Token == "" {
-			t.Errorf("PerformLoginTokenCreation Token: got %q, want non-empty", cresp.Metadata.Token)
-		}
-		if cresp.Metadata.Expiration.Before(time.Now()) {
-			t.Errorf("PerformLoginTokenCreation Expiration: got %v, want non-expired", cresp.Metadata.Expiration)
-		}
+			t.Log("Querying the login token like /login with m.login.token would...")
 
-		t.Log("Querying the login token like /login with m.login.token would...")
+			qreq := api.QueryLoginTokenRequest{Token: cresp.Metadata.Token}
+			var qresp api.QueryLoginTokenResponse
+			if err := userAPI.QueryLoginToken(ctx, &qreq, &qresp); err != nil {
+				t.Fatalf("QueryLoginToken failed: %v", err)
+			}
 
-		qreq := api.QueryLoginTokenRequest{Token: cresp.Metadata.Token}
-		var qresp api.QueryLoginTokenResponse
-		if err := userAPI.QueryLoginToken(ctx, &qreq, &qresp); err != nil {
-			t.Fatalf("QueryLoginToken failed: %v", err)
-		}
+			if qresp.Data == nil {
+				t.Errorf("QueryLoginToken Data: got %v, want non-nil", qresp.Data)
+			} else if want := "@auser:example.com"; qresp.Data.UserID != want {
+				t.Errorf("QueryLoginToken UserID: got %q, want %q", qresp.Data.UserID, want)
+			}
 
-		if qresp.Data == nil {
-			t.Errorf("QueryLoginToken Data: got %v, want non-nil", qresp.Data)
-		} else if want := "@auser:example.com"; qresp.Data.UserID != want {
-			t.Errorf("QueryLoginToken UserID: got %q, want %q", qresp.Data.UserID, want)
-		}
+			t.Log("Deleting the login token like /login with m.login.token would...")
 
-		t.Log("Deleting the login token like /login with m.login.token would...")
-
-		dreq := api.PerformLoginTokenDeletionRequest{Token: cresp.Metadata.Token}
-		var dresp api.PerformLoginTokenDeletionResponse
-		if err := userAPI.PerformLoginTokenDeletion(ctx, &dreq, &dresp); err != nil {
-			t.Fatalf("PerformLoginTokenDeletion failed: %v", err)
-		}
+			dreq := api.PerformLoginTokenDeletionRequest{Token: cresp.Metadata.Token}
+			var dresp api.PerformLoginTokenDeletionResponse
+			if err := userAPI.PerformLoginTokenDeletion(ctx, &dreq, &dresp); err != nil {
+				t.Fatalf("PerformLoginTokenDeletion failed: %v", err)
+			}
+		})
 	})
 
 	t.Run("expiredTokenIsNotReturned", func(t *testing.T) {
-		userAPI, _ := MustMakeInternalAPI(t, apiTestOpts{loginTokenLifetime: -1 * time.Second})
+		test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+			userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{loginTokenLifetime: -1 * time.Second}, dbType)
+			defer close()
 
-		creq := api.PerformLoginTokenCreationRequest{
-			Data: api.LoginTokenData{UserID: "@auser:example.com"},
-		}
-		var cresp api.PerformLoginTokenCreationResponse
-		if err := userAPI.PerformLoginTokenCreation(ctx, &creq, &cresp); err != nil {
-			t.Fatalf("PerformLoginTokenCreation failed: %v", err)
-		}
+			creq := api.PerformLoginTokenCreationRequest{
+				Data: api.LoginTokenData{UserID: "@auser:example.com"},
+			}
+			var cresp api.PerformLoginTokenCreationResponse
+			if err := userAPI.PerformLoginTokenCreation(ctx, &creq, &cresp); err != nil {
+				t.Fatalf("PerformLoginTokenCreation failed: %v", err)
+			}
 
-		qreq := api.QueryLoginTokenRequest{Token: cresp.Metadata.Token}
-		var qresp api.QueryLoginTokenResponse
-		if err := userAPI.QueryLoginToken(ctx, &qreq, &qresp); err != nil {
-			t.Fatalf("QueryLoginToken failed: %v", err)
-		}
+			qreq := api.QueryLoginTokenRequest{Token: cresp.Metadata.Token}
+			var qresp api.QueryLoginTokenResponse
+			if err := userAPI.QueryLoginToken(ctx, &qreq, &qresp); err != nil {
+				t.Fatalf("QueryLoginToken failed: %v", err)
+			}
 
-		if qresp.Data != nil {
-			t.Errorf("QueryLoginToken Data: got %v, want nil", qresp.Data)
-		}
+			if qresp.Data != nil {
+				t.Errorf("QueryLoginToken Data: got %v, want nil", qresp.Data)
+			}
+		})
 	})
 
 	t.Run("deleteWorks", func(t *testing.T) {
-		userAPI, _ := MustMakeInternalAPI(t, apiTestOpts{})
+		test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+			userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType)
+			defer close()
 
-		creq := api.PerformLoginTokenCreationRequest{
-			Data: api.LoginTokenData{UserID: "@auser:example.com"},
-		}
-		var cresp api.PerformLoginTokenCreationResponse
-		if err := userAPI.PerformLoginTokenCreation(ctx, &creq, &cresp); err != nil {
-			t.Fatalf("PerformLoginTokenCreation failed: %v", err)
-		}
+			creq := api.PerformLoginTokenCreationRequest{
+				Data: api.LoginTokenData{UserID: "@auser:example.com"},
+			}
+			var cresp api.PerformLoginTokenCreationResponse
+			if err := userAPI.PerformLoginTokenCreation(ctx, &creq, &cresp); err != nil {
+				t.Fatalf("PerformLoginTokenCreation failed: %v", err)
+			}
 
-		dreq := api.PerformLoginTokenDeletionRequest{Token: cresp.Metadata.Token}
-		var dresp api.PerformLoginTokenDeletionResponse
-		if err := userAPI.PerformLoginTokenDeletion(ctx, &dreq, &dresp); err != nil {
-			t.Fatalf("PerformLoginTokenDeletion failed: %v", err)
-		}
+			dreq := api.PerformLoginTokenDeletionRequest{Token: cresp.Metadata.Token}
+			var dresp api.PerformLoginTokenDeletionResponse
+			if err := userAPI.PerformLoginTokenDeletion(ctx, &dreq, &dresp); err != nil {
+				t.Fatalf("PerformLoginTokenDeletion failed: %v", err)
+			}
 
-		qreq := api.QueryLoginTokenRequest{Token: cresp.Metadata.Token}
-		var qresp api.QueryLoginTokenResponse
-		if err := userAPI.QueryLoginToken(ctx, &qreq, &qresp); err != nil {
-			t.Fatalf("QueryLoginToken failed: %v", err)
-		}
+			qreq := api.QueryLoginTokenRequest{Token: cresp.Metadata.Token}
+			var qresp api.QueryLoginTokenResponse
+			if err := userAPI.QueryLoginToken(ctx, &qreq, &qresp); err != nil {
+				t.Fatalf("QueryLoginToken failed: %v", err)
+			}
 
-		if qresp.Data != nil {
-			t.Errorf("QueryLoginToken Data: got %v, want nil", qresp.Data)
-		}
+			if qresp.Data != nil {
+				t.Errorf("QueryLoginToken Data: got %v, want nil", qresp.Data)
+			}
+		})
 	})
 
 	t.Run("deleteUnknownIsNoOp", func(t *testing.T) {
-		userAPI, _ := MustMakeInternalAPI(t, apiTestOpts{})
-
-		dreq := api.PerformLoginTokenDeletionRequest{Token: "non-existent token"}
-		var dresp api.PerformLoginTokenDeletionResponse
-		if err := userAPI.PerformLoginTokenDeletion(ctx, &dreq, &dresp); err != nil {
-			t.Fatalf("PerformLoginTokenDeletion failed: %v", err)
-		}
+		test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+			userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType)
+			defer close()
+			dreq := api.PerformLoginTokenDeletionRequest{Token: "non-existent token"}
+			var dresp api.PerformLoginTokenDeletionResponse
+			if err := userAPI.PerformLoginTokenDeletion(ctx, &dreq, &dresp); err != nil {
+				t.Fatalf("PerformLoginTokenDeletion failed: %v", err)
+			}
+		})
 	})
 }

From 1bfe87aa5614d68cbb0cad127b375048cdc70ca9 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 5 May 2022 12:01:28 +0200
Subject: [PATCH 072/391] Fix user already joined when using server notices
 (#2364)

---
 clientapi/routing/server_notices.go | 49 +++++++++++++----------------
 1 file changed, 22 insertions(+), 27 deletions(-)

diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go
index eec3d7e38..47b0da7bb 100644
--- a/clientapi/routing/server_notices.go
+++ b/clientapi/routing/server_notices.go
@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
@@ -95,29 +96,16 @@ func SendServerNotice(
 	// get rooms for specified user
 	allUserRooms := []string{}
 	userRooms := api.QueryRoomsForUserResponse{}
-	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-		UserID:         r.UserID,
-		WantMembership: "join",
-	}, &userRooms); err != nil {
-		return util.ErrorResponse(err)
+	// Get rooms the user is either joined, invited or has left.
+	for _, membership := range []string{"join", "invite", "leave"} {
+		if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+			UserID:         r.UserID,
+			WantMembership: membership,
+		}, &userRooms); err != nil {
+			return util.ErrorResponse(err)
+		}
+		allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
 	}
-	allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
-	// get invites for specified user
-	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-		UserID:         r.UserID,
-		WantMembership: "invite",
-	}, &userRooms); err != nil {
-		return util.ErrorResponse(err)
-	}
-	allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
-	// get left rooms for specified user
-	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
-		UserID:         r.UserID,
-		WantMembership: "leave",
-	}, &userRooms); err != nil {
-		return util.ErrorResponse(err)
-	}
-	allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
 
 	// get rooms of the sender
 	senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName)
@@ -145,7 +133,7 @@ func SendServerNotice(
 
 	var (
 		roomID      string
-		roomVersion = gomatrixserverlib.RoomVersionV6
+		roomVersion = version.DefaultRoomVersion()
 	)
 
 	// create a new room for the user
@@ -194,14 +182,21 @@ func SendServerNotice(
 			// if we didn't get a createRoomResponse, we probably received an error, so return that.
 			return roomRes
 		}
-
 	} else {
 		// we've found a room in common, check the membership
 		roomID = commonRooms[0]
-		// re-invite the user
-		res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
+		membershipRes := api.QueryMembershipForUserResponse{}
+		err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: r.UserID, RoomID: roomID}, &membershipRes)
 		if err != nil {
-			return res
+			util.GetLogger(ctx).WithError(err).Error("unable to query membership for user")
+			return jsonerror.InternalServerError()
+		}
+		if !membershipRes.IsInRoom {
+			// re-invite the user
+			res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
+			if err != nil {
+				return res
+			}
 		}
 	}
 

From d9e71b93b68efb57582d02448883b8a1259205e8 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 5 May 2022 11:33:16 +0100
Subject: [PATCH 073/391] Use `gomatrixserverlib.Client` instead of
 `http.Client` (#2421)

* Update to matrix-org/gomatrixserverlib#303

* Use `gomatrixserverlib.Client` for phone-home stats

* Use `gomatrixserverlib.Client` for push notifications

* Use `gomatrixserverlib.Client` for appservices

* Use `gomatrixserverlib.Client` for three-PID invites
---
 appservice/appservice.go                    | 18 +++++++-----------
 appservice/query/query.go                   |  8 ++++----
 appservice/workers/transaction_scheduler.go |  8 ++++----
 clientapi/threepid/invites.go               |  4 ++--
 go.mod                                      |  4 ++--
 go.sum                                      |  4 ++--
 internal/pushgateway/client.go              | 21 +++++++++------------
 userapi/util/phonehomestats.go              | 12 +++++-------
 8 files changed, 35 insertions(+), 44 deletions(-)

diff --git a/appservice/appservice.go b/appservice/appservice.go
index 0db2c1009..ac4ed967e 100644
--- a/appservice/appservice.go
+++ b/appservice/appservice.go
@@ -16,8 +16,6 @@ package appservice
 
 import (
 	"context"
-	"crypto/tls"
-	"net/http"
 	"sync"
 	"time"
 
@@ -36,6 +34,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddInternalRoutes registers HTTP handlers for internal API calls
@@ -50,15 +49,12 @@ func NewInternalAPI(
 	userAPI userapi.UserInternalAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceQueryAPI {
-	client := &http.Client{
-		Timeout: time.Second * 30,
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
-			},
-		},
-	}
+	client := gomatrixserverlib.NewClient(
+		gomatrixserverlib.WithTimeout(time.Second*30),
+		gomatrixserverlib.WithKeepAlives(false),
+		gomatrixserverlib.WithSkipVerify(base.Cfg.AppServiceAPI.DisableTLSValidation),
+	)
+
 	js, _ := jetstream.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 
 	// Create a connection to the appservice postgres DB
diff --git a/appservice/query/query.go b/appservice/query/query.go
index dacd3caa8..b7b0b335a 100644
--- a/appservice/query/query.go
+++ b/appservice/query/query.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
 	opentracing "github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 )
@@ -32,7 +33,7 @@ const userIDExistsPath = "/users/"
 
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
-	HTTPClient *http.Client
+	HTTPClient *gomatrixserverlib.Client
 	Cfg        *config.Dendrite
 }
 
@@ -64,9 +65,8 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
-			req = req.WithContext(ctx)
 
-			resp, err := a.HTTPClient.Do(req)
+			resp, err := a.HTTPClient.DoHTTPRequest(ctx, req)
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -130,7 +130,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
-			resp, err := a.HTTPClient.Do(req.WithContext(ctx))
+			resp, err := a.HTTPClient.DoHTTPRequest(ctx, req)
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go
index 4dab00bd7..47d447c2c 100644
--- a/appservice/workers/transaction_scheduler.go
+++ b/appservice/workers/transaction_scheduler.go
@@ -42,7 +42,7 @@ var (
 // size), then send that off to the AS's /transactions/{txnID} endpoint. It also
 // handles exponentially backing off in case the AS isn't currently available.
 func SetupTransactionWorkers(
-	client *http.Client,
+	client *gomatrixserverlib.Client,
 	appserviceDB storage.Database,
 	workerStates []types.ApplicationServiceWorkerState,
 ) error {
@@ -58,7 +58,7 @@ func SetupTransactionWorkers(
 
 // worker is a goroutine that sends any queued events to the application service
 // it is given.
-func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
+func worker(client *gomatrixserverlib.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
 	log.WithFields(log.Fields{
 		"appservice": ws.AppService.ID,
 	}).Info("Starting application service")
@@ -200,7 +200,7 @@ func createTransaction(
 // send sends events to an application service. Returns an error if an OK was not
 // received back from the application service or the request timed out.
 func send(
-	client *http.Client,
+	client *gomatrixserverlib.Client,
 	appservice config.ApplicationService,
 	txnID int,
 	transaction []byte,
@@ -213,7 +213,7 @@ func send(
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.Do(req)
+	resp, err := client.DoHTTPRequest(context.TODO(), req)
 	if err != nil {
 		return err
 	}
diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go
index 6b750199b..eee6992f8 100644
--- a/clientapi/threepid/invites.go
+++ b/clientapi/threepid/invites.go
@@ -231,7 +231,7 @@ func queryIDServerStoreInvite(
 		profile = &authtypes.Profile{}
 	}
 
-	client := http.Client{}
+	client := gomatrixserverlib.NewClient()
 
 	data := url.Values{}
 	data.Add("medium", body.Medium)
@@ -253,7 +253,7 @@ func queryIDServerStoreInvite(
 	}
 
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	resp, err := client.Do(req.WithContext(ctx))
+	resp, err := client.DoHTTPRequest(ctx, req)
 	if err != nil {
 		return nil, err
 	}
diff --git a/go.mod b/go.mod
index a7caadfb5..56f5dcbbb 100644
--- a/go.mod
+++ b/go.mod
@@ -25,12 +25,12 @@ require (
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
-	github.com/kardianos/minwinsvc v1.0.0 // indirect
+	github.com/kardianos/minwinsvc v1.0.0
 	github.com/lib/pq v1.10.5
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220505092512-c4ceb4751ac2
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index f8daca79e..fccda40c4 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f h1:MZrl4TgTnlaOn2Cu9gJCoJ3oyW5mT4/3QIZGgZXzKl4=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220408160933-cf558306b56f/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220505092512-c4ceb4751ac2 h1:5/Y4BpiMk1D/l/HkJz8Ng8bLBz1BHwV6V4e+yMNySzk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220505092512-c4ceb4751ac2/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/internal/pushgateway/client.go b/internal/pushgateway/client.go
index 49907cee8..231327a1e 100644
--- a/internal/pushgateway/client.go
+++ b/internal/pushgateway/client.go
@@ -3,31 +3,28 @@ package pushgateway
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/opentracing/opentracing-go"
 )
 
 type httpClient struct {
-	hc *http.Client
+	hc *gomatrixserverlib.Client
 }
 
 // NewHTTPClient creates a new Push Gateway client.
 func NewHTTPClient(disableTLSValidation bool) Client {
-	hc := &http.Client{
-		Timeout: 30 * time.Second,
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: disableTLSValidation,
-			},
-		},
+	return &httpClient{
+		hc: gomatrixserverlib.NewClient(
+			gomatrixserverlib.WithTimeout(time.Second*30),
+			gomatrixserverlib.WithKeepAlives(false),
+			gomatrixserverlib.WithSkipVerify(disableTLSValidation),
+		),
 	}
-	return &httpClient{hc: hc}
 }
 
 func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest, resp *NotifyResponse) error {
@@ -44,7 +41,7 @@ func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest,
 	}
 	hreq.Header.Set("Content-Type", "application/json")
 
-	hresp, err := h.hc.Do(hreq)
+	hresp, err := h.hc.DoHTTPRequest(ctx, hreq)
 	if err != nil {
 		return err
 	}
diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go
index ad93a50e3..e24daba6b 100644
--- a/userapi/util/phonehomestats.go
+++ b/userapi/util/phonehomestats.go
@@ -39,7 +39,7 @@ type phoneHomeStats struct {
 	cfg        *config.Dendrite
 	db         storage.Statistics
 	isMonolith bool
-	client     *http.Client
+	client     *gomatrixserverlib.Client
 }
 
 type timestampToRUUsage struct {
@@ -55,10 +55,9 @@ func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB
 		cfg:        cfg,
 		db:         statsDB,
 		isMonolith: cfg.IsMonolith,
-		client: &http.Client{
-			Timeout:   time.Second * 30,
-			Transport: http.DefaultTransport,
-		},
+		client: gomatrixserverlib.NewClient(
+			gomatrixserverlib.WithTimeout(time.Second * 30),
+		),
 	}
 
 	// start initial run after 5min
@@ -152,8 +151,7 @@ func (p *phoneHomeStats) collect() {
 	}
 	request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
 
-	_, err = p.client.Do(request)
-	if err != nil {
+	if _, err = p.client.DoHTTPRequest(ctx, request); err != nil {
 		logrus.WithError(err).Error("unable to send anonymous stats")
 		return
 	}

From 506de4bb3d69fef27e60d09d06712a72e588a198 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 5 May 2022 13:17:38 +0100
Subject: [PATCH 074/391] Define component interfaces based on consumers (1/2)
 (#2423)

* Specify interfaces used by appservice, do half of clientapi

* convert more deps of clientapi to finer-grained interfaces

* Convert mediaapi and rest of clientapi

* Somehow this got missed
---
 appservice/api/query.go                 |   2 +-
 appservice/appservice.go                |   6 +-
 appservice/consumers/roomserver.go      |   4 +-
 clientapi/auth/login.go                 |   2 +-
 clientapi/auth/login_test.go            |   5 +-
 clientapi/auth/user_interactive.go      |   2 +-
 clientapi/auth/user_interactive_test.go |   4 +-
 clientapi/clientapi.go                  |   8 +-
 clientapi/producers/syncapi.go          |   2 +-
 clientapi/routing/account_data.go       |   6 +-
 clientapi/routing/admin.go              |   2 +-
 clientapi/routing/admin_whois.go        |   2 +-
 clientapi/routing/aliases.go            |   2 +-
 clientapi/routing/capabilities.go       |   2 +-
 clientapi/routing/createroom.go         |  30 ++--
 clientapi/routing/deactivate.go         |   2 +-
 clientapi/routing/device.go             |  10 +-
 clientapi/routing/directory.go          |  12 +-
 clientapi/routing/directory_public.go   |   6 +-
 clientapi/routing/getevent.go           |   5 +-
 clientapi/routing/joinroom.go           |   4 +-
 clientapi/routing/key_backup.go         |  12 +-
 clientapi/routing/key_crosssigning.go   |   6 +-
 clientapi/routing/keys.go               |   6 +-
 clientapi/routing/leaveroom.go          |   2 +-
 clientapi/routing/login.go              |   4 +-
 clientapi/routing/logout.go             |   4 +-
 clientapi/routing/membership.go         |  74 ++++-----
 clientapi/routing/memberships.go        |   4 +-
 clientapi/routing/notification.go       |   2 +-
 clientapi/routing/openid.go             |   2 +-
 clientapi/routing/password.go           |   2 +-
 clientapi/routing/peekroom.go           |   4 +-
 clientapi/routing/profile.go            |  18 +--
 clientapi/routing/pusher.go             |   4 +-
 clientapi/routing/pushrules.go          |  20 +--
 clientapi/routing/redaction.go          |   2 +-
 clientapi/routing/register.go           |  16 +-
 clientapi/routing/room_tagging.go       |  10 +-
 clientapi/routing/routing.go            |  12 +-
 clientapi/routing/sendevent.go          |   4 +-
 clientapi/routing/sendtyping.go         |   2 +-
 clientapi/routing/server_notices.go     |   6 +-
 clientapi/routing/state.go              |   4 +-
 clientapi/routing/threepid.go           |   8 +-
 clientapi/routing/upgrade_room.go       |   4 +-
 clientapi/routing/userdirectory.go      |  14 +-
 clientapi/threepid/invites.go           |  14 +-
 federationapi/api/api.go                |   5 +
 federationapi/routing/invite.go         |  49 +++---
 internal/eventutil/events.go            |   4 +-
 keyserver/api/api.go                    |  18 ++-
 mediaapi/mediaapi.go                    |   2 +-
 mediaapi/routing/routing.go             |   2 +-
 roomserver/api/api.go                   | 190 +++++++++++++++---------
 roomserver/api/wrapper.go               |  42 +-----
 userapi/api/api.go                      |  96 ++++++------
 57 files changed, 414 insertions(+), 372 deletions(-)

diff --git a/appservice/api/query.go b/appservice/api/query.go
index cf25a9616..6db8be85b 100644
--- a/appservice/api/query.go
+++ b/appservice/api/query.go
@@ -84,7 +84,7 @@ func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceQueryAPI,
-	profileAPI userapi.UserProfileAPI,
+	profileAPI userapi.ClientUserAPI,
 ) (*authtypes.Profile, error) {
 	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
diff --git a/appservice/appservice.go b/appservice/appservice.go
index ac4ed967e..e026a7875 100644
--- a/appservice/appservice.go
+++ b/appservice/appservice.go
@@ -46,8 +46,8 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQuer
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite,
-	userAPI userapi.UserInternalAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	userAPI userapi.AppserviceUserAPI,
+	rsAPI roomserverAPI.AppserviceRoomserverAPI,
 ) appserviceAPI.AppServiceQueryAPI {
 	client := gomatrixserverlib.NewClient(
 		gomatrixserverlib.WithTimeout(time.Second*30),
@@ -113,7 +113,7 @@ func NewInternalAPI(
 // `sender_localpart` field of each application service if it doesn't
 // exist already
 func generateAppServiceAccount(
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.AppserviceUserAPI,
 	as config.ApplicationService,
 ) error {
 	var accRes userapi.PerformAccountCreationResponse
diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index 31e05caa0..e406e88a7 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -37,7 +37,7 @@ type OutputRoomEventConsumer struct {
 	durable      string
 	topic        string
 	asDB         storage.Database
-	rsAPI        api.RoomserverInternalAPI
+	rsAPI        api.AppserviceRoomserverAPI
 	serverName   string
 	workerStates []types.ApplicationServiceWorkerState
 }
@@ -49,7 +49,7 @@ func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	js nats.JetStreamContext,
 	appserviceDB storage.Database,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.AppserviceRoomserverAPI,
 	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go
index 020731c9f..5f51c662a 100644
--- a/clientapi/auth/login.go
+++ b/clientapi/auth/login.go
@@ -33,7 +33,7 @@ import (
 // called after authorization has completed, with the result of the authorization.
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
-func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserAccountAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
 	reqBytes, err := ioutil.ReadAll(r)
 	if err != nil {
 		err := &util.JSONResponse{
diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go
index d401469c1..5085f0170 100644
--- a/clientapi/auth/login_test.go
+++ b/clientapi/auth/login_test.go
@@ -160,7 +160,6 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 
 type fakeUserInternalAPI struct {
 	UserInternalAPIForLogin
-	uapi.UserAccountAPI
 	DeletedTokens []string
 }
 
@@ -179,6 +178,10 @@ func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, re
 	return nil
 }
 
+func (ua *fakeUserInternalAPI) PerformLoginTokenCreation(ctx context.Context, req *uapi.PerformLoginTokenCreationRequest, res *uapi.PerformLoginTokenCreationResponse) error {
+	return nil
+}
+
 func (*fakeUserInternalAPI) QueryLoginToken(ctx context.Context, req *uapi.QueryLoginTokenRequest, res *uapi.QueryLoginTokenResponse) error {
 	if req.Token == "invalidtoken" {
 		return nil
diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go
index 22c430f97..6caf7dcdc 100644
--- a/clientapi/auth/user_interactive.go
+++ b/clientapi/auth/user_interactive.go
@@ -110,7 +110,7 @@ type UserInteractive struct {
 	Sessions map[string][]string
 }
 
-func NewUserInteractive(userAccountAPI api.UserAccountAPI, cfg *config.ClientAPI) *UserInteractive {
+func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
 		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
 		Config:               cfg,
diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go
index a4b4587a3..262e48103 100644
--- a/clientapi/auth/user_interactive_test.go
+++ b/clientapi/auth/user_interactive_test.go
@@ -24,9 +24,7 @@ var (
 	}
 )
 
-type fakeAccountDatabase struct {
-	api.UserAccountAPI
-}
+type fakeAccountDatabase struct{}
 
 func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error {
 	return nil
diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index 0d16e4c14..957d082a6 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -33,13 +33,13 @@ import (
 func AddPublicRoutes(
 	base *base.BaseDendrite,
 	federation *gomatrixserverlib.FederationClient,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
-	fsAPI federationAPI.FederationInternalAPI,
-	userAPI userapi.UserInternalAPI,
+	fsAPI federationAPI.ClientFederationAPI,
+	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.UserDirectoryProvider,
-	keyAPI keyserverAPI.KeyInternalAPI,
+	keyAPI keyserverAPI.ClientKeyAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 ) {
 	cfg := &base.Cfg.ClientAPI
diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go
index 187e3412d..48b1ae88d 100644
--- a/clientapi/producers/syncapi.go
+++ b/clientapi/producers/syncapi.go
@@ -38,7 +38,7 @@ type SyncAPIProducer struct {
 	TopicPresenceEvent     string
 	JetStream              nats.JetStreamContext
 	ServerName             gomatrixserverlib.ServerName
-	UserAPI                userapi.UserInternalAPI
+	UserAPI                userapi.ClientUserAPI
 }
 
 // SendData sends account data to the sync API server
diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go
index d0dd3ab8d..a5a3014ab 100644
--- a/clientapi/routing/account_data.go
+++ b/clientapi/routing/account_data.go
@@ -33,7 +33,7 @@ import (
 
 // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type}
 func GetAccountData(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 	userID string, roomID string, dataType string,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -76,7 +76,7 @@ func GetAccountData(
 
 // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
 func SaveAccountData(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -152,7 +152,7 @@ type fullyReadEvent struct {
 // SaveReadMarker implements POST /rooms/{roomId}/read_markers
 func SaveReadMarker(
 	req *http.Request,
-	userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
+	userAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
 ) util.JSONResponse {
 	// Verify that the user is a member of this room
diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index 31e431c78..125b3847d 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -11,7 +11,7 @@ import (
 	"github.com/matrix-org/util"
 )
 
-func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.RoomserverInternalAPI) util.JSONResponse {
+func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	if device.AccountType != userapi.AccountTypeAdmin {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
diff --git a/clientapi/routing/admin_whois.go b/clientapi/routing/admin_whois.go
index 87bb79366..f1cbd3467 100644
--- a/clientapi/routing/admin_whois.go
+++ b/clientapi/routing/admin_whois.go
@@ -44,7 +44,7 @@ type connectionInfo struct {
 
 // GetAdminWhois implements GET /admin/whois/{userId}
 func GetAdminWhois(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 	userID string,
 ) util.JSONResponse {
 	allowed := device.AccountType == api.AccountTypeAdmin || userID == device.UserID
diff --git a/clientapi/routing/aliases.go b/clientapi/routing/aliases.go
index 8c4830532..504d60265 100644
--- a/clientapi/routing/aliases.go
+++ b/clientapi/routing/aliases.go
@@ -28,7 +28,7 @@ import (
 
 // GetAliases implements GET /_matrix/client/r0/rooms/{roomId}/aliases
 func GetAliases(
-	req *http.Request, rsAPI api.RoomserverInternalAPI, device *userapi.Device, roomID string,
+	req *http.Request, rsAPI api.ClientRoomserverAPI, device *userapi.Device, roomID string,
 ) util.JSONResponse {
 	stateTuple := gomatrixserverlib.StateKeyTuple{
 		EventType: gomatrixserverlib.MRoomHistoryVisibility,
diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go
index 72668fa5a..b7d47e916 100644
--- a/clientapi/routing/capabilities.go
+++ b/clientapi/routing/capabilities.go
@@ -26,7 +26,7 @@ import (
 // GetCapabilities returns information about the server's supported feature set
 // and other relevant capabilities to an authenticated user.
 func GetCapabilities(
-	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
+	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
 ) util.JSONResponse {
 	roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
 	roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}
diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 4976b3e50..a21abb0eb 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -137,7 +137,7 @@ type fledglingEvent struct {
 func CreateRoom(
 	req *http.Request, device *api.Device,
 	cfg *config.ClientAPI,
-	profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
+	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	var r createRoomRequest
@@ -164,7 +164,7 @@ func createRoom(
 	ctx context.Context,
 	r createRoomRequest, device *api.Device,
 	cfg *config.ClientAPI,
-	profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
+	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	evTime time.Time,
 ) util.JSONResponse {
@@ -531,25 +531,23 @@ func createRoom(
 				gomatrixserverlib.NewInviteV2StrippedState(inviteEvent.Event),
 			)
 			// Send the invite event to the roomserver.
-			err = roomserverAPI.SendInvite(
-				ctx,
-				rsAPI,
-				inviteEvent.Headered(roomVersion),
-				inviteStrippedState,   // invite room state
-				cfg.Matrix.ServerName, // send as server
-				nil,                   // transaction ID
-			)
-			switch e := err.(type) {
-			case *roomserverAPI.PerformError:
-				return e.JSONResponse()
-			case nil:
-			default:
-				util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInvite failed")
+			var inviteRes roomserverAPI.PerformInviteResponse
+			event := inviteEvent.Headered(roomVersion)
+			if err := rsAPI.PerformInvite(ctx, &roomserverAPI.PerformInviteRequest{
+				Event:           event,
+				InviteRoomState: inviteStrippedState,
+				RoomVersion:     event.RoomVersion,
+				SendAsServer:    string(cfg.Matrix.ServerName),
+			}, &inviteRes); err != nil {
+				util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
 				return util.JSONResponse{
 					Code: http.StatusInternalServerError,
 					JSON: jsonerror.InternalServerError(),
 				}
 			}
+			if inviteRes.Error != nil {
+				return inviteRes.Error.JSONResponse()
+			}
 		}
 	}
 
diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go
index da1b6dcf9..c8aa6a3bc 100644
--- a/clientapi/routing/deactivate.go
+++ b/clientapi/routing/deactivate.go
@@ -15,7 +15,7 @@ import (
 func Deactivate(
 	req *http.Request,
 	userInteractiveAuth *auth.UserInteractive,
-	accountAPI api.UserAccountAPI,
+	accountAPI api.ClientUserAPI,
 	deviceAPI *api.Device,
 ) util.JSONResponse {
 	ctx := req.Context()
diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go
index 161bc2731..bb1cf47bd 100644
--- a/clientapi/routing/device.go
+++ b/clientapi/routing/device.go
@@ -50,7 +50,7 @@ type devicesDeleteJSON struct {
 
 // GetDeviceByID handles /devices/{deviceID}
 func GetDeviceByID(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	var queryRes api.QueryDevicesResponse
@@ -88,7 +88,7 @@ func GetDeviceByID(
 
 // GetDevicesByLocalpart handles /devices
 func GetDevicesByLocalpart(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	var queryRes api.QueryDevicesResponse
 	err := userAPI.QueryDevices(req.Context(), &api.QueryDevicesRequest{
@@ -118,7 +118,7 @@ func GetDevicesByLocalpart(
 
 // UpdateDeviceByID handles PUT on /devices/{deviceID}
 func UpdateDeviceByID(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 
@@ -161,7 +161,7 @@ func UpdateDeviceByID(
 
 // DeleteDeviceById handles DELETE requests to /devices/{deviceId}
 func DeleteDeviceById(
-	req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device,
 	deviceID string,
 ) util.JSONResponse {
 	var (
@@ -242,7 +242,7 @@ func DeleteDeviceById(
 
 // DeleteDevices handles POST requests to /delete_devices
 func DeleteDevices(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	ctx := req.Context()
 	payload := devicesDeleteJSON{}
diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go
index ac355b5d4..53ba3f190 100644
--- a/clientapi/routing/directory.go
+++ b/clientapi/routing/directory.go
@@ -46,8 +46,8 @@ func DirectoryRoom(
 	roomAlias string,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
-	fedSenderAPI federationAPI.FederationInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+	fedSenderAPI federationAPI.ClientFederationAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
 	if err != nil {
@@ -117,7 +117,7 @@ func SetLocalAlias(
 	device *userapi.Device,
 	alias string,
 	cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
 	if err != nil {
@@ -199,7 +199,7 @@ func RemoveLocalAlias(
 	req *http.Request,
 	device *userapi.Device,
 	alias string,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 ) util.JSONResponse {
 	queryReq := roomserverAPI.RemoveRoomAliasRequest{
 		Alias:  alias,
@@ -237,7 +237,7 @@ type roomVisibility struct {
 
 // GetVisibility implements GET /directory/list/room/{roomID}
 func GetVisibility(
-	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
+	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
 	var res roomserverAPI.QueryPublishedRoomsResponse
@@ -265,7 +265,7 @@ func GetVisibility(
 // SetVisibility implements PUT /directory/list/room/{roomID}
 // TODO: Allow admin users to edit the room visibility
 func SetVisibility(
-	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, dev *userapi.Device,
+	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device,
 	roomID string,
 ) util.JSONResponse {
 	resErr := checkMemberInRoom(req.Context(), rsAPI, dev.UserID, roomID)
diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go
index 0dacfced5..c3e6141b2 100644
--- a/clientapi/routing/directory_public.go
+++ b/clientapi/routing/directory_public.go
@@ -50,7 +50,7 @@ type filter struct {
 
 // GetPostPublicRooms implements GET and POST /publicRooms
 func GetPostPublicRooms(
-	req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
+	req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.ClientAPI,
@@ -91,7 +91,7 @@ func GetPostPublicRooms(
 }
 
 func publicRooms(
-	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
+	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
 ) (*gomatrixserverlib.RespPublicRooms, error) {
 
 	response := gomatrixserverlib.RespPublicRooms{
@@ -229,7 +229,7 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (
 }
 
 func refreshPublicRoomCache(
-	ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
+	ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
 ) []gomatrixserverlib.PublicRoom {
 	cacheMu.Lock()
 	defer cacheMu.Unlock()
diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go
index 36f3ee9e3..7f5842800 100644
--- a/clientapi/routing/getevent.go
+++ b/clientapi/routing/getevent.go
@@ -31,7 +31,6 @@ type getEventRequest struct {
 	roomID         string
 	eventID        string
 	cfg            *config.ClientAPI
-	federation     *gomatrixserverlib.FederationClient
 	requestedEvent *gomatrixserverlib.Event
 }
 
@@ -43,8 +42,7 @@ func GetEvent(
 	roomID string,
 	eventID string,
 	cfg *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	eventsReq := api.QueryEventsByIDRequest{
 		EventIDs: []string{eventID},
@@ -72,7 +70,6 @@ func GetEvent(
 		roomID:         roomID,
 		eventID:        eventID,
 		cfg:            cfg,
-		federation:     federation,
 		requestedEvent: requestedEvent,
 	}
 
diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go
index dc15f4bda..4e6acebc3 100644
--- a/clientapi/routing/joinroom.go
+++ b/clientapi/routing/joinroom.go
@@ -29,8 +29,8 @@ import (
 func JoinRoomByIDOrAlias(
 	req *http.Request,
 	device *api.Device,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
-	profileAPI api.UserProfileAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+	profileAPI api.ClientUserAPI,
 	roomIDOrAlias string,
 ) util.JSONResponse {
 	// Prepare to ask the roomserver to perform the room join.
diff --git a/clientapi/routing/key_backup.go b/clientapi/routing/key_backup.go
index 9d2ff87fd..28c80415b 100644
--- a/clientapi/routing/key_backup.go
+++ b/clientapi/routing/key_backup.go
@@ -55,7 +55,7 @@ type keyBackupSessionResponse struct {
 
 // Create a new key backup. Request must contain a `keyBackupVersion`. Returns a `keyBackupVersionCreateResponse`.
 // Implements  POST /_matrix/client/r0/room_keys/version
-func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device) util.JSONResponse {
+func CreateKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device) util.JSONResponse {
 	var kb keyBackupVersion
 	resErr := httputil.UnmarshalJSONRequest(req, &kb)
 	if resErr != nil {
@@ -89,7 +89,7 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI,
 
 // KeyBackupVersion returns the key backup version specified. If `version` is empty, the latest `keyBackupVersionResponse` is returned.
 // Implements GET  /_matrix/client/r0/room_keys/version and GET /_matrix/client/r0/room_keys/version/{version}
-func KeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
+func KeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
 	var queryResp userapi.QueryKeyBackupResponse
 	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
 		UserID:  device.UserID,
@@ -118,7 +118,7 @@ func KeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device
 
 // Modify the auth data of a key backup. Version must not be empty. Request must contain a `keyBackupVersion`
 // Implements PUT  /_matrix/client/r0/room_keys/version/{version}
-func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
+func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
 	var kb keyBackupVersion
 	resErr := httputil.UnmarshalJSONRequest(req, &kb)
 	if resErr != nil {
@@ -159,7 +159,7 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter
 
 // Delete a version of key backup. Version must not be empty. If the key backup was previously deleted, will return 200 OK.
 // Implements DELETE  /_matrix/client/r0/room_keys/version/{version}
-func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
+func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
 	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:       device.UserID,
@@ -194,7 +194,7 @@ func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI,
 
 // Upload a bunch of session keys for a given `version`.
 func UploadBackupKeys(
-	req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest,
+	req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest,
 ) util.JSONResponse {
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
 	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
@@ -230,7 +230,7 @@ func UploadBackupKeys(
 
 // Get keys from a given backup version. Response returned varies depending on if roomID and sessionID are set.
 func GetBackupKeys(
-	req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version, roomID, sessionID string,
+	req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version, roomID, sessionID string,
 ) util.JSONResponse {
 	var queryResp userapi.QueryKeyBackupResponse
 	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go
index c73e0a10d..8fbb86f7a 100644
--- a/clientapi/routing/key_crosssigning.go
+++ b/clientapi/routing/key_crosssigning.go
@@ -34,8 +34,8 @@ type crossSigningRequest struct {
 
 func UploadCrossSigningDeviceKeys(
 	req *http.Request, userInteractiveAuth *auth.UserInteractive,
-	keyserverAPI api.KeyInternalAPI, device *userapi.Device,
-	accountAPI userapi.UserAccountAPI, cfg *config.ClientAPI,
+	keyserverAPI api.ClientKeyAPI, device *userapi.Device,
+	accountAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
 	uploadReq := &crossSigningRequest{}
 	uploadRes := &api.PerformUploadDeviceKeysResponse{}
@@ -105,7 +105,7 @@ func UploadCrossSigningDeviceKeys(
 	}
 }
 
-func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
+func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
 	uploadReq := &api.PerformUploadDeviceSignaturesRequest{}
 	uploadRes := &api.PerformUploadDeviceSignaturesResponse{}
 
diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go
index 2d65ac353..fdda34a53 100644
--- a/clientapi/routing/keys.go
+++ b/clientapi/routing/keys.go
@@ -31,7 +31,7 @@ type uploadKeysRequest struct {
 	OneTimeKeys map[string]json.RawMessage `json:"one_time_keys"`
 }
 
-func UploadKeys(req *http.Request, keyAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
+func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
 	var r uploadKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
@@ -100,7 +100,7 @@ func (r *queryKeysRequest) GetTimeout() time.Duration {
 	return time.Duration(r.Timeout) * time.Millisecond
 }
 
-func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
+func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
 	var r queryKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
@@ -138,7 +138,7 @@ func (r *claimKeysRequest) GetTimeout() time.Duration {
 	return time.Duration(r.TimeoutMS) * time.Millisecond
 }
 
-func ClaimKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse {
+func ClaimKeys(req *http.Request, keyAPI api.ClientKeyAPI) util.JSONResponse {
 	var r claimKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
diff --git a/clientapi/routing/leaveroom.go b/clientapi/routing/leaveroom.go
index a34dd02d3..a71661851 100644
--- a/clientapi/routing/leaveroom.go
+++ b/clientapi/routing/leaveroom.go
@@ -26,7 +26,7 @@ import (
 func LeaveRoomByID(
 	req *http.Request,
 	device *api.Device,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
 	// Prepare to ask the roomserver to perform the room join.
diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go
index 2329df504..6017b5840 100644
--- a/clientapi/routing/login.go
+++ b/clientapi/routing/login.go
@@ -53,7 +53,7 @@ func passwordLogin() flows {
 
 // Login implements GET and POST /login
 func Login(
-	req *http.Request, userAPI userapi.UserInternalAPI,
+	req *http.Request, userAPI userapi.ClientUserAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
@@ -79,7 +79,7 @@ func Login(
 }
 
 func completeAuth(
-	ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.UserInternalAPI, login *auth.Login,
+	ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.ClientUserAPI, login *auth.Login,
 	ipAddr, userAgent string,
 ) util.JSONResponse {
 	token, err := auth.GenerateAccessToken()
diff --git a/clientapi/routing/logout.go b/clientapi/routing/logout.go
index cfbb6f9f2..73bae7af7 100644
--- a/clientapi/routing/logout.go
+++ b/clientapi/routing/logout.go
@@ -24,7 +24,7 @@ import (
 
 // Logout handles POST /logout
 func Logout(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	var performRes api.PerformDeviceDeletionResponse
 	err := userAPI.PerformDeviceDeletion(req.Context(), &api.PerformDeviceDeletionRequest{
@@ -44,7 +44,7 @@ func Logout(
 
 // LogoutAll handles POST /logout/all
 func LogoutAll(
-	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
+	req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	var performRes api.PerformDeviceDeletionResponse
 	err := userAPI.PerformDeviceDeletion(req.Context(), &api.PerformDeviceDeletionRequest{
diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index df8447b14..7d91c7b03 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -27,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -38,9 +39,9 @@ import (
 var errMissingUserID = errors.New("'user_id' must be supplied")
 
 func SendBan(
-	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
+	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -80,10 +81,10 @@ func SendBan(
 	return sendMembership(req.Context(), profileAPI, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
 }
 
-func sendMembership(ctx context.Context, profileAPI userapi.UserProfileAPI, device *userapi.Device,
+func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
 	roomVer gomatrixserverlib.RoomVersion,
-	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
 
 	event, err := buildMembershipEvent(
 		ctx, targetUserID, reason, profileAPI, device, membership,
@@ -124,9 +125,9 @@ func sendMembership(ctx context.Context, profileAPI userapi.UserProfileAPI, devi
 }
 
 func SendKick(
-	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
+	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -164,9 +165,9 @@ func SendKick(
 }
 
 func SendUnban(
-	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
+	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -199,9 +200,9 @@ func SendUnban(
 }
 
 func SendInvite(
-	req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
+	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	body, evTime, _, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -233,11 +234,11 @@ func SendInvite(
 // sendInvite sends an invitation to a user. Returns a JSONResponse and an error
 func sendInvite(
 	ctx context.Context,
-	profileAPI userapi.UserProfileAPI,
+	profileAPI userapi.ClientUserAPI,
 	device *userapi.Device,
 	roomID, userID, reason string,
 	cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI, evTime time.Time,
 ) (util.JSONResponse, error) {
 	event, err := buildMembershipEvent(
@@ -259,37 +260,36 @@ func sendInvite(
 		return jsonerror.InternalServerError(), err
 	}
 
-	err = roomserverAPI.SendInvite(
-		ctx, rsAPI,
-		event,
-		nil, // ask the roomserver to draw up invite room state for us
-		cfg.Matrix.ServerName,
-		nil,
-	)
-	switch e := err.(type) {
-	case *roomserverAPI.PerformError:
-		return e.JSONResponse(), err
-	case nil:
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: struct{}{},
-		}, nil
-	default:
-		util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInvite failed")
+	var inviteRes api.PerformInviteResponse
+	if err := rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
+		Event:           event,
+		InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
+		RoomVersion:     event.RoomVersion,
+		SendAsServer:    string(cfg.Matrix.ServerName),
+	}, &inviteRes); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
 		}, err
 	}
+	if inviteRes.Error != nil {
+		return inviteRes.Error.JSONResponse(), inviteRes.Error
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}, nil
 }
 
 func buildMembershipEvent(
 	ctx context.Context,
-	targetUserID, reason string, profileAPI userapi.UserProfileAPI,
+	targetUserID, reason string, profileAPI userapi.ClientUserAPI,
 	device *userapi.Device,
 	membership, roomID string, isDirect bool,
 	cfg *config.ClientAPI, evTime time.Time,
-	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
 	profile, err := loadProfile(ctx, targetUserID, cfg, profileAPI, asAPI)
 	if err != nil {
@@ -326,7 +326,7 @@ func loadProfile(
 	ctx context.Context,
 	userID string,
 	cfg *config.ClientAPI,
-	profileAPI userapi.UserProfileAPI,
+	profileAPI userapi.ClientUserAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*authtypes.Profile, error) {
 	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
@@ -344,7 +344,7 @@ func loadProfile(
 	return profile, err
 }
 
-func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.RoomserverInternalAPI) (
+func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.ClientRoomserverAPI) (
 	body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse,
 ) {
 	verReq := roomserverAPI.QueryRoomVersionForRoomRequest{RoomID: roomID}
@@ -379,8 +379,8 @@ func checkAndProcessThreepid(
 	device *userapi.Device,
 	body *threepid.MembershipRequest,
 	cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
-	profileAPI userapi.UserProfileAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+	profileAPI userapi.ClientUserAPI,
 	roomID string,
 	evTime time.Time,
 ) (inviteStored bool, errRes *util.JSONResponse) {
@@ -418,7 +418,7 @@ func checkAndProcessThreepid(
 	return
 }
 
-func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, userID, roomID string) *util.JSONResponse {
+func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse {
 	tuple := gomatrixserverlib.StateKeyTuple{
 		EventType: gomatrixserverlib.MRoomMember,
 		StateKey:  userID,
@@ -457,7 +457,7 @@ func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.RoomserverIntern
 
 func SendForget(
 	req *http.Request, device *userapi.Device,
-	roomID string, rsAPI roomserverAPI.RoomserverInternalAPI,
+	roomID string, rsAPI roomserverAPI.ClientRoomserverAPI,
 ) util.JSONResponse {
 	ctx := req.Context()
 	logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID)
diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go
index 6ddcf1be3..9bdd8a4f4 100644
--- a/clientapi/routing/memberships.go
+++ b/clientapi/routing/memberships.go
@@ -55,7 +55,7 @@ type databaseJoinedMember struct {
 func GetMemberships(
 	req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
 	_ *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	queryReq := api.QueryMembershipsForRoomRequest{
 		JoinedOnly: joinedOnly,
@@ -100,7 +100,7 @@ func GetMemberships(
 func GetJoinedRooms(
 	req *http.Request,
 	device *userapi.Device,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	var res api.QueryRoomsForUserResponse
 	err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
diff --git a/clientapi/routing/notification.go b/clientapi/routing/notification.go
index ee715d323..8a424a141 100644
--- a/clientapi/routing/notification.go
+++ b/clientapi/routing/notification.go
@@ -27,7 +27,7 @@ import (
 // GetNotifications handles /_matrix/client/r0/notifications
 func GetNotifications(
 	req *http.Request, device *userapi.Device,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.ClientUserAPI,
 ) util.JSONResponse {
 	var limit int64
 	if limitStr := req.URL.Query().Get("limit"); limitStr != "" {
diff --git a/clientapi/routing/openid.go b/clientapi/routing/openid.go
index 13656e288..cfb440bea 100644
--- a/clientapi/routing/openid.go
+++ b/clientapi/routing/openid.go
@@ -34,7 +34,7 @@ type openIDTokenResponse struct {
 // can supply to an OpenID Relying Party to verify their identity
 func CreateOpenIDToken(
 	req *http.Request,
-	userAPI api.UserInternalAPI,
+	userAPI api.ClientUserAPI,
 	device *api.Device,
 	userID string,
 	cfg *config.ClientAPI,
diff --git a/clientapi/routing/password.go b/clientapi/routing/password.go
index 08ce1ffa1..6dc9af508 100644
--- a/clientapi/routing/password.go
+++ b/clientapi/routing/password.go
@@ -28,7 +28,7 @@ type newPasswordAuth struct {
 
 func Password(
 	req *http.Request,
-	userAPI api.UserInternalAPI,
+	userAPI api.ClientUserAPI,
 	device *api.Device,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
diff --git a/clientapi/routing/peekroom.go b/clientapi/routing/peekroom.go
index 41d1ff004..d0eeccf17 100644
--- a/clientapi/routing/peekroom.go
+++ b/clientapi/routing/peekroom.go
@@ -26,7 +26,7 @@ import (
 func PeekRoomByIDOrAlias(
 	req *http.Request,
 	device *api.Device,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	roomIDOrAlias string,
 ) util.JSONResponse {
 	// if this is a remote roomIDOrAlias, we have to ask the roomserver (or federation sender?) to
@@ -79,7 +79,7 @@ func PeekRoomByIDOrAlias(
 func UnpeekRoomByID(
 	req *http.Request,
 	device *api.Device,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
 	unpeekReq := roomserverAPI.PerformUnpeekRequest{
diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go
index 3f91b4c93..97f86afe2 100644
--- a/clientapi/routing/profile.go
+++ b/clientapi/routing/profile.go
@@ -35,7 +35,7 @@ import (
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
@@ -64,7 +64,7 @@ func GetProfile(
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -91,8 +91,8 @@ func GetAvatarURL(
 
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 func SetAvatarURL(
-	req *http.Request, profileAPI userapi.UserProfileAPI,
-	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
+	req *http.Request, profileAPI userapi.ClientUserAPI,
+	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -193,7 +193,7 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
+	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -220,8 +220,8 @@ func GetDisplayName(
 
 // SetDisplayName implements PUT /profile/{userID}/displayname
 func SetDisplayName(
-	req *http.Request, profileAPI userapi.UserProfileAPI,
-	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
+	req *http.Request, profileAPI userapi.ClientUserAPI,
+	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -325,7 +325,7 @@ func SetDisplayName(
 // Returns an error when something goes wrong or specifically
 // eventutil.ErrProfileNoExists when the profile doesn't exist.
 func getProfile(
-	ctx context.Context, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
+	ctx context.Context, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
@@ -366,7 +366,7 @@ func buildMembershipEvents(
 	ctx context.Context,
 	roomIDs []string,
 	newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
-	evTime time.Time, rsAPI api.RoomserverInternalAPI,
+	evTime time.Time, rsAPI api.ClientRoomserverAPI,
 ) ([]*gomatrixserverlib.HeaderedEvent, error) {
 	evs := []*gomatrixserverlib.HeaderedEvent{}
 
diff --git a/clientapi/routing/pusher.go b/clientapi/routing/pusher.go
index 9d6bef8bd..d6a6eb936 100644
--- a/clientapi/routing/pusher.go
+++ b/clientapi/routing/pusher.go
@@ -28,7 +28,7 @@ import (
 // GetPushers handles /_matrix/client/r0/pushers
 func GetPushers(
 	req *http.Request, device *userapi.Device,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.ClientUserAPI,
 ) util.JSONResponse {
 	var queryRes userapi.QueryPushersResponse
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@@ -57,7 +57,7 @@ func GetPushers(
 // The behaviour of this endpoint varies depending on the values in the JSON body.
 func SetPusher(
 	req *http.Request, device *userapi.Device,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.ClientUserAPI,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
diff --git a/clientapi/routing/pushrules.go b/clientapi/routing/pushrules.go
index 81a33b25a..856f52c75 100644
--- a/clientapi/routing/pushrules.go
+++ b/clientapi/routing/pushrules.go
@@ -30,7 +30,7 @@ func errorResponse(ctx context.Context, err error, msg string, args ...interface
 	return jsonerror.InternalServerError()
 }
 
-func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRulesJSON failed")
@@ -41,7 +41,7 @@ func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userap
 	}
 }
 
-func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRulesJSON failed")
@@ -56,7 +56,7 @@ func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Devi
 	}
 }
 
-func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
@@ -75,7 +75,7 @@ func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi
 	}
 }
 
-func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
@@ -98,7 +98,7 @@ func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device
 	}
 }
 
-func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID, beforeRuleID string, body io.Reader, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID, beforeRuleID string, body io.Reader, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	var newRule pushrules.Rule
 	if err := json.NewDecoder(body).Decode(&newRule); err != nil {
 		return errorResponse(ctx, err, "JSON Decode failed")
@@ -160,7 +160,7 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
 	return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
 }
 
-func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
 	if err != nil {
 		return errorResponse(ctx, err, "queryPushRules failed")
@@ -187,7 +187,7 @@ func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, dev
 	return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
 }
 
-func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	attrGet, err := pushRuleAttrGetter(attr)
 	if err != nil {
 		return errorResponse(ctx, err, "pushRuleAttrGetter failed")
@@ -216,7 +216,7 @@ func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	}
 }
 
-func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, body io.Reader, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
+func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, body io.Reader, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
 	var newPartialRule pushrules.Rule
 	if err := json.NewDecoder(body).Decode(&newPartialRule); err != nil {
 		return util.JSONResponse{
@@ -266,7 +266,7 @@ func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
 	return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
 }
 
-func queryPushRules(ctx context.Context, userID string, userAPI userapi.UserInternalAPI) (*pushrules.AccountRuleSets, error) {
+func queryPushRules(ctx context.Context, userID string, userAPI userapi.ClientUserAPI) (*pushrules.AccountRuleSets, error) {
 	var res userapi.QueryPushRulesResponse
 	if err := userAPI.QueryPushRules(ctx, &userapi.QueryPushRulesRequest{UserID: userID}, &res); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("userAPI.QueryPushRules failed")
@@ -275,7 +275,7 @@ func queryPushRules(ctx context.Context, userID string, userAPI userapi.UserInte
 	return res.RuleSets, nil
 }
 
-func putPushRules(ctx context.Context, userID string, ruleSets *pushrules.AccountRuleSets, userAPI userapi.UserInternalAPI) error {
+func putPushRules(ctx context.Context, userID string, ruleSets *pushrules.AccountRuleSets, userAPI userapi.ClientUserAPI) error {
 	req := userapi.PerformPushRulesPutRequest{
 		UserID:   userID,
 		RuleSets: ruleSets,
diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go
index e8d14ce34..27f0ba5d0 100644
--- a/clientapi/routing/redaction.go
+++ b/clientapi/routing/redaction.go
@@ -40,7 +40,7 @@ type redactionResponse struct {
 
 func SendRedaction(
 	req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	txnID *string,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go
index 8253f3155..eba4920c6 100644
--- a/clientapi/routing/register.go
+++ b/clientapi/routing/register.go
@@ -518,7 +518,7 @@ func validateApplicationService(
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 func Register(
 	req *http.Request,
-	userAPI userapi.UserRegisterAPI,
+	userAPI userapi.ClientUserAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	defer req.Body.Close() // nolint: errcheck
@@ -614,7 +614,7 @@ func handleGuestRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserRegisterAPI,
+	userAPI userapi.ClientUserAPI,
 ) util.JSONResponse {
 	if cfg.RegistrationDisabled || cfg.GuestsDisabled {
 		return util.JSONResponse{
@@ -679,7 +679,7 @@ func handleRegistrationFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserRegisterAPI,
+	userAPI userapi.ClientUserAPI,
 	accessToken string,
 	accessTokenErr error,
 ) util.JSONResponse {
@@ -768,7 +768,7 @@ func handleApplicationServiceRegistration(
 	req *http.Request,
 	r registerRequest,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserRegisterAPI,
+	userAPI userapi.ClientUserAPI,
 ) util.JSONResponse {
 	// Check if we previously had issues extracting the access token from the
 	// request.
@@ -806,7 +806,7 @@ func checkAndCompleteFlow(
 	r registerRequest,
 	sessionID string,
 	cfg *config.ClientAPI,
-	userAPI userapi.UserRegisterAPI,
+	userAPI userapi.ClientUserAPI,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
@@ -833,7 +833,7 @@ func checkAndCompleteFlow(
 // not all
 func completeRegistration(
 	ctx context.Context,
-	userAPI userapi.UserRegisterAPI,
+	userAPI userapi.ClientUserAPI,
 	username, password, appserviceID, ipAddr, userAgent, sessionID string,
 	inhibitLogin eventutil.WeakBoolean,
 	displayName, deviceID *string,
@@ -992,7 +992,7 @@ type availableResponse struct {
 func RegisterAvailable(
 	req *http.Request,
 	cfg *config.ClientAPI,
-	registerAPI userapi.UserRegisterAPI,
+	registerAPI userapi.ClientUserAPI,
 ) util.JSONResponse {
 	username := req.URL.Query().Get("username")
 
@@ -1040,7 +1040,7 @@ func RegisterAvailable(
 	}
 }
 
-func handleSharedSecretRegistration(userAPI userapi.UserInternalAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
+func handleSharedSecretRegistration(userAPI userapi.ClientUserAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
 	ssrr, err := NewSharedSecretRegistrationRequest(req.Body)
 	if err != nil {
 		return util.JSONResponse{
diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go
index ce173613e..039289569 100644
--- a/clientapi/routing/room_tagging.go
+++ b/clientapi/routing/room_tagging.go
@@ -31,7 +31,7 @@ import (
 // GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags
 func GetTags(
 	req *http.Request,
-	userAPI api.UserInternalAPI,
+	userAPI api.ClientUserAPI,
 	device *api.Device,
 	userID string,
 	roomID string,
@@ -62,7 +62,7 @@ func GetTags(
 // the tag to the "map" and saving the new "map" to the DB
 func PutTag(
 	req *http.Request,
-	userAPI api.UserInternalAPI,
+	userAPI api.ClientUserAPI,
 	device *api.Device,
 	userID string,
 	roomID string,
@@ -113,7 +113,7 @@ func PutTag(
 // the "map" and then saving the new "map" in the DB
 func DeleteTag(
 	req *http.Request,
-	userAPI api.UserInternalAPI,
+	userAPI api.ClientUserAPI,
 	device *api.Device,
 	userID string,
 	roomID string,
@@ -167,7 +167,7 @@ func obtainSavedTags(
 	req *http.Request,
 	userID string,
 	roomID string,
-	userAPI api.UserInternalAPI,
+	userAPI api.ClientUserAPI,
 ) (tags gomatrix.TagContent, err error) {
 	dataReq := api.QueryAccountDataRequest{
 		UserID:   userID,
@@ -194,7 +194,7 @@ func saveTagData(
 	req *http.Request,
 	userID string,
 	roomID string,
-	userAPI api.UserInternalAPI,
+	userAPI api.ClientUserAPI,
 	Tag gomatrix.TagContent,
 ) error {
 	newTagData, err := json.Marshal(Tag)
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index ba1c76b81..6da467073 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -50,15 +50,15 @@ import (
 func Setup(
 	publicAPIMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.UserDirectoryProvider,
 	federation *gomatrixserverlib.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
-	federationSender federationAPI.FederationInternalAPI,
-	keyAPI keyserverAPI.KeyInternalAPI,
+	federationSender federationAPI.ClientFederationAPI,
+	keyAPI keyserverAPI.ClientKeyAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs, natsClient *nats.Conn,
 ) {
@@ -325,7 +325,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation)
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
@@ -897,7 +897,7 @@ func Setup(
 			if resErr := clientutil.UnmarshalJSONRequest(req, &postContent); resErr != nil {
 				return *resErr
 			}
-			return *SearchUserDirectory(
+			return SearchUserDirectory(
 				req.Context(),
 				device,
 				userAPI,
diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index 1211fa72d..5f84739d0 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -70,7 +70,7 @@ func SendEvent(
 	device *userapi.Device,
 	roomID, eventType string, txnID, stateKey *string,
 	cfg *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.ClientRoomserverAPI,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
@@ -207,7 +207,7 @@ func generateSendEvent(
 	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
 	cfg *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	// parse the incoming http request
diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go
index 6a27ee615..3f92e4227 100644
--- a/clientapi/routing/sendtyping.go
+++ b/clientapi/routing/sendtyping.go
@@ -32,7 +32,7 @@ type typingContentJSON struct {
 // sends the typing events to client API typingProducer
 func SendTyping(
 	req *http.Request, device *userapi.Device, roomID string,
-	userID string, rsAPI roomserverAPI.RoomserverInternalAPI,
+	userID string, rsAPI roomserverAPI.ClientRoomserverAPI,
 	syncProducer *producers.SyncAPIProducer,
 ) util.JSONResponse {
 	if device.UserID != userID {
diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go
index 47b0da7bb..9c34f2e1c 100644
--- a/clientapi/routing/server_notices.go
+++ b/clientapi/routing/server_notices.go
@@ -56,8 +56,8 @@ func SendServerNotice(
 	req *http.Request,
 	cfgNotices *config.ServerNotices,
 	cfgClient *config.ClientAPI,
-	userAPI userapi.UserInternalAPI,
-	rsAPI api.RoomserverInternalAPI,
+	userAPI userapi.ClientUserAPI,
+	rsAPI api.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	device *userapi.Device,
 	senderDevice *userapi.Device,
@@ -276,7 +276,7 @@ func (r sendServerNoticeRequest) valid() (ok bool) {
 // It returns an userapi.Device, which is used for building the event
 func getSenderDevice(
 	ctx context.Context,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.ClientUserAPI,
 	cfg *config.ClientAPI,
 ) (*userapi.Device, error) {
 	var accRes userapi.PerformAccountCreationResponse
diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index d25ee8237..c6e9e91d0 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -41,7 +41,7 @@ type stateEventInStateResp struct {
 // TODO: Check if the user is in the room. If not, check if the room's history
 // is publicly visible. Current behaviour is returning an empty array if the
 // user cannot see the room's history.
-func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse {
+func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI api.ClientRoomserverAPI, roomID string) util.JSONResponse {
 	var worldReadable bool
 	var wantLatestState bool
 
@@ -162,7 +162,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 // is then (by default) we return the content, otherwise a 404.
 // If eventFormat=true, sends the whole event else just the content.
 func OnIncomingStateTypeRequest(
-	ctx context.Context, device *userapi.Device, rsAPI api.RoomserverInternalAPI,
+	ctx context.Context, device *userapi.Device, rsAPI api.ClientRoomserverAPI,
 	roomID, evType, stateKey string, eventFormat bool,
 ) util.JSONResponse {
 	var worldReadable bool
diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go
index a4898ca46..94b658ee3 100644
--- a/clientapi/routing/threepid.go
+++ b/clientapi/routing/threepid.go
@@ -40,7 +40,7 @@ type threePIDsResponse struct {
 // RequestEmailToken implements:
 //     POST /account/3pid/email/requestToken
 //     POST /register/email/requestToken
-func RequestEmailToken(req *http.Request, threePIDAPI api.UserThreePIDAPI, cfg *config.ClientAPI) util.JSONResponse {
+func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
@@ -90,7 +90,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.UserThreePIDAPI, cfg *
 
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
-	req *http.Request, threePIDAPI api.UserThreePIDAPI, device *api.Device,
+	req *http.Request, threePIDAPI api.ClientUserAPI, device *api.Device,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
@@ -158,7 +158,7 @@ func CheckAndSave3PIDAssociation(
 
 // GetAssociated3PIDs implements GET /account/3pid
 func GetAssociated3PIDs(
-	req *http.Request, threepidAPI api.UserThreePIDAPI, device *api.Device,
+	req *http.Request, threepidAPI api.ClientUserAPI, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -182,7 +182,7 @@ func GetAssociated3PIDs(
 }
 
 // Forget3PID implements POST /account/3pid/delete
-func Forget3PID(req *http.Request, threepidAPI api.UserThreePIDAPI) util.JSONResponse {
+func Forget3PID(req *http.Request, threepidAPI api.ClientUserAPI) util.JSONResponse {
 	var body authtypes.ThreePID
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
diff --git a/clientapi/routing/upgrade_room.go b/clientapi/routing/upgrade_room.go
index 00bde36b3..505bf8f53 100644
--- a/clientapi/routing/upgrade_room.go
+++ b/clientapi/routing/upgrade_room.go
@@ -40,8 +40,8 @@ type upgradeRoomResponse struct {
 func UpgradeRoom(
 	req *http.Request, device *userapi.Device,
 	cfg *config.ClientAPI,
-	roomID string, profileAPI userapi.UserProfileAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	roomID string, profileAPI userapi.ClientUserAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	var r upgradeRoomRequest
diff --git a/clientapi/routing/userdirectory.go b/clientapi/routing/userdirectory.go
index ab73cf430..4d41da734 100644
--- a/clientapi/routing/userdirectory.go
+++ b/clientapi/routing/userdirectory.go
@@ -34,13 +34,13 @@ type UserDirectoryResponse struct {
 func SearchUserDirectory(
 	ctx context.Context,
 	device *userapi.Device,
-	userAPI userapi.UserInternalAPI,
-	rsAPI api.RoomserverInternalAPI,
+	userAPI userapi.ClientUserAPI,
+	rsAPI api.ClientRoomserverAPI,
 	provider userapi.UserDirectoryProvider,
 	serverName gomatrixserverlib.ServerName,
 	searchString string,
 	limit int,
-) *util.JSONResponse {
+) util.JSONResponse {
 	if limit < 10 {
 		limit = 10
 	}
@@ -58,8 +58,7 @@ func SearchUserDirectory(
 	}
 	userRes := &userapi.QuerySearchProfilesResponse{}
 	if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
-		errRes := util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
-		return &errRes
+		return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
 	}
 
 	for _, user := range userRes.Profiles {
@@ -94,8 +93,7 @@ func SearchUserDirectory(
 		}
 		stateRes := &api.QueryKnownUsersResponse{}
 		if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows {
-			errRes := util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
-			return &errRes
+			return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
 		}
 
 		for _, user := range stateRes.Users {
@@ -114,7 +112,7 @@ func SearchUserDirectory(
 		response.Results = append(response.Results, result)
 	}
 
-	return &util.JSONResponse{
+	return util.JSONResponse{
 		Code: 200,
 		JSON: response,
 	}
diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go
index eee6992f8..6e7426a7f 100644
--- a/clientapi/threepid/invites.go
+++ b/clientapi/threepid/invites.go
@@ -86,7 +86,7 @@ var (
 func CheckAndProcessInvite(
 	ctx context.Context,
 	device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI, db userapi.UserProfileAPI,
+	rsAPI api.ClientRoomserverAPI, db userapi.ClientUserAPI,
 	roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
@@ -136,7 +136,7 @@ func CheckAndProcessInvite(
 // Returns an error if a check or a request failed.
 func queryIDServer(
 	ctx context.Context,
-	db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device,
+	userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
 	if err = isTrusted(body.IDServer, cfg); err != nil {
@@ -152,7 +152,7 @@ func queryIDServer(
 	if lookupRes.MXID == "" {
 		// No Matrix ID matches with the given 3PID, ask the server to store the
 		// invite and return a token
-		storeInviteRes, err = queryIDServerStoreInvite(ctx, db, cfg, device, body, roomID)
+		storeInviteRes, err = queryIDServerStoreInvite(ctx, userAPI, cfg, device, body, roomID)
 		return
 	}
 
@@ -163,7 +163,7 @@ func queryIDServer(
 	if lookupRes.NotBefore > now || now > lookupRes.NotAfter {
 		// If the current timestamp isn't in the time frame in which the association
 		// is known to be valid, re-run the query
-		return queryIDServer(ctx, db, cfg, device, body, roomID)
+		return queryIDServer(ctx, userAPI, cfg, device, body, roomID)
 	}
 
 	// Check the request signatures and send an error if one isn't valid
@@ -205,7 +205,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
 // Returns an error if the request failed to send or if the response couldn't be parsed.
 func queryIDServerStoreInvite(
 	ctx context.Context,
-	db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device,
+	userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
@@ -217,7 +217,7 @@ func queryIDServerStoreInvite(
 	var profile *authtypes.Profile
 	if serverName == cfg.Matrix.ServerName {
 		res := &userapi.QueryProfileResponse{}
-		err = db.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
+		err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
 		if err != nil {
 			return nil, err
 		}
@@ -337,7 +337,7 @@ func emit3PIDInviteEvent(
 	ctx context.Context,
 	body *MembershipRequest, res *idServerStoreInviteResponse,
 	device *userapi.Device, roomID string, cfg *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
 ) error {
 	builder := &gomatrixserverlib.EventBuilder{
diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index 4d6b0211c..ce6a0f2ed 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -42,6 +42,7 @@ func (e *FederationClientError) Error() string {
 type FederationInternalAPI interface {
 	FederationClient
 	gomatrixserverlib.KeyDatabase
+	ClientFederationAPI
 
 	KeyRing() *gomatrixserverlib.KeyRing
 
@@ -100,6 +101,10 @@ type FederationInternalAPI interface {
 	) error
 }
 
+type ClientFederationAPI interface {
+	QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
+}
+
 type QueryServerKeysRequest struct {
 	ServerName      gomatrixserverlib.ServerName
 	KeyIDToCriteria map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyNotaryQueryCriteria
diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go
index 58bf99f4a..25faff0cb 100644
--- a/federationapi/routing/invite.go
+++ b/federationapi/routing/invite.go
@@ -166,31 +166,36 @@ func processInvite(
 	)
 
 	// Add the invite event to the roomserver.
-	err = api.SendInvite(
-		ctx, rsAPI, signedEvent.Headered(roomVer), strippedState, api.DoNotSendToOtherServers, nil,
-	)
-	switch e := err.(type) {
-	case *api.PerformError:
-		return e.JSONResponse()
-	case nil:
-		// Return the signed event to the originating server, it should then tell
-		// the other servers in the room that we have been invited.
-		if isInviteV2 {
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent.JSON()},
-			}
-		} else {
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: gomatrixserverlib.RespInvite{Event: signedEvent.JSON()},
-			}
-		}
-	default:
-		util.GetLogger(ctx).WithError(err).Error("api.SendInvite failed")
+	inviteEvent := signedEvent.Headered(roomVer)
+	request := &api.PerformInviteRequest{
+		Event:           inviteEvent,
+		InviteRoomState: strippedState,
+		RoomVersion:     inviteEvent.RoomVersion,
+		SendAsServer:    string(api.DoNotSendToOtherServers),
+		TransactionID:   nil,
+	}
+	response := &api.PerformInviteResponse{}
+	if err := rsAPI.PerformInvite(ctx, request, response); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
 		}
 	}
+	if response.Error != nil {
+		return response.Error.JSONResponse()
+	}
+	// Return the signed event to the originating server, it should then tell
+	// the other servers in the room that we have been invited.
+	if isInviteV2 {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent.JSON()},
+		}
+	} else {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: gomatrixserverlib.RespInvite{Event: signedEvent.JSON()},
+		}
+	}
 }
diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go
index 47c83d515..ee67a6daf 100644
--- a/internal/eventutil/events.go
+++ b/internal/eventutil/events.go
@@ -39,7 +39,7 @@ var ErrRoomNoExists = errors.New("room does not exist")
 func QueryAndBuildEvent(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder, cfg *config.Global, evTime time.Time,
-	rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse,
+	rsAPI api.QueryLatestEventsAndStateAPI, queryRes *api.QueryLatestEventsAndStateResponse,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
 	if queryRes == nil {
 		queryRes = &api.QueryLatestEventsAndStateResponse{}
@@ -80,7 +80,7 @@ func BuildEvent(
 func queryRequiredEventsForBuilder(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder,
-	rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse,
+	rsAPI api.QueryLatestEventsAndStateAPI, queryRes *api.QueryLatestEventsAndStateResponse,
 ) (*gomatrixserverlib.StateNeeded, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
 	if err != nil {
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index ce651ba4e..5564eb271 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -28,21 +28,27 @@ import (
 
 type KeyInternalAPI interface {
 	SyncKeyAPI
+	ClientKeyAPI
 	// SetUserAPI assigns a user API to query when extracting device names.
 	SetUserAPI(i userapi.UserInternalAPI)
 	// InputDeviceListUpdate from a federated server EDU
 	InputDeviceListUpdate(ctx context.Context, req *InputDeviceListUpdateRequest, res *InputDeviceListUpdateResponse)
-	PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse)
-	// PerformClaimKeys claims one-time keys for use in pre-key messages
-	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse)
+
 	PerformDeleteKeys(ctx context.Context, req *PerformDeleteKeysRequest, res *PerformDeleteKeysResponse)
-	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse)
-	PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse)
-	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse)
 	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse)
 	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse)
 }
 
+// API functions required by the clientapi
+type ClientKeyAPI interface {
+	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse)
+	PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse)
+	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse)
+	PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse)
+	// PerformClaimKeys claims one-time keys for use in pre-key messages
+	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse)
+}
+
 // API functions required by the syncapi
 type SyncKeyAPI interface {
 	QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse)
diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go
index 5976957ca..4792c996d 100644
--- a/mediaapi/mediaapi.go
+++ b/mediaapi/mediaapi.go
@@ -26,7 +26,7 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.
 func AddPublicRoutes(
 	base *base.BaseDendrite,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.MediaUserAPI,
 	client *gomatrixserverlib.Client,
 ) {
 	cfg := &base.Cfg.MediaAPI
diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go
index 97dfd3341..76f07415b 100644
--- a/mediaapi/routing/routing.go
+++ b/mediaapi/routing/routing.go
@@ -48,7 +48,7 @@ func Setup(
 	cfg *config.MediaAPI,
 	rateLimit *config.RateLimiting,
 	db storage.Database,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.MediaUserAPI,
 	client *gomatrixserverlib.Client,
 ) {
 	rateLimits := httputil.NewRateLimits(rateLimit)
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 2e4ec3ffd..33c3d157b 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -12,7 +12,13 @@ import (
 
 // RoomserverInputAPI is used to write events to the room server.
 type RoomserverInternalAPI interface {
+	InputRoomEventsAPI
+	QueryLatestEventsAndStateAPI
+	QueryEventsAPI
+
 	SyncRoomserverAPI
+	AppserviceRoomserverAPI
+	ClientRoomserverAPI
 
 	// needed to avoid chicken and egg scenario when setting up the
 	// interdependencies between the roomserver and other input APIs
@@ -20,12 +26,6 @@ type RoomserverInternalAPI interface {
 	SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI)
 	SetUserAPI(userAPI userapi.UserInternalAPI)
 
-	InputRoomEvents(
-		ctx context.Context,
-		request *InputRoomEventsRequest,
-		response *InputRoomEventsResponse,
-	)
-
 	PerformInvite(
 		ctx context.Context,
 		req *PerformInviteRequest,
@@ -68,44 +68,31 @@ type RoomserverInternalAPI interface {
 		res *PerformInboundPeekResponse,
 	) error
 
-	PerformAdminEvacuateRoom(
-		ctx context.Context,
-		req *PerformAdminEvacuateRoomRequest,
-		res *PerformAdminEvacuateRoomResponse,
-	)
-
 	QueryPublishedRooms(
 		ctx context.Context,
 		req *QueryPublishedRoomsRequest,
 		res *QueryPublishedRoomsResponse,
 	) error
 
-	// Query a list of membership events for a room
-	QueryMembershipsForRoom(
-		ctx context.Context,
-		request *QueryMembershipsForRoomRequest,
-		response *QueryMembershipsForRoomResponse,
-	) error
-
 	// Query if we think we're still in a room.
 	QueryServerJoinedToRoom(
 		ctx context.Context,
-		request *QueryServerJoinedToRoomRequest,
-		response *QueryServerJoinedToRoomResponse,
+		req *QueryServerJoinedToRoomRequest,
+		res *QueryServerJoinedToRoomResponse,
 	) error
 
 	// Query whether a server is allowed to see an event
 	QueryServerAllowedToSeeEvent(
 		ctx context.Context,
-		request *QueryServerAllowedToSeeEventRequest,
-		response *QueryServerAllowedToSeeEventResponse,
+		req *QueryServerAllowedToSeeEventRequest,
+		res *QueryServerAllowedToSeeEventResponse,
 	) error
 
 	// Query missing events for a room from roomserver
 	QueryMissingEvents(
 		ctx context.Context,
-		request *QueryMissingEventsRequest,
-		response *QueryMissingEventsResponse,
+		req *QueryMissingEventsRequest,
+		res *QueryMissingEventsResponse,
 	) error
 
 	// Query to get state and auth chain for a (potentially hypothetical) event.
@@ -113,8 +100,8 @@ type RoomserverInternalAPI interface {
 	// the state and auth chain to return.
 	QueryStateAndAuthChain(
 		ctx context.Context,
-		request *QueryStateAndAuthChainRequest,
-		response *QueryStateAndAuthChainResponse,
+		req *QueryStateAndAuthChainRequest,
+		res *QueryStateAndAuthChainResponse,
 	) error
 
 	// QueryAuthChain returns the entire auth chain for the event IDs given.
@@ -122,112 +109,179 @@ type RoomserverInternalAPI interface {
 	// Omits without error for any missing auth events. There will be no duplicates.
 	QueryAuthChain(
 		ctx context.Context,
-		request *QueryAuthChainRequest,
-		response *QueryAuthChainResponse,
+		req *QueryAuthChainRequest,
+		res *QueryAuthChainResponse,
 	) error
 
-	// QueryCurrentState retrieves the requested state events. If state events are not found, they will be missing from
-	// the response.
-	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
 	// QueryRoomsForUser retrieves a list of room IDs matching the given query.
 	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
-	// QueryKnownUsers returns a list of users that we know about from our joined rooms.
-	QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error
 	// QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs.
 	QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error
 
-	// PerformForget forgets a rooms history for a specific user
-	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
-
 	// PerformRoomUpgrade upgrades a room to a newer version
 	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
 
 	// Asks for the default room version as preferred by the server.
 	QueryRoomVersionCapabilities(
 		ctx context.Context,
-		request *QueryRoomVersionCapabilitiesRequest,
-		response *QueryRoomVersionCapabilitiesResponse,
+		req *QueryRoomVersionCapabilitiesRequest,
+		res *QueryRoomVersionCapabilitiesResponse,
 	) error
 
 	// Asks for the room version for a given room.
 	QueryRoomVersionForRoom(
 		ctx context.Context,
-		request *QueryRoomVersionForRoomRequest,
-		response *QueryRoomVersionForRoomResponse,
+		req *QueryRoomVersionForRoomRequest,
+		res *QueryRoomVersionForRoomResponse,
 	) error
 
 	// Set a room alias
 	SetRoomAlias(
 		ctx context.Context,
 		req *SetRoomAliasRequest,
-		response *SetRoomAliasResponse,
+		res *SetRoomAliasResponse,
 	) error
 
 	// Get the room ID for an alias
 	GetRoomIDForAlias(
 		ctx context.Context,
 		req *GetRoomIDForAliasRequest,
-		response *GetRoomIDForAliasResponse,
-	) error
-
-	// Get all known aliases for a room ID
-	GetAliasesForRoomID(
-		ctx context.Context,
-		req *GetAliasesForRoomIDRequest,
-		response *GetAliasesForRoomIDResponse,
+		res *GetRoomIDForAliasResponse,
 	) error
 
 	// Get the user ID of the creator of an alias
 	GetCreatorIDForAlias(
 		ctx context.Context,
 		req *GetCreatorIDForAliasRequest,
-		response *GetCreatorIDForAliasResponse,
+		res *GetCreatorIDForAliasResponse,
 	) error
 
 	// Remove a room alias
 	RemoveRoomAlias(
 		ctx context.Context,
 		req *RemoveRoomAliasRequest,
-		response *RemoveRoomAliasResponse,
+		res *RemoveRoomAliasResponse,
 	) error
 }
 
+type InputRoomEventsAPI interface {
+	InputRoomEvents(
+		ctx context.Context,
+		req *InputRoomEventsRequest,
+		res *InputRoomEventsResponse,
+	)
+}
+
+// Query the latest events and state for a room from the room server.
+type QueryLatestEventsAndStateAPI interface {
+	QueryLatestEventsAndState(ctx context.Context, req *QueryLatestEventsAndStateRequest, res *QueryLatestEventsAndStateResponse) error
+}
+
+// QueryBulkStateContent does a bulk query for state event content in the given rooms.
+type QueryBulkStateContentAPI interface {
+	QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
+}
+
+type QueryEventsAPI interface {
+	// Query a list of events by event ID.
+	QueryEventsByID(
+		ctx context.Context,
+		req *QueryEventsByIDRequest,
+		res *QueryEventsByIDResponse,
+	) error
+	// QueryCurrentState retrieves the requested state events. If state events are not found, they will be missing from
+	// the response.
+	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
+}
+
 // API functions required by the syncapi
 type SyncRoomserverAPI interface {
-	// Query the latest events and state for a room from the room server.
-	QueryLatestEventsAndState(
-		ctx context.Context,
-		request *QueryLatestEventsAndStateRequest,
-		response *QueryLatestEventsAndStateResponse,
-	) error
-	// QueryBulkStateContent does a bulk query for state event content in the given rooms.
-	QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
+	QueryLatestEventsAndStateAPI
+	QueryBulkStateContentAPI
 	// QuerySharedUsers returns a list of users who share at least 1 room in common with the given user.
 	QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error
 	// Query a list of events by event ID.
 	QueryEventsByID(
 		ctx context.Context,
-		request *QueryEventsByIDRequest,
-		response *QueryEventsByIDResponse,
+		req *QueryEventsByIDRequest,
+		res *QueryEventsByIDResponse,
 	) error
 	// Query the membership event for an user for a room.
 	QueryMembershipForUser(
 		ctx context.Context,
-		request *QueryMembershipForUserRequest,
-		response *QueryMembershipForUserResponse,
+		req *QueryMembershipForUserRequest,
+		res *QueryMembershipForUserResponse,
 	) error
 
 	// Query the state after a list of events in a room from the room server.
 	QueryStateAfterEvents(
 		ctx context.Context,
-		request *QueryStateAfterEventsRequest,
-		response *QueryStateAfterEventsResponse,
+		req *QueryStateAfterEventsRequest,
+		res *QueryStateAfterEventsResponse,
 	) error
 
 	// Query a given amount (or less) of events prior to a given set of events.
 	PerformBackfill(
 		ctx context.Context,
-		request *PerformBackfillRequest,
-		response *PerformBackfillResponse,
+		req *PerformBackfillRequest,
+		res *PerformBackfillResponse,
 	) error
 }
+
+type AppserviceRoomserverAPI interface {
+	// Query a list of events by event ID.
+	QueryEventsByID(
+		ctx context.Context,
+		req *QueryEventsByIDRequest,
+		res *QueryEventsByIDResponse,
+	) error
+	// Query a list of membership events for a room
+	QueryMembershipsForRoom(
+		ctx context.Context,
+		req *QueryMembershipsForRoomRequest,
+		res *QueryMembershipsForRoomResponse,
+	) error
+	// Get all known aliases for a room ID
+	GetAliasesForRoomID(
+		ctx context.Context,
+		req *GetAliasesForRoomIDRequest,
+		res *GetAliasesForRoomIDResponse,
+	) error
+}
+
+type ClientRoomserverAPI interface {
+	InputRoomEventsAPI
+	QueryLatestEventsAndStateAPI
+	QueryBulkStateContentAPI
+	QueryEventsAPI
+	QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error
+	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
+	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+	QueryStateAfterEvents(ctx context.Context, req *QueryStateAfterEventsRequest, res *QueryStateAfterEventsResponse) error
+	// QueryKnownUsers returns a list of users that we know about from our joined rooms.
+	QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error
+	QueryRoomVersionForRoom(ctx context.Context, req *QueryRoomVersionForRoomRequest, res *QueryRoomVersionForRoomResponse) error
+	QueryPublishedRooms(ctx context.Context, req *QueryPublishedRoomsRequest, res *QueryPublishedRoomsResponse) error
+	QueryRoomVersionCapabilities(ctx context.Context, req *QueryRoomVersionCapabilitiesRequest, res *QueryRoomVersionCapabilitiesResponse) error
+
+	GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error
+	GetAliasesForRoomID(ctx context.Context, req *GetAliasesForRoomIDRequest, res *GetAliasesForRoomIDResponse) error
+
+	// PerformRoomUpgrade upgrades a room to a newer version
+	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
+	PerformAdminEvacuateRoom(
+		ctx context.Context,
+		req *PerformAdminEvacuateRoomRequest,
+		res *PerformAdminEvacuateRoomResponse,
+	)
+	PerformPeek(ctx context.Context, req *PerformPeekRequest, res *PerformPeekResponse)
+	PerformUnpeek(ctx context.Context, req *PerformUnpeekRequest, res *PerformUnpeekResponse)
+	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
+	PerformJoin(ctx context.Context, req *PerformJoinRequest, res *PerformJoinResponse)
+	PerformLeave(ctx context.Context, req *PerformLeaveRequest, res *PerformLeaveResponse) error
+	PerformPublish(ctx context.Context, req *PerformPublishRequest, res *PerformPublishResponse)
+	// PerformForget forgets a rooms history for a specific user
+	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
+	SetRoomAlias(ctx context.Context, req *SetRoomAliasRequest, res *SetRoomAliasResponse) error
+	RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error
+}
diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go
index 5491d36b3..9f7a09ddd 100644
--- a/roomserver/api/wrapper.go
+++ b/roomserver/api/wrapper.go
@@ -16,7 +16,6 @@ package api
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -24,7 +23,7 @@ import (
 
 // SendEvents to the roomserver The events are written with KindNew.
 func SendEvents(
-	ctx context.Context, rsAPI RoomserverInternalAPI,
+	ctx context.Context, rsAPI InputRoomEventsAPI,
 	kind Kind, events []*gomatrixserverlib.HeaderedEvent,
 	origin gomatrixserverlib.ServerName,
 	sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID,
@@ -47,7 +46,7 @@ func SendEvents(
 // with the state at the event as KindOutlier before it. Will not send any event that is
 // marked as `true` in haveEventIDs.
 func SendEventWithState(
-	ctx context.Context, rsAPI RoomserverInternalAPI, kind Kind,
+	ctx context.Context, rsAPI InputRoomEventsAPI, kind Kind,
 	state *gomatrixserverlib.RespState, event *gomatrixserverlib.HeaderedEvent,
 	origin gomatrixserverlib.ServerName, haveEventIDs map[string]bool, async bool,
 ) error {
@@ -83,7 +82,7 @@ func SendEventWithState(
 
 // SendInputRoomEvents to the roomserver.
 func SendInputRoomEvents(
-	ctx context.Context, rsAPI RoomserverInternalAPI,
+	ctx context.Context, rsAPI InputRoomEventsAPI,
 	ires []InputRoomEvent, async bool,
 ) error {
 	request := InputRoomEventsRequest{
@@ -95,37 +94,8 @@ func SendInputRoomEvents(
 	return response.Err()
 }
 
-// SendInvite event to the roomserver.
-// This should only be needed for invite events that occur outside of a known room.
-// If we are in the room then the event should be sent using the SendEvents method.
-func SendInvite(
-	ctx context.Context,
-	rsAPI RoomserverInternalAPI, inviteEvent *gomatrixserverlib.HeaderedEvent,
-	inviteRoomState []gomatrixserverlib.InviteV2StrippedState,
-	sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID,
-) error {
-	// Start by sending the invite request into the roomserver. This will
-	// trigger the federation request amongst other things if needed.
-	request := &PerformInviteRequest{
-		Event:           inviteEvent,
-		InviteRoomState: inviteRoomState,
-		RoomVersion:     inviteEvent.RoomVersion,
-		SendAsServer:    string(sendAsServer),
-		TransactionID:   txnID,
-	}
-	response := &PerformInviteResponse{}
-	if err := rsAPI.PerformInvite(ctx, request, response); err != nil {
-		return fmt.Errorf("rsAPI.PerformInvite: %w", err)
-	}
-	if response.Error != nil {
-		return response.Error
-	}
-
-	return nil
-}
-
 // GetEvent returns the event or nil, even on errors.
-func GetEvent(ctx context.Context, rsAPI RoomserverInternalAPI, eventID string) *gomatrixserverlib.HeaderedEvent {
+func GetEvent(ctx context.Context, rsAPI QueryEventsAPI, eventID string) *gomatrixserverlib.HeaderedEvent {
 	var res QueryEventsByIDResponse
 	err := rsAPI.QueryEventsByID(ctx, &QueryEventsByIDRequest{
 		EventIDs: []string{eventID},
@@ -141,7 +111,7 @@ func GetEvent(ctx context.Context, rsAPI RoomserverInternalAPI, eventID string)
 }
 
 // GetStateEvent returns the current state event in the room or nil.
-func GetStateEvent(ctx context.Context, rsAPI RoomserverInternalAPI, roomID string, tuple gomatrixserverlib.StateKeyTuple) *gomatrixserverlib.HeaderedEvent {
+func GetStateEvent(ctx context.Context, rsAPI QueryEventsAPI, roomID string, tuple gomatrixserverlib.StateKeyTuple) *gomatrixserverlib.HeaderedEvent {
 	var res QueryCurrentStateResponse
 	err := rsAPI.QueryCurrentState(ctx, &QueryCurrentStateRequest{
 		RoomID:      roomID,
@@ -175,7 +145,7 @@ func IsServerBannedFromRoom(ctx context.Context, rsAPI RoomserverInternalAPI, ro
 // PopulatePublicRooms extracts PublicRoom information for all the provided room IDs. The IDs are not checked to see if they are visible in the
 // published room directory.
 // due to lots of switches
-func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI RoomserverInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
+func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkStateContentAPI) ([]gomatrixserverlib.PublicRoom, error) {
 	avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
 	nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
 	canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
diff --git a/userapi/api/api.go b/userapi/api/api.go
index 6f00fe44f..928b91e6d 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -26,44 +26,77 @@ import (
 
 // UserInternalAPI is the internal API for information about users and devices.
 type UserInternalAPI interface {
-	LoginTokenInternalAPI
 	UserProfileAPI
-	UserRegisterAPI
-	UserAccountAPI
-	UserThreePIDAPI
 	QueryAcccessTokenAPI
+
+	AppserviceUserAPI
 	SyncUserAPI
-
-	InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error
-
-	PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error
-	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error
-	PerformPusherSet(ctx context.Context, req *PerformPusherSetRequest, res *struct{}) error
-	PerformPusherDeletion(ctx context.Context, req *PerformPusherDeletionRequest, res *struct{}) error
-	PerformPushRulesPut(ctx context.Context, req *PerformPushRulesPutRequest, res *struct{}) error
-
-	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse)
+	ClientUserAPI
+	MediaUserAPI
 
 	QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error
-	QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error
-	QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error
-	QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error
 }
 
 type QueryAcccessTokenAPI interface {
 	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
 }
 
+type UserLoginAPI interface {
+	QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error
+}
+
+type AppserviceUserAPI interface {
+	PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error
+	PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
+}
+
+type MediaUserAPI interface {
+	QueryAcccessTokenAPI
+}
+
 type SyncUserAPI interface {
+	QueryAcccessTokenAPI
 	QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error
-	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
-	PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error
 	PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error
 	PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error
 	QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error
 	QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error
 }
 
+type ClientUserAPI interface {
+	QueryAcccessTokenAPI
+	LoginTokenInternalAPI
+	UserLoginAPI
+	QueryNumericLocalpart(ctx context.Context, res *QueryNumericLocalpartResponse) error
+	QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error
+	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
+	QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error
+	QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error
+	QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error
+	QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error
+	PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error
+	PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
+	PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error
+	PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error
+	PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error
+	PerformPusherDeletion(ctx context.Context, req *PerformPusherDeletionRequest, res *struct{}) error
+	PerformPusherSet(ctx context.Context, req *PerformPusherSetRequest, res *struct{}) error
+	PerformPushRulesPut(ctx context.Context, req *PerformPushRulesPutRequest, res *struct{}) error
+	PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error
+	PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error
+	SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error
+	SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error
+	QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error
+	InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error
+	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error
+	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse)
+
+	QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error
+	QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error
+	PerformForgetThreePID(ctx context.Context, req *PerformForgetThreePIDRequest, res *struct{}) error
+	PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error
+}
+
 type UserDirectoryProvider interface {
 	QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error
 }
@@ -72,31 +105,6 @@ type UserDirectoryProvider interface {
 type UserProfileAPI interface {
 	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
 	QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error
-	SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error
-	SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error
-}
-
-// UserRegisterAPI defines functions for registering accounts
-type UserRegisterAPI interface {
-	QueryNumericLocalpart(ctx context.Context, res *QueryNumericLocalpartResponse) error
-	QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error
-	PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error
-	PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
-}
-
-// UserAccountAPI defines functions for changing an account
-type UserAccountAPI interface {
-	PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error
-	PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error
-	QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error
-}
-
-// UserThreePIDAPI defines functions for 3PID
-type UserThreePIDAPI interface {
-	QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error
-	QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error
-	PerformForgetThreePID(ctx context.Context, req *PerformForgetThreePIDRequest, res *struct{}) error
-	PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error
 }
 
 type PerformKeyBackupRequest struct {

From 530fd488a91dd1644799920328a12810b70b1b49 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 5 May 2022 13:29:39 +0100
Subject: [PATCH 075/391] Don't log consumer errors on shutdown

---
 roomserver/internal/input/input.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 1fea6ef06..10c210220 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -202,7 +202,7 @@ func (w *worker) _next() {
 			return
 		}
 
-	case context.DeadlineExceeded:
+	case context.DeadlineExceeded, context.Canceled:
 		// The context exceeded, so we've been waiting for more than a
 		// minute for activity in this room. At this point we will shut
 		// down the subscriber to free up resources. It'll get started

From 42f35a57ac82e78e7035547504806733089f21a0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 5 May 2022 13:42:12 +0100
Subject: [PATCH 076/391] Update table names for user API stats table

---
 userapi/storage/postgres/stats_table.go | 14 +++++++-------
 userapi/storage/sqlite3/stats_table.go  | 14 +++++++-------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/userapi/storage/postgres/stats_table.go b/userapi/storage/postgres/stats_table.go
index f71900015..c0b317503 100644
--- a/userapi/storage/postgres/stats_table.go
+++ b/userapi/storage/postgres/stats_table.go
@@ -30,7 +30,7 @@ import (
 )
 
 const userDailyVisitsSchema = `
-CREATE TABLE IF NOT EXISTS user_daily_visits (
+CREATE TABLE IF NOT EXISTS userapi_daily_visits (
     localpart TEXT NOT NULL,
 	device_id TEXT NOT NULL,
 	timestamp BIGINT NOT NULL,
@@ -38,9 +38,9 @@ CREATE TABLE IF NOT EXISTS user_daily_visits (
 );
 
 -- Device IDs and timestamp must be unique for a given user per day
-CREATE UNIQUE INDEX IF NOT EXISTS localpart_device_timestamp_idx ON user_daily_visits(localpart, device_id, timestamp);
-CREATE INDEX IF NOT EXISTS timestamp_idx ON user_daily_visits(timestamp);
-CREATE INDEX IF NOT EXISTS localpart_timestamp_idx ON user_daily_visits(localpart, timestamp);
+CREATE UNIQUE INDEX IF NOT EXISTS userapi_daily_visits_localpart_device_timestamp_idx ON userapi_daily_visits(localpart, device_id, timestamp);
+CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_visits(timestamp);
+CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp);
 `
 
 const countUsersLastSeenAfterSQL = "" +
@@ -112,7 +112,7 @@ FROM
 				WHEN LOWER(user_agent) LIKE '%%mozilla%%' OR LOWER(user_agent) LIKE '%%gecko%%' THEN 'web'
 				ELSE 'unknown'
 			END as client_type
-		FROM user_daily_visits
+		FROM userapi_daily_visits
 		WHERE timestamp > $1 AND timestamp < $2
 		GROUP BY localpart, client_type
 		HAVING max(timestamp) - min(timestamp) > $3
@@ -141,11 +141,11 @@ SELECT user_type, COUNT(*) AS count FROM (
 
 // account_type 1 = users; 3 = admins
 const updateUserDailyVisitsSQL = `
-INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent)
+INSERT INTO userapi_daily_visits(localpart, device_id, timestamp, user_agent)
 	SELECT u.localpart, u.device_id, $1, MAX(u.user_agent)
 	FROM device_devices AS u
 	LEFT JOIN (
-		SELECT localpart, device_id, timestamp FROM user_daily_visits
+		SELECT localpart, device_id, timestamp FROM userapi_daily_visits
 		WHERE timestamp = $1
 	) udv
 	ON u.localpart = udv.localpart AND u.device_id = udv.device_id
diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go
index af4c7ff98..e00ed417b 100644
--- a/userapi/storage/sqlite3/stats_table.go
+++ b/userapi/storage/sqlite3/stats_table.go
@@ -30,7 +30,7 @@ import (
 )
 
 const userDailyVisitsSchema = `
-CREATE TABLE IF NOT EXISTS user_daily_visits (
+CREATE TABLE IF NOT EXISTS userapi_daily_visits (
     localpart TEXT NOT NULL,
 	device_id TEXT NOT NULL,
 	timestamp BIGINT NOT NULL,
@@ -38,9 +38,9 @@ CREATE TABLE IF NOT EXISTS user_daily_visits (
 );
 
 -- Device IDs and timestamp must be unique for a given user per day
-CREATE UNIQUE INDEX IF NOT EXISTS localpart_device_timestamp_idx ON user_daily_visits(localpart, device_id, timestamp);
-CREATE INDEX IF NOT EXISTS timestamp_idx ON user_daily_visits(timestamp);
-CREATE INDEX IF NOT EXISTS localpart_timestamp_idx ON user_daily_visits(localpart, timestamp);
+CREATE UNIQUE INDEX IF NOT EXISTS userapi_daily_visits_localpart_device_timestamp_idx ON userapi_daily_visits(localpart, device_id, timestamp);
+CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_visits(timestamp);
+CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp);
 `
 
 const countUsersLastSeenAfterSQL = "" +
@@ -116,7 +116,7 @@ FROM
 				WHEN LOWER(user_agent) LIKE '%%mozilla%%' OR LOWER(user_agent) LIKE '%%gecko%%' THEN 'web'
 				ELSE 'unknown'
 			END as client_type
-		FROM user_daily_visits
+		FROM userapi_daily_visits
 		WHERE timestamp > $1 AND timestamp < $2
 		GROUP BY localpart, client_type
 		HAVING max(timestamp) - min(timestamp) > $3
@@ -145,11 +145,11 @@ SELECT user_type, COUNT(*) AS count FROM (
 
 // account_type 1 = users; 3 = admins
 const updateUserDailyVisitsSQL = `
-INSERT INTO user_daily_visits(localpart, device_id, timestamp, user_agent)
+INSERT INTO userapi_daily_visits(localpart, device_id, timestamp, user_agent)
 	SELECT u.localpart, u.device_id, $1, MAX(u.user_agent)
 	FROM device_devices AS u
 	LEFT JOIN (
-		SELECT localpart, device_id, timestamp FROM user_daily_visits
+		SELECT localpart, device_id, timestamp FROM userapi_daily_visits
 		WHERE timestamp = $1
 	) udv
 	ON u.localpart = udv.localpart AND u.device_id = udv.device_id

From e4da04e75b4cba1c9afb63b9973444e1da12021b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 5 May 2022 14:06:05 +0100
Subject: [PATCH 077/391] Update to matrix-org/gomatrixserverlib#303

---
 go.mod             | 2 +-
 go.sum             | 4 ++--
 setup/base/base.go | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 56f5dcbbb..b6070b94f 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220505092512-c4ceb4751ac2
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220505130352-f72a63510060
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index fccda40c4..cbea7319e 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220505092512-c4ceb4751ac2 h1:5/Y4BpiMk1D/l/HkJz8Ng8bLBz1BHwV6V4e+yMNySzk=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220505092512-c4ceb4751ac2/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220505130352-f72a63510060 h1:tYi4mCOWgVLt8mpkG1LFRKcMfSTwp5NQ5wBKdtaxO9s=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220505130352-f72a63510060/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/setup/base/base.go b/setup/base/base.go
index 3641ad780..9326be1c0 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -331,6 +331,7 @@ func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client {
 	}
 	opts := []gomatrixserverlib.ClientOption{
 		gomatrixserverlib.WithSkipVerify(b.Cfg.FederationAPI.DisableTLSValidation),
+		gomatrixserverlib.WithWellKnownSRVLookups(true),
 	}
 	if b.Cfg.Global.DNSCache.Enabled {
 		opts = append(opts, gomatrixserverlib.WithDNSCache(b.DNSCache))

From 9957752a9d60d4519cc0b7e8b9b40a781240c27d Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 5 May 2022 19:30:38 +0100
Subject: [PATCH 078/391] Define component interfaces based on consumers (2/2) 
 (#2425)

* convert remaining interfaces

* Tidy up the userapi interfaces
---
 clientapi/clientapi.go                        |  2 +-
 clientapi/routing/routing.go                  |  2 +-
 clientapi/routing/userdirectory.go            |  2 +-
 cmd/dendrite-demo-pinecone/users/users.go     |  4 +-
 federationapi/api/api.go                      | 62 ++++++--------
 federationapi/federationapi.go                |  2 +-
 federationapi/routing/backfill.go             |  2 +-
 federationapi/routing/eventauth.go            |  2 +-
 federationapi/routing/events.go               |  6 +-
 federationapi/routing/invite.go               |  6 +-
 federationapi/routing/join.go                 |  4 +-
 federationapi/routing/leave.go                |  4 +-
 federationapi/routing/missingevents.go        |  2 +-
 federationapi/routing/openid.go               |  2 +-
 federationapi/routing/peek.go                 |  2 +-
 federationapi/routing/profile.go              |  2 +-
 federationapi/routing/publicrooms.go          |  6 +-
 federationapi/routing/query.go                |  2 +-
 federationapi/routing/routing.go              |  6 +-
 federationapi/routing/send.go                 |  4 +-
 federationapi/routing/send_test.go            |  2 +-
 federationapi/routing/state.go                |  6 +-
 federationapi/routing/threepid.go             | 12 +--
 keyserver/api/api.go                          |  9 ++-
 roomserver/api/api.go                         | 80 ++++++++-----------
 roomserver/api/api_trace.go                   |  2 +-
 roomserver/api/wrapper.go                     |  2 +-
 roomserver/internal/api.go                    |  4 +-
 roomserver/internal/input/input.go            |  2 +-
 roomserver/internal/input/input_missing.go    |  2 +-
 .../internal/perform/perform_backfill.go      |  6 +-
 roomserver/internal/perform/perform_invite.go |  2 +-
 roomserver/internal/perform/perform_join.go   |  2 +-
 roomserver/internal/perform/perform_leave.go  |  2 +-
 roomserver/internal/perform/perform_peek.go   |  2 +-
 roomserver/internal/perform/perform_unpeek.go |  2 +-
 roomserver/inthttp/client.go                  |  2 +-
 setup/monolith.go                             |  2 +-
 userapi/api/api.go                            | 38 +++++----
 userapi/consumers/syncapi_streamevent.go      |  6 +-
 userapi/internal/api.go                       |  2 +-
 userapi/userapi.go                            |  4 +-
 42 files changed, 153 insertions(+), 162 deletions(-)

diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index 957d082a6..ad4609080 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -38,7 +38,7 @@ func AddPublicRoutes(
 	transactionsCache *transactions.Cache,
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
-	userDirectoryProvider userapi.UserDirectoryProvider,
+	userDirectoryProvider userapi.QuerySearchProfilesAPI,
 	keyAPI keyserverAPI.ClientKeyAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 ) {
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 6da467073..f9f71ed7a 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -53,7 +53,7 @@ func Setup(
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	userAPI userapi.ClientUserAPI,
-	userDirectoryProvider userapi.UserDirectoryProvider,
+	userDirectoryProvider userapi.QuerySearchProfilesAPI,
 	federation *gomatrixserverlib.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
diff --git a/clientapi/routing/userdirectory.go b/clientapi/routing/userdirectory.go
index 4d41da734..f311457a0 100644
--- a/clientapi/routing/userdirectory.go
+++ b/clientapi/routing/userdirectory.go
@@ -36,7 +36,7 @@ func SearchUserDirectory(
 	device *userapi.Device,
 	userAPI userapi.ClientUserAPI,
 	rsAPI api.ClientRoomserverAPI,
-	provider userapi.UserDirectoryProvider,
+	provider userapi.QuerySearchProfilesAPI,
 	serverName gomatrixserverlib.ServerName,
 	searchString string,
 	limit int,
diff --git a/cmd/dendrite-demo-pinecone/users/users.go b/cmd/dendrite-demo-pinecone/users/users.go
index ebfb5cbe3..fc66bf299 100644
--- a/cmd/dendrite-demo-pinecone/users/users.go
+++ b/cmd/dendrite-demo-pinecone/users/users.go
@@ -37,7 +37,7 @@ import (
 type PineconeUserProvider struct {
 	r         *pineconeRouter.Router
 	s         *pineconeSessions.Sessions
-	userAPI   userapi.UserProfileAPI
+	userAPI   userapi.QuerySearchProfilesAPI
 	fedClient *gomatrixserverlib.FederationClient
 }
 
@@ -46,7 +46,7 @@ const PublicURL = "/_matrix/p2p/profiles"
 func NewPineconeUserProvider(
 	r *pineconeRouter.Router,
 	s *pineconeSessions.Sessions,
-	userAPI userapi.UserProfileAPI,
+	userAPI userapi.QuerySearchProfilesAPI,
 	fedClient *gomatrixserverlib.FederationClient,
 ) *PineconeUserProvider {
 	p := &PineconeUserProvider{
diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index ce6a0f2ed..87b037187 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -14,17 +14,13 @@ import (
 // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
 // this interface are of type FederationClientError
 type FederationClient interface {
-	gomatrixserverlib.BackfillClient
 	gomatrixserverlib.FederatedStateClient
 	GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
 	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
 	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
-	GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
 	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
 	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
 	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
-	GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
-	LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
 // FederationClientError is returned from FederationClient methods in the event of a problem.
@@ -43,17 +39,10 @@ type FederationInternalAPI interface {
 	FederationClient
 	gomatrixserverlib.KeyDatabase
 	ClientFederationAPI
-
-	KeyRing() *gomatrixserverlib.KeyRing
+	RoomserverFederationAPI
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
 
-	// PerformDirectoryLookup looks up a remote room ID from a room alias.
-	PerformDirectoryLookup(
-		ctx context.Context,
-		request *PerformDirectoryLookupRequest,
-		response *PerformDirectoryLookupResponse,
-	) error
 	// Query the server names of the joined hosts in a room.
 	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
 	// containing only the server names (without information for membership events).
@@ -63,30 +52,6 @@ type FederationInternalAPI interface {
 		request *QueryJoinedHostServerNamesInRoomRequest,
 		response *QueryJoinedHostServerNamesInRoomResponse,
 	) error
-	// Handle an instruction to make_join & send_join with a remote server.
-	PerformJoin(
-		ctx context.Context,
-		request *PerformJoinRequest,
-		response *PerformJoinResponse,
-	)
-	// Handle an instruction to peek a room on a remote server.
-	PerformOutboundPeek(
-		ctx context.Context,
-		request *PerformOutboundPeekRequest,
-		response *PerformOutboundPeekResponse,
-	) error
-	// Handle an instruction to make_leave & send_leave with a remote server.
-	PerformLeave(
-		ctx context.Context,
-		request *PerformLeaveRequest,
-		response *PerformLeaveResponse,
-	) error
-	// Handle sending an invite to a remote server.
-	PerformInvite(
-		ctx context.Context,
-		request *PerformInviteRequest,
-		response *PerformInviteResponse,
-	) error
 	// Notifies the federation sender that these servers may be online and to retry sending messages.
 	PerformServersAlive(
 		ctx context.Context,
@@ -105,6 +70,31 @@ type ClientFederationAPI interface {
 	QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
 }
 
+type RoomserverFederationAPI interface {
+	gomatrixserverlib.BackfillClient
+	gomatrixserverlib.FederatedStateClient
+	KeyRing() *gomatrixserverlib.KeyRing
+
+	// PerformDirectoryLookup looks up a remote room ID from a room alias.
+	PerformDirectoryLookup(ctx context.Context, request *PerformDirectoryLookupRequest, response *PerformDirectoryLookupResponse) error
+	// Handle an instruction to make_join & send_join with a remote server.
+	PerformJoin(ctx context.Context, request *PerformJoinRequest, response *PerformJoinResponse)
+	// Handle an instruction to make_leave & send_leave with a remote server.
+	PerformLeave(ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse) error
+	// Handle sending an invite to a remote server.
+	PerformInvite(ctx context.Context, request *PerformInviteRequest, response *PerformInviteResponse) error
+	// Handle an instruction to peek a room on a remote server.
+	PerformOutboundPeek(ctx context.Context, request *PerformOutboundPeekRequest, response *PerformOutboundPeekResponse) error
+	// Query the server names of the joined hosts in a room.
+	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
+	// containing only the server names (without information for membership events).
+	// The response will include this server if they are joined to the room.
+	QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
+	GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
+	GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
+	LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
+}
+
 type QueryServerKeysRequest struct {
 	ServerName      gomatrixserverlib.ServerName
 	KeyIDToCriteria map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyNotaryQueryCriteria
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index c627aab5c..632994db9 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -49,7 +49,7 @@ func AddPublicRoutes(
 	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
 	federationAPI federationAPI.FederationInternalAPI,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	servers federationAPI.ServersInRoomProvider,
diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go
index 82f6cbabf..7b9ca66f6 100644
--- a/federationapi/routing/backfill.go
+++ b/federationapi/routing/backfill.go
@@ -33,7 +33,7 @@ import (
 func Backfill(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID string,
 	cfg *config.FederationAPI,
 ) util.JSONResponse {
diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go
index e83cb8ad2..868785a9b 100644
--- a/federationapi/routing/eventauth.go
+++ b/federationapi/routing/eventauth.go
@@ -26,7 +26,7 @@ import (
 func GetEventAuth(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID string,
 	eventID string,
 ) util.JSONResponse {
diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go
index 312ef9f8e..23796edfa 100644
--- a/federationapi/routing/events.go
+++ b/federationapi/routing/events.go
@@ -29,7 +29,7 @@ import (
 func GetEvent(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	eventID string,
 	origin gomatrixserverlib.ServerName,
 ) util.JSONResponse {
@@ -56,7 +56,7 @@ func GetEvent(
 func allowedToSeeEvent(
 	ctx context.Context,
 	origin gomatrixserverlib.ServerName,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	eventID string,
 ) *util.JSONResponse {
 	var authResponse api.QueryServerAllowedToSeeEventResponse
@@ -82,7 +82,7 @@ func allowedToSeeEvent(
 }
 
 // fetchEvent fetches the event without auth checks. Returns an error if the event cannot be found.
-func fetchEvent(ctx context.Context, rsAPI api.RoomserverInternalAPI, eventID string) (*gomatrixserverlib.Event, *util.JSONResponse) {
+func fetchEvent(ctx context.Context, rsAPI api.FederationRoomserverAPI, eventID string) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	var eventsResponse api.QueryEventsByIDResponse
 	err := rsAPI.QueryEventsByID(
 		ctx,
diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go
index 25faff0cb..a5797645e 100644
--- a/federationapi/routing/invite.go
+++ b/federationapi/routing/invite.go
@@ -35,7 +35,7 @@ func InviteV2(
 	roomID string,
 	eventID string,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
 	inviteReq := gomatrixserverlib.InviteV2Request{}
@@ -72,7 +72,7 @@ func InviteV1(
 	roomID string,
 	eventID string,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
 	roomVer := gomatrixserverlib.RoomVersionV1
@@ -110,7 +110,7 @@ func processInvite(
 	roomID string,
 	eventID string,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
 
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 495b8c914..767699728 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -34,7 +34,7 @@ func MakeJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID, userID string,
 	remoteVersions []gomatrixserverlib.RoomVersion,
 ) util.JSONResponse {
@@ -165,7 +165,7 @@ func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
 ) util.JSONResponse {
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index 0b83f04ae..54b2c3e84 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -30,7 +30,7 @@ func MakeLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID, userID string,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
@@ -122,7 +122,7 @@ func SendLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
 ) util.JSONResponse {
diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go
index b826d69c4..531cb9e28 100644
--- a/federationapi/routing/missingevents.go
+++ b/federationapi/routing/missingevents.go
@@ -34,7 +34,7 @@ type getMissingEventRequest struct {
 func GetMissingEvents(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
 	var gme getMissingEventRequest
diff --git a/federationapi/routing/openid.go b/federationapi/routing/openid.go
index 829dbccad..cbc75a9a7 100644
--- a/federationapi/routing/openid.go
+++ b/federationapi/routing/openid.go
@@ -30,7 +30,7 @@ type openIDUserInfoResponse struct {
 // GetOpenIDUserInfo implements GET /_matrix/federation/v1/openid/userinfo
 func GetOpenIDUserInfo(
 	httpReq *http.Request,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.FederationUserAPI,
 ) util.JSONResponse {
 	token := httpReq.URL.Query().Get("access_token")
 	if len(token) == 0 {
diff --git a/federationapi/routing/peek.go b/federationapi/routing/peek.go
index 827d1116d..bc4dac90f 100644
--- a/federationapi/routing/peek.go
+++ b/federationapi/routing/peek.go
@@ -29,7 +29,7 @@ func Peek(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID, peekID string,
 	remoteVersions []gomatrixserverlib.RoomVersion,
 ) util.JSONResponse {
diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go
index dbc209ce1..f672811af 100644
--- a/federationapi/routing/profile.go
+++ b/federationapi/routing/profile.go
@@ -29,7 +29,7 @@ import (
 // GetProfile implements GET /_matrix/federation/v1/query/profile
 func GetProfile(
 	httpReq *http.Request,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.FederationUserAPI,
 	cfg *config.FederationAPI,
 ) util.JSONResponse {
 	userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field")
diff --git a/federationapi/routing/publicrooms.go b/federationapi/routing/publicrooms.go
index a253f86eb..1a54f5a7d 100644
--- a/federationapi/routing/publicrooms.go
+++ b/federationapi/routing/publicrooms.go
@@ -23,7 +23,7 @@ type filter struct {
 }
 
 // GetPostPublicRooms implements GET and POST /publicRooms
-func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI) util.JSONResponse {
+func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.FederationRoomserverAPI) util.JSONResponse {
 	var request PublicRoomReq
 	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
 		return *fillErr
@@ -42,7 +42,7 @@ func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.RoomserverInterna
 }
 
 func publicRooms(
-	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
+	ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.FederationRoomserverAPI,
 ) (*gomatrixserverlib.RespPublicRooms, error) {
 
 	var response gomatrixserverlib.RespPublicRooms
@@ -111,7 +111,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 }
 
 // due to lots of switches
-func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.RoomserverInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
+func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.FederationRoomserverAPI) ([]gomatrixserverlib.PublicRoom, error) {
 	avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
 	nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
 	canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go
index 47d3b2df9..707b7b019 100644
--- a/federationapi/routing/query.go
+++ b/federationapi/routing/query.go
@@ -32,7 +32,7 @@ func RoomAliasToID(
 	httpReq *http.Request,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.FederationAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
 	senderAPI federationAPI.FederationInternalAPI,
 ) util.JSONResponse {
 	roomAlias := httpReq.FormValue("room_alias")
diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index 6d24c8b40..51adc279c 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -47,11 +47,11 @@ import (
 func Setup(
 	fedMux, keyMux, wkMux *mux.Router,
 	cfg *config.FederationAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fsAPI federationAPI.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.FederationUserAPI,
 	keyAPI keyserverAPI.KeyInternalAPI,
 	mscCfg *config.MSCs,
 	servers federationAPI.ServersInRoomProvider,
@@ -497,7 +497,7 @@ func Setup(
 
 func ErrorIfLocalServerNotInRoom(
 	ctx context.Context,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID string,
 ) *util.JSONResponse {
 	// Check if we think we're in this room. If we aren't then
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 2c01afb1b..b9b6d33b7 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -82,7 +82,7 @@ func Send(
 	request *gomatrixserverlib.FederationRequest,
 	txnID gomatrixserverlib.TransactionID,
 	cfg *config.FederationAPI,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	keyAPI keyapi.KeyInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
@@ -182,7 +182,7 @@ func Send(
 
 type txnReq struct {
 	gomatrixserverlib.Transaction
-	rsAPI                  api.RoomserverInternalAPI
+	rsAPI                  api.FederationRoomserverAPI
 	keyAPI                 keyapi.KeyInternalAPI
 	ourServerName          gomatrixserverlib.ServerName
 	keys                   gomatrixserverlib.JSONVerifier
diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go
index 8d2d85040..011d4e342 100644
--- a/federationapi/routing/send_test.go
+++ b/federationapi/routing/send_test.go
@@ -183,7 +183,7 @@ func (c *txnFedClient) LookupMissingEvents(ctx context.Context, s gomatrixserver
 	return c.getMissingEvents(missing)
 }
 
-func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq {
+func mustCreateTransaction(rsAPI api.FederationRoomserverAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq {
 	t := &txnReq{
 		rsAPI:      rsAPI,
 		keys:       &test.NopJSONVerifier{},
diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go
index e2b67776a..6fdce20ce 100644
--- a/federationapi/routing/state.go
+++ b/federationapi/routing/state.go
@@ -27,7 +27,7 @@ import (
 func GetState(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
 	eventID, err := parseEventIDParam(request)
@@ -50,7 +50,7 @@ func GetState(
 func GetStateIDs(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID string,
 ) util.JSONResponse {
 	eventID, err := parseEventIDParam(request)
@@ -97,7 +97,7 @@ func parseEventIDParam(
 func getState(
 	ctx context.Context,
 	request *gomatrixserverlib.FederationRequest,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	roomID string,
 	eventID string,
 ) (stateEvents, authEvents []*gomatrixserverlib.HeaderedEvent, errRes *util.JSONResponse) {
diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go
index 8ae7130c3..16f245cee 100644
--- a/federationapi/routing/threepid.go
+++ b/federationapi/routing/threepid.go
@@ -55,10 +55,10 @@ var (
 
 // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind
 func CreateInvitesFrom3PIDInvites(
-	req *http.Request, rsAPI api.RoomserverInternalAPI,
+	req *http.Request, rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
 	federation *gomatrixserverlib.FederationClient,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.FederationUserAPI,
 ) util.JSONResponse {
 	var body invites
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
@@ -105,7 +105,7 @@ func ExchangeThirdPartyInvite(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -203,10 +203,10 @@ func ExchangeThirdPartyInvite(
 // Returns an error if there was a problem building the event or fetching the
 // necessary data to do so.
 func createInviteFrom3PIDInvite(
-	ctx context.Context, rsAPI api.RoomserverInternalAPI,
+	ctx context.Context, rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
 	inv invite, federation *gomatrixserverlib.FederationClient,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.FederationUserAPI,
 ) (*gomatrixserverlib.Event, error) {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
 	verRes := api.QueryRoomVersionForRoomResponse{}
@@ -270,7 +270,7 @@ func createInviteFrom3PIDInvite(
 // Returns an error if something failed during the process.
 func buildMembershipEvent(
 	ctx context.Context,
-	builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI,
+	builder *gomatrixserverlib.EventBuilder, rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
 ) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index 5564eb271..6cee2c014 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -29,12 +29,13 @@ import (
 type KeyInternalAPI interface {
 	SyncKeyAPI
 	ClientKeyAPI
+	UserKeyAPI
+
 	// SetUserAPI assigns a user API to query when extracting device names.
 	SetUserAPI(i userapi.UserInternalAPI)
 	// InputDeviceListUpdate from a federated server EDU
 	InputDeviceListUpdate(ctx context.Context, req *InputDeviceListUpdateRequest, res *InputDeviceListUpdateResponse)
 
-	PerformDeleteKeys(ctx context.Context, req *PerformDeleteKeysRequest, res *PerformDeleteKeysResponse)
 	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse)
 	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse)
 }
@@ -49,6 +50,12 @@ type ClientKeyAPI interface {
 	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse)
 }
 
+// API functions required by the userapi
+type UserKeyAPI interface {
+	PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse)
+	PerformDeleteKeys(ctx context.Context, req *PerformDeleteKeysRequest, res *PerformDeleteKeysResponse)
+}
+
 // API functions required by the syncapi
 type SyncKeyAPI interface {
 	QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse)
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 33c3d157b..7e1e568ce 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -19,10 +19,12 @@ type RoomserverInternalAPI interface {
 	SyncRoomserverAPI
 	AppserviceRoomserverAPI
 	ClientRoomserverAPI
+	UserRoomserverAPI
+	FederationRoomserverAPI
 
 	// needed to avoid chicken and egg scenario when setting up the
 	// interdependencies between the roomserver and other input APIs
-	SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing)
+	SetFederationAPI(fsAPI fsAPI.RoomserverFederationAPI, keyRing *gomatrixserverlib.KeyRing)
 	SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI)
 	SetUserAPI(userAPI userapi.UserInternalAPI)
 
@@ -62,48 +64,6 @@ type RoomserverInternalAPI interface {
 		res *PerformPublishResponse,
 	)
 
-	PerformInboundPeek(
-		ctx context.Context,
-		req *PerformInboundPeekRequest,
-		res *PerformInboundPeekResponse,
-	) error
-
-	QueryPublishedRooms(
-		ctx context.Context,
-		req *QueryPublishedRoomsRequest,
-		res *QueryPublishedRoomsResponse,
-	) error
-
-	// Query if we think we're still in a room.
-	QueryServerJoinedToRoom(
-		ctx context.Context,
-		req *QueryServerJoinedToRoomRequest,
-		res *QueryServerJoinedToRoomResponse,
-	) error
-
-	// Query whether a server is allowed to see an event
-	QueryServerAllowedToSeeEvent(
-		ctx context.Context,
-		req *QueryServerAllowedToSeeEventRequest,
-		res *QueryServerAllowedToSeeEventResponse,
-	) error
-
-	// Query missing events for a room from roomserver
-	QueryMissingEvents(
-		ctx context.Context,
-		req *QueryMissingEventsRequest,
-		res *QueryMissingEventsResponse,
-	) error
-
-	// Query to get state and auth chain for a (potentially hypothetical) event.
-	// Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate
-	// the state and auth chain to return.
-	QueryStateAndAuthChain(
-		ctx context.Context,
-		req *QueryStateAndAuthChainRequest,
-		res *QueryStateAndAuthChainResponse,
-	) error
-
 	// QueryAuthChain returns the entire auth chain for the event IDs given.
 	// The response includes the events in the request.
 	// Omits without error for any missing auth events. There will be no duplicates.
@@ -115,8 +75,6 @@ type RoomserverInternalAPI interface {
 
 	// QueryRoomsForUser retrieves a list of room IDs matching the given query.
 	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
-	// QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs.
-	QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error
 
 	// PerformRoomUpgrade upgrades a room to a newer version
 	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
@@ -285,3 +243,35 @@ type ClientRoomserverAPI interface {
 	SetRoomAlias(ctx context.Context, req *SetRoomAliasRequest, res *SetRoomAliasResponse) error
 	RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error
 }
+
+type UserRoomserverAPI interface {
+	QueryLatestEventsAndStateAPI
+	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
+	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
+}
+
+type FederationRoomserverAPI interface {
+	InputRoomEventsAPI
+	QueryLatestEventsAndStateAPI
+	QueryBulkStateContentAPI
+	// QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs.
+	QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error
+	QueryRoomVersionForRoom(ctx context.Context, req *QueryRoomVersionForRoomRequest, res *QueryRoomVersionForRoomResponse) error
+	GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error
+	QueryEventsByID(ctx context.Context, req *QueryEventsByIDRequest, res *QueryEventsByIDResponse) error
+	// Query to get state and auth chain for a (potentially hypothetical) event.
+	// Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate
+	// the state and auth chain to return.
+	QueryStateAndAuthChain(ctx context.Context, req *QueryStateAndAuthChainRequest, res *QueryStateAndAuthChainResponse) error
+	// Query if we think we're still in a room.
+	QueryServerJoinedToRoom(ctx context.Context, req *QueryServerJoinedToRoomRequest, res *QueryServerJoinedToRoomResponse) error
+	QueryPublishedRooms(ctx context.Context, req *QueryPublishedRoomsRequest, res *QueryPublishedRoomsResponse) error
+	// Query missing events for a room from roomserver
+	QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error
+	// Query whether a server is allowed to see an event
+	QueryServerAllowedToSeeEvent(ctx context.Context, req *QueryServerAllowedToSeeEventRequest, res *QueryServerAllowedToSeeEventResponse) error
+	PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error
+	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
+	// Query a given amount (or less) of events prior to a given set of events.
+	PerformBackfill(ctx context.Context, req *PerformBackfillRequest, res *PerformBackfillResponse) error
+}
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index 61c06e886..bc60999e6 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -19,7 +19,7 @@ type RoomserverInternalAPITrace struct {
 	Impl RoomserverInternalAPI
 }
 
-func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) {
+func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.RoomserverFederationAPI, keyRing *gomatrixserverlib.KeyRing) {
 	t.Impl.SetFederationAPI(fsAPI, keyRing)
 }
 
diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go
index 9f7a09ddd..344e9b079 100644
--- a/roomserver/api/wrapper.go
+++ b/roomserver/api/wrapper.go
@@ -129,7 +129,7 @@ func GetStateEvent(ctx context.Context, rsAPI QueryEventsAPI, roomID string, tup
 }
 
 // IsServerBannedFromRoom returns whether the server is banned from a room by server ACLs.
-func IsServerBannedFromRoom(ctx context.Context, rsAPI RoomserverInternalAPI, roomID string, serverName gomatrixserverlib.ServerName) bool {
+func IsServerBannedFromRoom(ctx context.Context, rsAPI FederationRoomserverAPI, roomID string, serverName gomatrixserverlib.ServerName) bool {
 	req := &QueryServerBannedFromRoomRequest{
 		ServerName: serverName,
 		RoomID:     roomID,
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index 267cd4099..dc0a0a718 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -43,7 +43,7 @@ type RoomserverInternalAPI struct {
 	ServerName             gomatrixserverlib.ServerName
 	KeyRing                gomatrixserverlib.JSONVerifier
 	ServerACLs             *acls.ServerACLs
-	fsAPI                  fsAPI.FederationInternalAPI
+	fsAPI                  fsAPI.RoomserverFederationAPI
 	asAPI                  asAPI.AppServiceQueryAPI
 	NATSClient             *nats.Conn
 	JetStream              nats.JetStreamContext
@@ -87,7 +87,7 @@ func NewRoomserverAPI(
 // SetFederationInputAPI passes in a federation input API reference so that we can
 // avoid the chicken-and-egg problem of both the roomserver input API and the
 // federation input API being interdependent.
-func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) {
+func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederationAPI, keyRing *gomatrixserverlib.KeyRing) {
 	r.fsAPI = fsAPI
 	r.KeyRing = keyRing
 
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 10c210220..600994c5a 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -82,7 +82,7 @@ type Inputer struct {
 	JetStream            nats.JetStreamContext
 	Durable              nats.SubOpt
 	ServerName           gomatrixserverlib.ServerName
-	FSAPI                fedapi.FederationInternalAPI
+	FSAPI                fedapi.RoomserverFederationAPI
 	KeyRing              gomatrixserverlib.JSONVerifier
 	ACLs                 *acls.ServerACLs
 	InputRoomEventTopic  string
diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index 2c958335d..9c70076c2 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -44,7 +44,7 @@ type missingStateReq struct {
 	roomInfo        *types.RoomInfo
 	inputer         *Inputer
 	keys            gomatrixserverlib.JSONVerifier
-	federation      fedapi.FederationInternalAPI
+	federation      fedapi.RoomserverFederationAPI
 	roomsMu         *internal.MutexByRoom
 	servers         []gomatrixserverlib.ServerName
 	hadEvents       map[string]bool
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 081f694a1..1bc4c75ce 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -38,7 +38,7 @@ const maxBackfillServers = 5
 type Backfiller struct {
 	ServerName gomatrixserverlib.ServerName
 	DB         storage.Database
-	FSAPI      federationAPI.FederationInternalAPI
+	FSAPI      federationAPI.RoomserverFederationAPI
 	KeyRing    gomatrixserverlib.JSONVerifier
 
 	// The servers which should be preferred above other servers when backfilling
@@ -228,7 +228,7 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom
 // backfillRequester implements gomatrixserverlib.BackfillRequester
 type backfillRequester struct {
 	db           storage.Database
-	fsAPI        federationAPI.FederationInternalAPI
+	fsAPI        federationAPI.RoomserverFederationAPI
 	thisServer   gomatrixserverlib.ServerName
 	preferServer map[gomatrixserverlib.ServerName]bool
 	bwExtrems    map[string][]string
@@ -240,7 +240,7 @@ type backfillRequester struct {
 }
 
 func newBackfillRequester(
-	db storage.Database, fsAPI federationAPI.FederationInternalAPI, thisServer gomatrixserverlib.ServerName,
+	db storage.Database, fsAPI federationAPI.RoomserverFederationAPI, thisServer gomatrixserverlib.ServerName,
 	bwExtrems map[string][]string, preferServers []gomatrixserverlib.ServerName,
 ) *backfillRequester {
 	preferServer := make(map[gomatrixserverlib.ServerName]bool)
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index 6111372d8..b0148a314 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -35,7 +35,7 @@ import (
 type Inviter struct {
 	DB      storage.Database
 	Cfg     *config.RoomServer
-	FSAPI   federationAPI.FederationInternalAPI
+	FSAPI   federationAPI.RoomserverFederationAPI
 	Inputer *input.Inputer
 }
 
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index a40f66d21..61a0206ef 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -38,7 +38,7 @@ import (
 type Joiner struct {
 	ServerName gomatrixserverlib.ServerName
 	Cfg        *config.RoomServer
-	FSAPI      fsAPI.FederationInternalAPI
+	FSAPI      fsAPI.RoomserverFederationAPI
 	RSAPI      rsAPI.RoomserverInternalAPI
 	DB         storage.Database
 
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index 5b4cd3c6f..b006843fb 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -37,7 +37,7 @@ import (
 type Leaver struct {
 	Cfg     *config.RoomServer
 	DB      storage.Database
-	FSAPI   fsAPI.FederationInternalAPI
+	FSAPI   fsAPI.RoomserverFederationAPI
 	UserAPI userapi.UserInternalAPI
 	Inputer *input.Inputer
 }
diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go
index 6a2c329b9..45e63888d 100644
--- a/roomserver/internal/perform/perform_peek.go
+++ b/roomserver/internal/perform/perform_peek.go
@@ -33,7 +33,7 @@ import (
 type Peeker struct {
 	ServerName gomatrixserverlib.ServerName
 	Cfg        *config.RoomServer
-	FSAPI      fsAPI.FederationInternalAPI
+	FSAPI      fsAPI.RoomserverFederationAPI
 	DB         storage.Database
 
 	Inputer *input.Inputer
diff --git a/roomserver/internal/perform/perform_unpeek.go b/roomserver/internal/perform/perform_unpeek.go
index 16b4eeaed..1057499cb 100644
--- a/roomserver/internal/perform/perform_unpeek.go
+++ b/roomserver/internal/perform/perform_unpeek.go
@@ -30,7 +30,7 @@ import (
 type Unpeeker struct {
 	ServerName gomatrixserverlib.ServerName
 	Cfg        *config.RoomServer
-	FSAPI      fsAPI.FederationInternalAPI
+	FSAPI      fsAPI.RoomserverFederationAPI
 	DB         storage.Database
 
 	Inputer *input.Inputer
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index 3b29001e9..4fc75ff41 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -87,7 +87,7 @@ func NewRoomserverClient(
 }
 
 // SetFederationInputAPI no-ops in HTTP client mode as there is no chicken/egg scenario
-func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.FederationInternalAPI, keyRing *gomatrixserverlib.KeyRing) {
+func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.RoomserverFederationAPI, keyRing *gomatrixserverlib.KeyRing) {
 }
 
 // SetAppserviceAPI no-ops in HTTP client mode as there is no chicken/egg scenario
diff --git a/setup/monolith.go b/setup/monolith.go
index e033c14d7..a0e850d83 100644
--- a/setup/monolith.go
+++ b/setup/monolith.go
@@ -47,7 +47,7 @@ type Monolith struct {
 
 	// Optional
 	ExtPublicRoomsProvider   api.ExtraPublicRoomsProvider
-	ExtUserDirectoryProvider userapi.UserDirectoryProvider
+	ExtUserDirectoryProvider userapi.QuerySearchProfilesAPI
 }
 
 // AddAllPublicRoutes attaches all public paths to the given router
diff --git a/userapi/api/api.go b/userapi/api/api.go
index 928b91e6d..dc8c12b74 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -26,34 +26,33 @@ import (
 
 // UserInternalAPI is the internal API for information about users and devices.
 type UserInternalAPI interface {
-	UserProfileAPI
-	QueryAcccessTokenAPI
-
 	AppserviceUserAPI
 	SyncUserAPI
 	ClientUserAPI
 	MediaUserAPI
+	FederationUserAPI
 
-	QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error
-}
-
-type QueryAcccessTokenAPI interface {
-	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
-}
-
-type UserLoginAPI interface {
-	QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error
+	QuerySearchProfilesAPI // used by p2p demos
 }
 
+// api functions required by the appservice api
 type AppserviceUserAPI interface {
 	PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error
 	PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
 }
 
+// api functions required by the media api
 type MediaUserAPI interface {
 	QueryAcccessTokenAPI
 }
 
+// api functions required by the federation api
+type FederationUserAPI interface {
+	QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error
+	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
+}
+
+// api functions required by the sync api
 type SyncUserAPI interface {
 	QueryAcccessTokenAPI
 	QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error
@@ -63,6 +62,7 @@ type SyncUserAPI interface {
 	QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error
 }
 
+// api functions required by the client api
 type ClientUserAPI interface {
 	QueryAcccessTokenAPI
 	LoginTokenInternalAPI
@@ -97,14 +97,18 @@ type ClientUserAPI interface {
 	PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error
 }
 
-type UserDirectoryProvider interface {
+// custom api functions required by pinecone / p2p demos
+type QuerySearchProfilesAPI interface {
 	QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error
 }
 
-// UserProfileAPI provides functions for getting user profiles
-type UserProfileAPI interface {
-	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
-	QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error
+// common function for creating authenticated endpoints (used in client/media/sync api)
+type QueryAcccessTokenAPI interface {
+	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
+}
+
+type UserLoginAPI interface {
+	QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error
 }
 
 type PerformKeyBackupRequest struct {
diff --git a/userapi/consumers/syncapi_streamevent.go b/userapi/consumers/syncapi_streamevent.go
index 9ef7b5083..7807c7637 100644
--- a/userapi/consumers/syncapi_streamevent.go
+++ b/userapi/consumers/syncapi_streamevent.go
@@ -29,7 +29,7 @@ type OutputStreamEventConsumer struct {
 	ctx          context.Context
 	cfg          *config.UserAPI
 	userAPI      api.UserInternalAPI
-	rsAPI        rsapi.RoomserverInternalAPI
+	rsAPI        rsapi.UserRoomserverAPI
 	jetstream    nats.JetStreamContext
 	durable      string
 	db           storage.Database
@@ -45,7 +45,7 @@ func NewOutputStreamEventConsumer(
 	store storage.Database,
 	pgClient pushgateway.Client,
 	userAPI api.UserInternalAPI,
-	rsAPI rsapi.RoomserverInternalAPI,
+	rsAPI rsapi.UserRoomserverAPI,
 	syncProducer *producers.SyncAPI,
 ) *OutputStreamEventConsumer {
 	return &OutputStreamEventConsumer{
@@ -455,7 +455,7 @@ func (s *OutputStreamEventConsumer) evaluatePushRules(ctx context.Context, event
 
 type ruleSetEvalContext struct {
 	ctx      context.Context
-	rsAPI    rsapi.RoomserverInternalAPI
+	rsAPI    rsapi.UserRoomserverAPI
 	mem      *localMembership
 	roomID   string
 	roomSize int
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 394bfa224..9d2f63c72 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -48,7 +48,7 @@ type UserInternalAPI struct {
 	ServerName           gomatrixserverlib.ServerName
 	// AppServices is the list of all registered AS
 	AppServices []config.ApplicationService
-	KeyAPI      keyapi.KeyInternalAPI
+	KeyAPI      keyapi.UserKeyAPI
 }
 
 func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error {
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 5b11665db..03a46807f 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -44,8 +44,8 @@ func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI) {
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite, cfg *config.UserAPI,
-	appServices []config.ApplicationService, keyAPI keyapi.KeyInternalAPI,
-	rsAPI rsapi.RoomserverInternalAPI, pgClient pushgateway.Client,
+	appServices []config.ApplicationService, keyAPI keyapi.UserKeyAPI,
+	rsAPI rsapi.UserRoomserverAPI, pgClient pushgateway.Client,
 ) api.UserInternalAPI {
 	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 

From 4705f5761e620e7f8a35151eeb2007e884847152 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 6 May 2022 08:26:24 +0200
Subject: [PATCH 079/391] Add FAQ entry for anonymous stats (#2419)

---
 docs/FAQ.md | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index 978212cce..47eaecf0f 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -74,3 +74,46 @@ If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated m
 ### Dendrite is running out of PostgreSQL database connections
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!
+
+### What is being reported when enabling anonymous stats?
+
+If anonymous stats reporting is enabled, the following data is send to the defined endpoint.
+
+```json
+{
+    "cpu_average": 0,
+    "daily_active_users": 97,
+    "daily_e2ee_messages": 0,
+    "daily_messages": 0,
+    "daily_sent_e2ee_messages": 0,
+    "daily_sent_messages": 0,
+    "daily_user_type_bridged": 2,
+    "daily_user_type_native": 97,
+    "database_engine": "Postgres",
+    "database_server_version": "11.14 (Debian 11.14-0+deb10u1)",
+    "federation_disabled": false,
+    "go_arch": "amd64",
+    "go_os": "linux",
+    "go_version": "go1.16.13",
+    "homeserver": "localhost:8800",
+    "log_level": "trace",
+    "memory_rss": 93452,
+    "monolith": true,
+    "monthly_active_users": 97,
+    "nats_embedded": true,
+    "nats_in_memory": true,
+    "num_cpu": 8,
+    "num_go_routine": 203,
+    "r30v2_users_all": 0,
+    "r30v2_users_android": 0,
+    "r30v2_users_electron": 0,
+    "r30v2_users_ios": 0,
+    "r30v2_users_web": 0,
+    "timestamp": 1651741851,
+    "total_nonbridged_users": 97,
+    "total_room_count": 0,
+    "total_users": 99,
+    "uptime_seconds": 30,
+    "version": "0.8.2"
+}
+```
\ No newline at end of file

From 85704eff207f7690d197172abb991ae1ac238239 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Fri, 6 May 2022 12:39:26 +0100
Subject: [PATCH 080/391] Clean up interface definitions (#2427)

* tidy up interfaces

* remove unused GetCreatorIDForAlias

* Add RoomserverUserAPI interface

* Define more interfaces

* Use AppServiceInternalAPI for consistent naming

* clean up federationapi constructor a bit

* Fix monolith in -http mode
---
 appservice/api/query.go                      |  36 +++---
 appservice/appservice.go                     |   4 +-
 appservice/inthttp/client.go                 |   2 +-
 appservice/inthttp/server.go                 |   2 +-
 clientapi/clientapi.go                       |   2 +-
 clientapi/routing/createroom.go              |   4 +-
 clientapi/routing/membership.go              |  16 +--
 clientapi/routing/profile.go                 |   8 +-
 clientapi/routing/routing.go                 |   2 +-
 clientapi/routing/server_notices.go          |   2 +-
 clientapi/routing/upgrade_room.go            |   2 +-
 cmd/dendrite-monolith-server/main.go         |   6 +-
 federationapi/api/api.go                     |  76 +++++-------
 federationapi/federationapi.go               |  17 ++-
 federationapi/federationapi_test.go          |   4 +-
 federationapi/internal/perform.go            |  30 ++---
 federationapi/inthttp/client.go              |  13 --
 federationapi/inthttp/server.go              |  14 ---
 federationapi/routing/devices.go             |   2 +-
 federationapi/routing/keys.go                |   4 +-
 federationapi/routing/routing.go             | 118 +++++++++++++++----
 federationapi/routing/send.go                |   4 +-
 internal/httputil/httpapi.go                 |  79 -------------
 keyserver/api/api.go                         |  18 ++-
 keyserver/internal/internal.go               |   4 +-
 keyserver/inthttp/client.go                  |   2 +-
 roomserver/api/alias.go                      |  12 --
 roomserver/api/api.go                        |  93 +--------------
 roomserver/api/api_trace.go                  |  14 +--
 roomserver/internal/alias.go                 |  19 ---
 roomserver/internal/api.go                   |   6 +-
 roomserver/internal/perform/perform_leave.go |   2 +-
 roomserver/inthttp/client.go                 |  17 +--
 roomserver/inthttp/server.go                 |  14 ---
 setup/base/base.go                           |   4 +-
 setup/monolith.go                            |   2 +-
 userapi/api/api.go                           |  11 ++
 37 files changed, 236 insertions(+), 429 deletions(-)

diff --git a/appservice/api/query.go b/appservice/api/query.go
index 6db8be85b..4d1cf9474 100644
--- a/appservice/api/query.go
+++ b/appservice/api/query.go
@@ -26,6 +26,23 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+// AppServiceInternalAPI is used to query user and room alias data from application
+// services
+type AppServiceInternalAPI interface {
+	// Check whether a room alias exists within any application service namespaces
+	RoomAliasExists(
+		ctx context.Context,
+		req *RoomAliasExistsRequest,
+		resp *RoomAliasExistsResponse,
+	) error
+	// Check whether a user ID exists within any application service namespaces
+	UserIDExists(
+		ctx context.Context,
+		req *UserIDExistsRequest,
+		resp *UserIDExistsResponse,
+	) error
+}
+
 // RoomAliasExistsRequest is a request to an application service
 // about whether a room alias exists
 type RoomAliasExistsRequest struct {
@@ -60,30 +77,13 @@ type UserIDExistsResponse struct {
 	UserIDExists bool `json:"exists"`
 }
 
-// AppServiceQueryAPI is used to query user and room alias data from application
-// services
-type AppServiceQueryAPI interface {
-	// Check whether a room alias exists within any application service namespaces
-	RoomAliasExists(
-		ctx context.Context,
-		req *RoomAliasExistsRequest,
-		resp *RoomAliasExistsResponse,
-	) error
-	// Check whether a user ID exists within any application service namespaces
-	UserIDExists(
-		ctx context.Context,
-		req *UserIDExistsRequest,
-		resp *UserIDExistsResponse,
-	) error
-}
-
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
 // TODO: Remove this, it's called from federationapi and clientapi but is a pure function
 func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
-	asAPI AppServiceQueryAPI,
+	asAPI AppServiceInternalAPI,
 	profileAPI userapi.ClientUserAPI,
 ) (*authtypes.Profile, error) {
 	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
diff --git a/appservice/appservice.go b/appservice/appservice.go
index e026a7875..bd292767b 100644
--- a/appservice/appservice.go
+++ b/appservice/appservice.go
@@ -38,7 +38,7 @@ import (
 )
 
 // AddInternalRoutes registers HTTP handlers for internal API calls
-func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQueryAPI) {
+func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
 	inthttp.AddRoutes(queryAPI, router)
 }
 
@@ -48,7 +48,7 @@ func NewInternalAPI(
 	base *base.BaseDendrite,
 	userAPI userapi.AppserviceUserAPI,
 	rsAPI roomserverAPI.AppserviceRoomserverAPI,
-) appserviceAPI.AppServiceQueryAPI {
+) appserviceAPI.AppServiceInternalAPI {
 	client := gomatrixserverlib.NewClient(
 		gomatrixserverlib.WithTimeout(time.Second*30),
 		gomatrixserverlib.WithKeepAlives(false),
diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go
index 7e3cb208f..0a8baea99 100644
--- a/appservice/inthttp/client.go
+++ b/appservice/inthttp/client.go
@@ -29,7 +29,7 @@ type httpAppServiceQueryAPI struct {
 func NewAppserviceClient(
 	appserviceURL string,
 	httpClient *http.Client,
-) (api.AppServiceQueryAPI, error) {
+) (api.AppServiceInternalAPI, error) {
 	if httpClient == nil {
 		return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
 	}
diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go
index 009b7b5db..645b43871 100644
--- a/appservice/inthttp/server.go
+++ b/appservice/inthttp/server.go
@@ -11,7 +11,7 @@ import (
 )
 
 // AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
-func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) {
+func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		AppServiceRoomAliasExistsPath,
 		httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index ad4609080..c1e86114b 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -34,7 +34,7 @@ func AddPublicRoutes(
 	base *base.BaseDendrite,
 	federation *gomatrixserverlib.FederationClient,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 	transactionsCache *transactions.Cache,
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index a21abb0eb..d40d84a79 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -138,7 +138,7 @@ func CreateRoom(
 	req *http.Request, device *api.Device,
 	cfg *config.ClientAPI,
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
 	var r createRoomRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
@@ -165,7 +165,7 @@ func createRoom(
 	r createRoomRequest, device *api.Device,
 	cfg *config.ClientAPI,
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 	evTime time.Time,
 ) util.JSONResponse {
 	// TODO (#267): Check room ID doesn't clash with an existing one, and we
diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index 7d91c7b03..cfdf6f2de 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -41,7 +41,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 func SendBan(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -84,7 +84,7 @@ func SendBan(
 func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
 	roomVer gomatrixserverlib.RoomVersion,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI) util.JSONResponse {
 
 	event, err := buildMembershipEvent(
 		ctx, targetUserID, reason, profileAPI, device, membership,
@@ -127,7 +127,7 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
 func SendKick(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -167,7 +167,7 @@ func SendKick(
 func SendUnban(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -202,7 +202,7 @@ func SendUnban(
 func SendInvite(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
 	body, evTime, _, reqErr := extractRequestData(req, roomID, rsAPI)
 	if reqErr != nil {
@@ -239,7 +239,7 @@ func sendInvite(
 	roomID, userID, reason string,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI, evTime time.Time,
+	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
 ) (util.JSONResponse, error) {
 	event, err := buildMembershipEvent(
 		ctx, userID, reason, profileAPI, device, "invite",
@@ -289,7 +289,7 @@ func buildMembershipEvent(
 	device *userapi.Device,
 	membership, roomID string, isDirect bool,
 	cfg *config.ClientAPI, evTime time.Time,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
 	profile, err := loadProfile(ctx, targetUserID, cfg, profileAPI, asAPI)
 	if err != nil {
@@ -327,7 +327,7 @@ func loadProfile(
 	userID string,
 	cfg *config.ClientAPI,
 	profileAPI userapi.ClientUserAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 ) (*authtypes.Profile, error) {
 	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go
index 97f86afe2..0685c7352 100644
--- a/clientapi/routing/profile.go
+++ b/clientapi/routing/profile.go
@@ -37,7 +37,7 @@ import (
 func GetProfile(
 	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
 	userID string,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
@@ -65,7 +65,7 @@ func GetProfile(
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
 	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
-	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+	userID string, asAPI appserviceAPI.AppServiceInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
@@ -194,7 +194,7 @@ func SetAvatarURL(
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
 	req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
-	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+	userID string, asAPI appserviceAPI.AppServiceInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
 	profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
@@ -327,7 +327,7 @@ func SetDisplayName(
 func getProfile(
 	ctx context.Context, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
 	userID string,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) (*authtypes.Profile, error) {
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index f9f71ed7a..94becf465 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -51,7 +51,7 @@ func Setup(
 	publicAPIMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
 	federation *gomatrixserverlib.FederationClient,
diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go
index 9c34f2e1c..9edeed2f7 100644
--- a/clientapi/routing/server_notices.go
+++ b/clientapi/routing/server_notices.go
@@ -58,7 +58,7 @@ func SendServerNotice(
 	cfgClient *config.ClientAPI,
 	userAPI userapi.ClientUserAPI,
 	rsAPI api.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 	device *userapi.Device,
 	senderDevice *userapi.Device,
 	txnID *string,
diff --git a/clientapi/routing/upgrade_room.go b/clientapi/routing/upgrade_room.go
index 505bf8f53..744e2d889 100644
--- a/clientapi/routing/upgrade_room.go
+++ b/clientapi/routing/upgrade_room.go
@@ -42,7 +42,7 @@ func UpgradeRoom(
 	cfg *config.ClientAPI,
 	roomID string, profileAPI userapi.ClientUserAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
+	asAPI appserviceAPI.AppServiceInternalAPI,
 ) util.JSONResponse {
 	var r upgradeRoomRequest
 	if rErr := httputil.UnmarshalJSONRequest(req, &r); rErr != nil {
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index 2fa4675a4..845b9e465 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -89,6 +89,7 @@ func main() {
 	fsAPI := federationapi.NewInternalAPI(
 		base, federation, rsAPI, base.Caches, nil, false,
 	)
+	fsImplAPI := fsAPI
 	if base.UseHTTPAPIs {
 		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
 		fsAPI = base.FederationAPIHTTPClient()
@@ -138,7 +139,10 @@ func main() {
 		FedClient: federation,
 		KeyRing:   keyRing,
 
-		AppserviceAPI: asAPI, FederationAPI: fsAPI,
+		AppserviceAPI: asAPI,
+		// always use the concrete impl here even in -http mode because adding public routes
+		// must be done on the concrete impl not an HTTP client else fedapi will call itself
+		FederationAPI: fsImplAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
 		KeyAPI:        keyAPI,
diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index 87b037187..fc25194e0 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -10,30 +10,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// FederationClient is a subset of gomatrixserverlib.FederationClient functions which the fedsender
-// implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
-// this interface are of type FederationClientError
-type FederationClient interface {
-	gomatrixserverlib.FederatedStateClient
-	GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
-	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
-	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
-	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
-	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
-	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
-}
-
-// FederationClientError is returned from FederationClient methods in the event of a problem.
-type FederationClientError struct {
-	Err         string
-	RetryAfter  time.Duration
-	Blacklisted bool
-}
-
-func (e *FederationClientError) Error() string {
-	return fmt.Sprintf("%s - (retry_after=%s, blacklisted=%v)", e.Err, e.RetryAfter.String(), e.Blacklisted)
-}
-
 // FederationInternalAPI is used to query information from the federation sender.
 type FederationInternalAPI interface {
 	FederationClient
@@ -43,22 +19,7 @@ type FederationInternalAPI interface {
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
 
-	// Query the server names of the joined hosts in a room.
-	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
-	// containing only the server names (without information for membership events).
-	// The response will include this server if they are joined to the room.
-	QueryJoinedHostServerNamesInRoom(
-		ctx context.Context,
-		request *QueryJoinedHostServerNamesInRoomRequest,
-		response *QueryJoinedHostServerNamesInRoomResponse,
-	) error
-	// Notifies the federation sender that these servers may be online and to retry sending messages.
-	PerformServersAlive(
-		ctx context.Context,
-		request *PerformServersAliveRequest,
-		response *PerformServersAliveResponse,
-	) error
-	// Broadcasts an EDU to all servers in rooms we are joined to.
+	// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
 	PerformBroadcastEDU(
 		ctx context.Context,
 		request *PerformBroadcastEDURequest,
@@ -67,6 +28,10 @@ type FederationInternalAPI interface {
 }
 
 type ClientFederationAPI interface {
+	// Query the server names of the joined hosts in a room.
+	// Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice
+	// containing only the server names (without information for membership events).
+	// The response will include this server if they are joined to the room.
 	QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
 }
 
@@ -95,6 +60,30 @@ type RoomserverFederationAPI interface {
 	LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
+// FederationClient is a subset of gomatrixserverlib.FederationClient functions which the fedsender
+// implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
+// this interface are of type FederationClientError
+type FederationClient interface {
+	gomatrixserverlib.FederatedStateClient
+	GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
+	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
+	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
+	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
+	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
+	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
+}
+
+// FederationClientError is returned from FederationClient methods in the event of a problem.
+type FederationClientError struct {
+	Err         string
+	RetryAfter  time.Duration
+	Blacklisted bool
+}
+
+func (e *FederationClientError) Error() string {
+	return fmt.Sprintf("%s - (retry_after=%s, blacklisted=%v)", e.Err, e.RetryAfter.String(), e.Blacklisted)
+}
+
 type QueryServerKeysRequest struct {
 	ServerName      gomatrixserverlib.ServerName
 	KeyIDToCriteria map[gomatrixserverlib.KeyID]gomatrixserverlib.PublicKeyNotaryQueryCriteria
@@ -174,13 +163,6 @@ type PerformInviteResponse struct {
 	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
 }
 
-type PerformServersAliveRequest struct {
-	Servers []gomatrixserverlib.ServerName
-}
-
-type PerformServersAliveResponse struct {
-}
-
 // QueryJoinedHostServerNamesInRoomRequest is a request to QueryJoinedHostServerNames
 type QueryJoinedHostServerNamesInRoomRequest struct {
 	RoomID      string `json:"room_id"`
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index 632994db9..e52377c94 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -50,8 +50,8 @@ func AddPublicRoutes(
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
-	federationAPI federationAPI.FederationInternalAPI,
-	keyAPI keyserverAPI.KeyInternalAPI,
+	fedAPI federationAPI.FederationInternalAPI,
+	keyAPI keyserverAPI.FederationKeyAPI,
 	servers federationAPI.ServersInRoomProvider,
 ) {
 	cfg := &base.Cfg.FederationAPI
@@ -67,12 +67,23 @@ func AddPublicRoutes(
 		UserAPI:                userAPI,
 	}
 
+	// the federationapi component is a bit unique in that it attaches public routes AND serves
+	// internal APIs (because it used to be 2 components: the 2nd being fedsender). As a result,
+	// the constructor shape is a bit wonky in that it is not valid to AddPublicRoutes without a
+	// concrete impl of FederationInternalAPI as the public routes and the internal API _should_
+	// be the same thing now.
+	f, ok := fedAPI.(*internal.FederationInternalAPI)
+	if !ok {
+		panic("federationapi.AddPublicRoutes called with a FederationInternalAPI impl which was not " +
+			"FederationInternalAPI. This is a programming error.")
+	}
+
 	routing.Setup(
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
 		base.PublicWellKnownAPIMux,
 		cfg,
-		rsAPI, federationAPI, keyRing,
+		rsAPI, f, keyRing,
 		federation, userAPI, keyAPI, mscCfg,
 		servers, producer,
 	)
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index 687241646..eedebc6cd 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -27,10 +28,9 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	keyRing := &test.NopJSONVerifier{}
-	fsAPI := base.FederationAPIHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base, nil, nil, keyRing, nil, fsAPI, nil, nil)
+	federationapi.AddPublicRoutes(base, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, nil)
 	baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index aac36cc76..577cb70e0 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -563,20 +563,6 @@ func (r *FederationInternalAPI) PerformInvite(
 	return nil
 }
 
-// PerformServersAlive implements api.FederationInternalAPI
-func (r *FederationInternalAPI) PerformServersAlive(
-	ctx context.Context,
-	request *api.PerformServersAliveRequest,
-	response *api.PerformServersAliveResponse,
-) (err error) {
-	for _, srv := range request.Servers {
-		_ = r.db.RemoveServerFromBlacklist(srv)
-		r.queues.RetryServer(srv)
-	}
-
-	return nil
-}
-
 // PerformServersAlive implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformBroadcastEDU(
 	ctx context.Context,
@@ -600,18 +586,18 @@ func (r *FederationInternalAPI) PerformBroadcastEDU(
 	if err = r.queues.SendEDU(edu, r.cfg.Matrix.ServerName, destinations); err != nil {
 		return fmt.Errorf("r.queues.SendEDU: %w", err)
 	}
-
-	wakeReq := &api.PerformServersAliveRequest{
-		Servers: destinations,
-	}
-	wakeRes := &api.PerformServersAliveResponse{}
-	if err := r.PerformServersAlive(ctx, wakeReq, wakeRes); err != nil {
-		return fmt.Errorf("r.PerformServersAlive: %w", err)
-	}
+	r.MarkServersAlive(destinations)
 
 	return nil
 }
 
+func (r *FederationInternalAPI) MarkServersAlive(destinations []gomatrixserverlib.ServerName) {
+	for _, srv := range destinations {
+		_ = r.db.RemoveServerFromBlacklist(srv)
+		r.queues.RetryServer(srv)
+	}
+}
+
 func sanityCheckAuthChain(authChain []*gomatrixserverlib.Event) error {
 	// sanity check we have a create event and it has a known room version
 	for _, ev := range authChain {
diff --git a/federationapi/inthttp/client.go b/federationapi/inthttp/client.go
index 01ca6595d..295ddc495 100644
--- a/federationapi/inthttp/client.go
+++ b/federationapi/inthttp/client.go
@@ -23,7 +23,6 @@ const (
 	FederationAPIPerformLeaveRequestPath           = "/federationapi/performLeaveRequest"
 	FederationAPIPerformInviteRequestPath          = "/federationapi/performInviteRequest"
 	FederationAPIPerformOutboundPeekRequestPath    = "/federationapi/performOutboundPeekRequest"
-	FederationAPIPerformServersAlivePath           = "/federationapi/performServersAlive"
 	FederationAPIPerformBroadcastEDUPath           = "/federationapi/performBroadcastEDU"
 
 	FederationAPIGetUserDevicesPath      = "/federationapi/client/getUserDevices"
@@ -97,18 +96,6 @@ func (h *httpFederationInternalAPI) PerformOutboundPeek(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
-func (h *httpFederationInternalAPI) PerformServersAlive(
-	ctx context.Context,
-	request *api.PerformServersAliveRequest,
-	response *api.PerformServersAliveResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformServersAlive")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIPerformServersAlivePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
 // QueryJoinedHostServerNamesInRoom implements FederationInternalAPI
 func (h *httpFederationInternalAPI) QueryJoinedHostServerNamesInRoom(
 	ctx context.Context,
diff --git a/federationapi/inthttp/server.go b/federationapi/inthttp/server.go
index ca4930f20..28e52b32d 100644
--- a/federationapi/inthttp/server.go
+++ b/federationapi/inthttp/server.go
@@ -81,20 +81,6 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	internalAPIMux.Handle(
-		FederationAPIPerformServersAlivePath,
-		httputil.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse {
-			var request api.PerformServersAliveRequest
-			var response api.PerformServersAliveResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformServersAlive(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
 	internalAPIMux.Handle(
 		FederationAPIPerformBroadcastEDUPath,
 		httputil.MakeInternalAPI("PerformBroadcastEDU", func(req *http.Request) util.JSONResponse {
diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go
index 57286fa90..1a092645f 100644
--- a/federationapi/routing/devices.go
+++ b/federationapi/routing/devices.go
@@ -26,7 +26,7 @@ import (
 // GetUserDevices for the given user id
 func GetUserDevices(
 	req *http.Request,
-	keyAPI keyapi.KeyInternalAPI,
+	keyAPI keyapi.FederationKeyAPI,
 	userID string,
 ) util.JSONResponse {
 	var res keyapi.QueryDeviceMessagesResponse
diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go
index 49a6c558f..b1a9b6710 100644
--- a/federationapi/routing/keys.go
+++ b/federationapi/routing/keys.go
@@ -37,7 +37,7 @@ type queryKeysRequest struct {
 // QueryDeviceKeys returns device keys for users on this server.
 // https://matrix.org/docs/spec/server_server/latest#post-matrix-federation-v1-user-keys-query
 func QueryDeviceKeys(
-	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.KeyInternalAPI, thisServer gomatrixserverlib.ServerName,
+	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.FederationKeyAPI, thisServer gomatrixserverlib.ServerName,
 ) util.JSONResponse {
 	var qkr queryKeysRequest
 	err := json.Unmarshal(request.Content(), &qkr)
@@ -89,7 +89,7 @@ type claimOTKsRequest struct {
 // ClaimOneTimeKeys claims OTKs for users on this server.
 // https://matrix.org/docs/spec/server_server/latest#post-matrix-federation-v1-user-keys-claim
 func ClaimOneTimeKeys(
-	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.KeyInternalAPI, thisServer gomatrixserverlib.ServerName,
+	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.FederationKeyAPI, thisServer gomatrixserverlib.ServerName,
 ) util.JSONResponse {
 	var cor claimOTKsRequest
 	err := json.Unmarshal(request.Content(), &cor)
diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index 51adc279c..9f95ed07e 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -18,10 +18,14 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sync"
+	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
+	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/producers"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -48,11 +52,11 @@ func Setup(
 	fedMux, keyMux, wkMux *mux.Router,
 	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
-	fsAPI federationAPI.FederationInternalAPI,
+	fsAPI *fedInternal.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
 	userAPI userapi.FederationUserAPI,
-	keyAPI keyserverAPI.KeyInternalAPI,
+	keyAPI keyserverAPI.FederationKeyAPI,
 	mscCfg *config.MSCs,
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,
@@ -65,7 +69,7 @@ func Setup(
 	v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
 	v2fedmux := fedMux.PathPrefix("/v2").Subrouter()
 
-	wakeup := &httputil.FederationWakeups{
+	wakeup := &FederationWakeups{
 		FsAPI: fsAPI,
 	}
 
@@ -119,7 +123,7 @@ func Setup(
 	v2keysmux.Handle("/query/{serverName}/{keyID}", notaryKeys).Methods(http.MethodGet)
 
 	mu := internal.NewMutexByRoom()
-	v1fedmux.Handle("/send/{txnID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/send/{txnID}", MakeFedAPI(
 		"federation_send", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return Send(
@@ -129,7 +133,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
-	v1fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/invite/{roomID}/{eventID}", MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -145,7 +149,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
-	v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v2fedmux.Handle("/invite/{roomID}/{eventID}", MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -167,7 +171,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPost, http.MethodOptions)
 
-	v1fedmux.Handle("/exchange_third_party_invite/{roomID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/exchange_third_party_invite/{roomID}", MakeFedAPI(
 		"exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return ExchangeThirdPartyInvite(
@@ -176,7 +180,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut, http.MethodOptions)
 
-	v1fedmux.Handle("/event/{eventID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/event/{eventID}", MakeFedAPI(
 		"federation_get_event", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetEvent(
@@ -185,7 +189,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/state/{roomID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/state/{roomID}", MakeFedAPI(
 		"federation_get_state", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -200,7 +204,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/state_ids/{roomID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/state_ids/{roomID}", MakeFedAPI(
 		"federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -215,7 +219,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/event_auth/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/event_auth/{roomID}/{eventID}", MakeFedAPI(
 		"federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -230,7 +234,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/query/directory", httputil.MakeFedAPI(
+	v1fedmux.Handle("/query/directory", MakeFedAPI(
 		"federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return RoomAliasToID(
@@ -239,7 +243,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/query/profile", httputil.MakeFedAPI(
+	v1fedmux.Handle("/query/profile", MakeFedAPI(
 		"federation_query_profile", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetProfile(
@@ -248,7 +252,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/user/devices/{userID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/user/devices/{userID}", MakeFedAPI(
 		"federation_user_devices", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetUserDevices(
@@ -258,7 +262,7 @@ func Setup(
 	)).Methods(http.MethodGet)
 
 	if mscCfg.Enabled("msc2444") {
-		v1fedmux.Handle("/peek/{roomID}/{peekID}", httputil.MakeFedAPI(
+		v1fedmux.Handle("/peek/{roomID}/{peekID}", MakeFedAPI(
 			"federation_peek", cfg.Matrix.ServerName, keys, wakeup,
 			func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 				if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -289,7 +293,7 @@ func Setup(
 		)).Methods(http.MethodPut, http.MethodDelete)
 	}
 
-	v1fedmux.Handle("/make_join/{roomID}/{userID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/make_join/{roomID}/{userID}", MakeFedAPI(
 		"federation_make_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -320,7 +324,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/send_join/{roomID}/{eventID}", MakeFedAPI(
 		"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -352,7 +356,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut)
 
-	v2fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v2fedmux.Handle("/send_join/{roomID}/{eventID}", MakeFedAPI(
 		"federation_send_join", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -369,7 +373,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut)
 
-	v1fedmux.Handle("/make_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/make_leave/{roomID}/{eventID}", MakeFedAPI(
 		"federation_make_leave", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -386,7 +390,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/send_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/send_leave/{roomID}/{eventID}", MakeFedAPI(
 		"federation_send_leave", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -418,7 +422,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPut)
 
-	v2fedmux.Handle("/send_leave/{roomID}/{eventID}", httputil.MakeFedAPI(
+	v2fedmux.Handle("/send_leave/{roomID}/{eventID}", MakeFedAPI(
 		"federation_send_leave", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -442,7 +446,7 @@ func Setup(
 		},
 	)).Methods(http.MethodGet)
 
-	v1fedmux.Handle("/get_missing_events/{roomID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/get_missing_events/{roomID}", MakeFedAPI(
 		"federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -455,7 +459,7 @@ func Setup(
 		},
 	)).Methods(http.MethodPost)
 
-	v1fedmux.Handle("/backfill/{roomID}", httputil.MakeFedAPI(
+	v1fedmux.Handle("/backfill/{roomID}", MakeFedAPI(
 		"federation_backfill", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@@ -474,14 +478,14 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodPost)
 
-	v1fedmux.Handle("/user/keys/claim", httputil.MakeFedAPI(
+	v1fedmux.Handle("/user/keys/claim", MakeFedAPI(
 		"federation_keys_claim", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return ClaimOneTimeKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName)
 		},
 	)).Methods(http.MethodPost)
 
-	v1fedmux.Handle("/user/keys/query", httputil.MakeFedAPI(
+	v1fedmux.Handle("/user/keys/query", MakeFedAPI(
 		"federation_keys_query", cfg.Matrix.ServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return QueryDeviceKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName)
@@ -518,3 +522,67 @@ func ErrorIfLocalServerNotInRoom(
 	}
 	return nil
 }
+
+// MakeFedAPI makes an http.Handler that checks matrix federation authentication.
+func MakeFedAPI(
+	metricsName string,
+	serverName gomatrixserverlib.ServerName,
+	keyRing gomatrixserverlib.JSONVerifier,
+	wakeup *FederationWakeups,
+	f func(*http.Request, *gomatrixserverlib.FederationRequest, map[string]string) util.JSONResponse,
+) http.Handler {
+	h := func(req *http.Request) util.JSONResponse {
+		fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
+			req, time.Now(), serverName, keyRing,
+		)
+		if fedReq == nil {
+			return errResp
+		}
+		// add the user to Sentry, if enabled
+		hub := sentry.GetHubFromContext(req.Context())
+		if hub != nil {
+			hub.Scope().SetTag("origin", string(fedReq.Origin()))
+			hub.Scope().SetTag("uri", fedReq.RequestURI())
+		}
+		defer func() {
+			if r := recover(); r != nil {
+				if hub != nil {
+					hub.CaptureException(fmt.Errorf("%s panicked", req.URL.Path))
+				}
+				// re-panic to return the 500
+				panic(r)
+			}
+		}()
+		go wakeup.Wakeup(req.Context(), fedReq.Origin())
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.MatrixErrorResponse(400, "M_UNRECOGNISED", "badly encoded query params")
+		}
+
+		jsonRes := f(req, fedReq, vars)
+		// do not log 4xx as errors as they are client fails, not server fails
+		if hub != nil && jsonRes.Code >= 500 {
+			hub.Scope().SetExtra("response", jsonRes)
+			hub.CaptureException(fmt.Errorf("%s returned HTTP %d", req.URL.Path, jsonRes.Code))
+		}
+		return jsonRes
+	}
+	return httputil.MakeExternalAPI(metricsName, h)
+}
+
+type FederationWakeups struct {
+	FsAPI   *fedInternal.FederationInternalAPI
+	origins sync.Map
+}
+
+func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib.ServerName) {
+	key, keyok := f.origins.Load(origin)
+	if keyok {
+		lastTime, ok := key.(time.Time)
+		if ok && time.Since(lastTime) < time.Minute {
+			return
+		}
+	}
+	f.FsAPI.MarkServersAlive([]gomatrixserverlib.ServerName{origin})
+	f.origins.Store(origin, time.Now())
+}
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index b9b6d33b7..55a113675 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -83,7 +83,7 @@ func Send(
 	txnID gomatrixserverlib.TransactionID,
 	cfg *config.FederationAPI,
 	rsAPI api.FederationRoomserverAPI,
-	keyAPI keyapi.KeyInternalAPI,
+	keyAPI keyapi.FederationKeyAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
 	mu *internal.MutexByRoom,
@@ -183,7 +183,7 @@ func Send(
 type txnReq struct {
 	gomatrixserverlib.Transaction
 	rsAPI                  api.FederationRoomserverAPI
-	keyAPI                 keyapi.KeyInternalAPI
+	keyAPI                 keyapi.FederationKeyAPI
 	ourServerName          gomatrixserverlib.ServerName
 	keys                   gomatrixserverlib.JSONVerifier
 	federation             txnFederationClient
diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index 3a818cc5e..aba50ae4d 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -15,7 +15,6 @@
 package httputil
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -23,15 +22,10 @@ import (
 	"net/http/httputil"
 	"os"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/getsentry/sentry-go"
-	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	federationapiAPI "github.com/matrix-org/dendrite/federationapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
@@ -226,79 +220,6 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
 	)
 }
 
-// MakeFedAPI makes an http.Handler that checks matrix federation authentication.
-func MakeFedAPI(
-	metricsName string,
-	serverName gomatrixserverlib.ServerName,
-	keyRing gomatrixserverlib.JSONVerifier,
-	wakeup *FederationWakeups,
-	f func(*http.Request, *gomatrixserverlib.FederationRequest, map[string]string) util.JSONResponse,
-) http.Handler {
-	h := func(req *http.Request) util.JSONResponse {
-		fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-			req, time.Now(), serverName, keyRing,
-		)
-		if fedReq == nil {
-			return errResp
-		}
-		// add the user to Sentry, if enabled
-		hub := sentry.GetHubFromContext(req.Context())
-		if hub != nil {
-			hub.Scope().SetTag("origin", string(fedReq.Origin()))
-			hub.Scope().SetTag("uri", fedReq.RequestURI())
-		}
-		defer func() {
-			if r := recover(); r != nil {
-				if hub != nil {
-					hub.CaptureException(fmt.Errorf("%s panicked", req.URL.Path))
-				}
-				// re-panic to return the 500
-				panic(r)
-			}
-		}()
-		go wakeup.Wakeup(req.Context(), fedReq.Origin())
-		vars, err := URLDecodeMapValues(mux.Vars(req))
-		if err != nil {
-			return util.MatrixErrorResponse(400, "M_UNRECOGNISED", "badly encoded query params")
-		}
-
-		jsonRes := f(req, fedReq, vars)
-		// do not log 4xx as errors as they are client fails, not server fails
-		if hub != nil && jsonRes.Code >= 500 {
-			hub.Scope().SetExtra("response", jsonRes)
-			hub.CaptureException(fmt.Errorf("%s returned HTTP %d", req.URL.Path, jsonRes.Code))
-		}
-		return jsonRes
-	}
-	return MakeExternalAPI(metricsName, h)
-}
-
-type FederationWakeups struct {
-	FsAPI   federationapiAPI.FederationInternalAPI
-	origins sync.Map
-}
-
-func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib.ServerName) {
-	key, keyok := f.origins.Load(origin)
-	if keyok {
-		lastTime, ok := key.(time.Time)
-		if ok && time.Since(lastTime) < time.Minute {
-			return
-		}
-	}
-	aliveReq := federationapiAPI.PerformServersAliveRequest{
-		Servers: []gomatrixserverlib.ServerName{origin},
-	}
-	aliveRes := federationapiAPI.PerformServersAliveResponse{}
-	if err := f.FsAPI.PerformServersAlive(ctx, &aliveReq, &aliveRes); err != nil {
-		util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
-			"origin": origin,
-		}).Warn("incoming federation request failed to notify server alive")
-	} else {
-		f.origins.Store(origin, time.Now())
-	}
-}
-
 // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics
 func WrapHandlerInBasicAuth(h http.Handler, b BasicAuth) http.HandlerFunc {
 	if b.Username == "" || b.Password == "" {
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index 6cee2c014..140f03569 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -29,15 +29,11 @@ import (
 type KeyInternalAPI interface {
 	SyncKeyAPI
 	ClientKeyAPI
+	FederationKeyAPI
 	UserKeyAPI
 
 	// SetUserAPI assigns a user API to query when extracting device names.
-	SetUserAPI(i userapi.UserInternalAPI)
-	// InputDeviceListUpdate from a federated server EDU
-	InputDeviceListUpdate(ctx context.Context, req *InputDeviceListUpdateRequest, res *InputDeviceListUpdateResponse)
-
-	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse)
-	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse)
+	SetUserAPI(i userapi.KeyserverUserAPI)
 }
 
 // API functions required by the clientapi
@@ -62,6 +58,16 @@ type SyncKeyAPI interface {
 	QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse)
 }
 
+type FederationKeyAPI interface {
+	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse)
+	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse)
+	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse)
+	// InputDeviceListUpdate from a federated server EDU
+	InputDeviceListUpdate(ctx context.Context, req *InputDeviceListUpdateRequest, res *InputDeviceListUpdateResponse)
+	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse)
+	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse)
+}
+
 // KeyError is returned if there was a problem performing/querying the server
 type KeyError struct {
 	Err                string `json:"error"`
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index e556f44b0..be71e5750 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -38,12 +38,12 @@ type KeyInternalAPI struct {
 	DB         storage.Database
 	ThisServer gomatrixserverlib.ServerName
 	FedClient  fedsenderapi.FederationClient
-	UserAPI    userapi.UserInternalAPI
+	UserAPI    userapi.KeyserverUserAPI
 	Producer   *producers.KeyChange
 	Updater    *DeviceListUpdater
 }
 
-func (a *KeyInternalAPI) SetUserAPI(i userapi.UserInternalAPI) {
+func (a *KeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) {
 	a.UserAPI = i
 }
 
diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go
index f50789b82..abce81582 100644
--- a/keyserver/inthttp/client.go
+++ b/keyserver/inthttp/client.go
@@ -60,7 +60,7 @@ type httpKeyInternalAPI struct {
 	httpClient *http.Client
 }
 
-func (h *httpKeyInternalAPI) SetUserAPI(i userapi.UserInternalAPI) {
+func (h *httpKeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) {
 	// no-op: doesn't need it
 }
 func (h *httpKeyInternalAPI) InputDeviceListUpdate(
diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go
index baab27751..37892a44a 100644
--- a/roomserver/api/alias.go
+++ b/roomserver/api/alias.go
@@ -59,18 +59,6 @@ type GetAliasesForRoomIDResponse struct {
 	Aliases []string `json:"aliases"`
 }
 
-// GetCreatorIDForAliasRequest is a request to GetCreatorIDForAlias
-type GetCreatorIDForAliasRequest struct {
-	// The alias we want to find the creator of
-	Alias string `json:"alias"`
-}
-
-// GetCreatorIDForAliasResponse is a response to GetCreatorIDForAlias
-type GetCreatorIDForAliasResponse struct {
-	// The user ID of the alias creator
-	UserID string `json:"user_id"`
-}
-
 // RemoveRoomAliasRequest is a request to RemoveRoomAlias
 type RemoveRoomAliasRequest struct {
 	// ID of the user removing the alias
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 7e1e568ce..cbb4cebca 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -12,10 +12,6 @@ import (
 
 // RoomserverInputAPI is used to write events to the room server.
 type RoomserverInternalAPI interface {
-	InputRoomEventsAPI
-	QueryLatestEventsAndStateAPI
-	QueryEventsAPI
-
 	SyncRoomserverAPI
 	AppserviceRoomserverAPI
 	ClientRoomserverAPI
@@ -25,101 +21,18 @@ type RoomserverInternalAPI interface {
 	// needed to avoid chicken and egg scenario when setting up the
 	// interdependencies between the roomserver and other input APIs
 	SetFederationAPI(fsAPI fsAPI.RoomserverFederationAPI, keyRing *gomatrixserverlib.KeyRing)
-	SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI)
-	SetUserAPI(userAPI userapi.UserInternalAPI)
-
-	PerformInvite(
-		ctx context.Context,
-		req *PerformInviteRequest,
-		res *PerformInviteResponse,
-	) error
-
-	PerformJoin(
-		ctx context.Context,
-		req *PerformJoinRequest,
-		res *PerformJoinResponse,
-	)
-
-	PerformLeave(
-		ctx context.Context,
-		req *PerformLeaveRequest,
-		res *PerformLeaveResponse,
-	) error
-
-	PerformPeek(
-		ctx context.Context,
-		req *PerformPeekRequest,
-		res *PerformPeekResponse,
-	)
-
-	PerformUnpeek(
-		ctx context.Context,
-		req *PerformUnpeekRequest,
-		res *PerformUnpeekResponse,
-	)
-
-	PerformPublish(
-		ctx context.Context,
-		req *PerformPublishRequest,
-		res *PerformPublishResponse,
-	)
+	SetAppserviceAPI(asAPI asAPI.AppServiceInternalAPI)
+	SetUserAPI(userAPI userapi.RoomserverUserAPI)
 
 	// QueryAuthChain returns the entire auth chain for the event IDs given.
 	// The response includes the events in the request.
 	// Omits without error for any missing auth events. There will be no duplicates.
+	// Used in MSC2836.
 	QueryAuthChain(
 		ctx context.Context,
 		req *QueryAuthChainRequest,
 		res *QueryAuthChainResponse,
 	) error
-
-	// QueryRoomsForUser retrieves a list of room IDs matching the given query.
-	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
-
-	// PerformRoomUpgrade upgrades a room to a newer version
-	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
-
-	// Asks for the default room version as preferred by the server.
-	QueryRoomVersionCapabilities(
-		ctx context.Context,
-		req *QueryRoomVersionCapabilitiesRequest,
-		res *QueryRoomVersionCapabilitiesResponse,
-	) error
-
-	// Asks for the room version for a given room.
-	QueryRoomVersionForRoom(
-		ctx context.Context,
-		req *QueryRoomVersionForRoomRequest,
-		res *QueryRoomVersionForRoomResponse,
-	) error
-
-	// Set a room alias
-	SetRoomAlias(
-		ctx context.Context,
-		req *SetRoomAliasRequest,
-		res *SetRoomAliasResponse,
-	) error
-
-	// Get the room ID for an alias
-	GetRoomIDForAlias(
-		ctx context.Context,
-		req *GetRoomIDForAliasRequest,
-		res *GetRoomIDForAliasResponse,
-	) error
-
-	// Get the user ID of the creator of an alias
-	GetCreatorIDForAlias(
-		ctx context.Context,
-		req *GetCreatorIDForAliasRequest,
-		res *GetCreatorIDForAliasResponse,
-	) error
-
-	// Remove a room alias
-	RemoveRoomAlias(
-		ctx context.Context,
-		req *RemoveRoomAliasRequest,
-		res *RemoveRoomAliasResponse,
-	) error
 }
 
 type InputRoomEventsAPI interface {
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index bc60999e6..711324644 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -23,11 +23,11 @@ func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.RoomserverFede
 	t.Impl.SetFederationAPI(fsAPI, keyRing)
 }
 
-func (t *RoomserverInternalAPITrace) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) {
+func (t *RoomserverInternalAPITrace) SetAppserviceAPI(asAPI asAPI.AppServiceInternalAPI) {
 	t.Impl.SetAppserviceAPI(asAPI)
 }
 
-func (t *RoomserverInternalAPITrace) SetUserAPI(userAPI userapi.UserInternalAPI) {
+func (t *RoomserverInternalAPITrace) SetUserAPI(userAPI userapi.RoomserverUserAPI) {
 	t.Impl.SetUserAPI(userAPI)
 }
 
@@ -293,16 +293,6 @@ func (t *RoomserverInternalAPITrace) GetAliasesForRoomID(
 	return err
 }
 
-func (t *RoomserverInternalAPITrace) GetCreatorIDForAlias(
-	ctx context.Context,
-	req *GetCreatorIDForAliasRequest,
-	res *GetCreatorIDForAliasResponse,
-) error {
-	err := t.Impl.GetCreatorIDForAlias(ctx, req, res)
-	util.GetLogger(ctx).WithError(err).Infof("GetCreatorIDForAlias req=%+v res=%+v", js(req), js(res))
-	return err
-}
-
 func (t *RoomserverInternalAPITrace) RemoveRoomAlias(
 	ctx context.Context,
 	req *RemoveRoomAliasRequest,
diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go
index 02fc4a5a7..f47ae47fe 100644
--- a/roomserver/internal/alias.go
+++ b/roomserver/internal/alias.go
@@ -41,9 +41,6 @@ type RoomserverInternalAPIDatabase interface {
 	// Look up all aliases referring to a given room ID.
 	// Returns an error if there was a problem talking to the database.
 	GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error)
-	// Get the user ID of the creator of an alias.
-	// Returns an error if there was a problem talking to the database.
-	GetCreatorIDForAlias(ctx context.Context, alias string) (string, error)
 	// Remove a given room alias.
 	// Returns an error if there was a problem talking to the database.
 	RemoveRoomAlias(ctx context.Context, alias string) error
@@ -134,22 +131,6 @@ func (r *RoomserverInternalAPI) GetAliasesForRoomID(
 	return nil
 }
 
-// GetCreatorIDForAlias implements alias.RoomserverInternalAPI
-func (r *RoomserverInternalAPI) GetCreatorIDForAlias(
-	ctx context.Context,
-	request *api.GetCreatorIDForAliasRequest,
-	response *api.GetCreatorIDForAliasResponse,
-) error {
-	// Look up the aliases in the database for the given RoomID
-	creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias)
-	if err != nil {
-		return err
-	}
-
-	response.UserID = creatorID
-	return nil
-}
-
 // RemoveRoomAlias implements alias.RoomserverInternalAPI
 func (r *RoomserverInternalAPI) RemoveRoomAlias(
 	ctx context.Context,
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index dc0a0a718..afef52da4 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -44,7 +44,7 @@ type RoomserverInternalAPI struct {
 	KeyRing                gomatrixserverlib.JSONVerifier
 	ServerACLs             *acls.ServerACLs
 	fsAPI                  fsAPI.RoomserverFederationAPI
-	asAPI                  asAPI.AppServiceQueryAPI
+	asAPI                  asAPI.AppServiceInternalAPI
 	NATSClient             *nats.Conn
 	JetStream              nats.JetStreamContext
 	Durable                string
@@ -177,11 +177,11 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio
 	}
 }
 
-func (r *RoomserverInternalAPI) SetUserAPI(userAPI userapi.UserInternalAPI) {
+func (r *RoomserverInternalAPI) SetUserAPI(userAPI userapi.RoomserverUserAPI) {
 	r.Leaver.UserAPI = userAPI
 }
 
-func (r *RoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) {
+func (r *RoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalAPI) {
 	r.asAPI = asAPI
 }
 
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index b006843fb..c5b62ac00 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -38,7 +38,7 @@ type Leaver struct {
 	Cfg     *config.RoomServer
 	DB      storage.Database
 	FSAPI   fsAPI.RoomserverFederationAPI
-	UserAPI userapi.UserInternalAPI
+	UserAPI userapi.RoomserverUserAPI
 	Inputer *input.Inputer
 }
 
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index 4fc75ff41..09358001b 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -91,11 +91,11 @@ func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.Roomserver
 }
 
 // SetAppserviceAPI no-ops in HTTP client mode as there is no chicken/egg scenario
-func (h *httpRoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) {
+func (h *httpRoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalAPI) {
 }
 
 // SetUserAPI no-ops in HTTP client mode as there is no chicken/egg scenario
-func (h *httpRoomserverInternalAPI) SetUserAPI(userAPI userapi.UserInternalAPI) {
+func (h *httpRoomserverInternalAPI) SetUserAPI(userAPI userapi.RoomserverUserAPI) {
 }
 
 // SetRoomAlias implements RoomserverAliasAPI
@@ -137,19 +137,6 @@ func (h *httpRoomserverInternalAPI) GetAliasesForRoomID(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
-// GetCreatorIDForAlias implements RoomserverAliasAPI
-func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias(
-	ctx context.Context,
-	request *api.GetCreatorIDForAliasRequest,
-	response *api.GetCreatorIDForAliasResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetCreatorIDForAlias")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}
-
 // RemoveRoomAlias implements RoomserverAliasAPI
 func (h *httpRoomserverInternalAPI) RemoveRoomAlias(
 	ctx context.Context,
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index c5159a63c..9042e341b 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -353,20 +353,6 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	internalAPIMux.Handle(
-		RoomserverGetCreatorIDForAliasPath,
-		httputil.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse {
-			var request api.GetCreatorIDForAliasRequest
-			var response api.GetCreatorIDForAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
 	internalAPIMux.Handle(
 		RoomserverGetAliasesForRoomIDPath,
 		httputil.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse {
diff --git a/setup/base/base.go b/setup/base/base.go
index 9326be1c0..d7d5119fc 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -270,8 +270,8 @@ func (b *BaseDendrite) DatabaseConnection(dbProperties *config.DatabaseOptions,
 	return nil, nil, fmt.Errorf("no database connections configured")
 }
 
-// AppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP.
-func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI {
+// AppserviceHTTPClient returns the AppServiceInternalAPI for hitting the appservice component over HTTP.
+func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceInternalAPI {
 	a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.apiHttpClient)
 	if err != nil {
 		logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed")
diff --git a/setup/monolith.go b/setup/monolith.go
index a0e850d83..41a897024 100644
--- a/setup/monolith.go
+++ b/setup/monolith.go
@@ -39,7 +39,7 @@ type Monolith struct {
 	Client    *gomatrixserverlib.Client
 	FedClient *gomatrixserverlib.FederationClient
 
-	AppserviceAPI appserviceAPI.AppServiceQueryAPI
+	AppserviceAPI appserviceAPI.AppServiceInternalAPI
 	FederationAPI federationAPI.FederationInternalAPI
 	RoomserverAPI roomserverAPI.RoomserverInternalAPI
 	UserAPI       userapi.UserInternalAPI
diff --git a/userapi/api/api.go b/userapi/api/api.go
index dc8c12b74..df9408acb 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -31,6 +31,8 @@ type UserInternalAPI interface {
 	ClientUserAPI
 	MediaUserAPI
 	FederationUserAPI
+	RoomserverUserAPI
+	KeyserverUserAPI
 
 	QuerySearchProfilesAPI // used by p2p demos
 }
@@ -41,6 +43,15 @@ type AppserviceUserAPI interface {
 	PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error
 }
 
+type KeyserverUserAPI interface {
+	QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error
+	QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error
+}
+
+type RoomserverUserAPI interface {
+	QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error
+}
+
 // api functions required by the media api
 type MediaUserAPI interface {
 	QueryAcccessTokenAPI

From a1a5357f799887fc5b7e3bf5c81bbf3198704645 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 6 May 2022 12:46:01 +0100
Subject: [PATCH 081/391] Produce more useful event auth errors (update to
 matrix-org/gomatrixserverlib#305)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index b6070b94f..38d28e11e 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220505130352-f72a63510060
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220506114534-f3951a683bad
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index cbea7319e..96ac72637 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220505130352-f72a63510060 h1:tYi4mCOWgVLt8mpkG1LFRKcMfSTwp5NQ5wBKdtaxO9s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220505130352-f72a63510060/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220506114534-f3951a683bad h1:QhPE4f2fijOSW3QvT/Ucwqz9KJwafKaiKwfhwgIpx3M=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220506114534-f3951a683bad/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 507f63d0fc8158f200f3e29fd36e5f09c83e62db Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 6 May 2022 13:51:48 +0100
Subject: [PATCH 082/391] Add `PolylithMode` base config option (#2428)

* Add `PolylithMode` base config option

* Polylith mode always uses HTTP APIs
---
 cmd/dendrite-polylith-multi/main.go | 4 ++--
 setup/base/base.go                  | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go
index 4fccaa922..e4845f649 100644
--- a/cmd/dendrite-polylith-multi/main.go
+++ b/cmd/dendrite-polylith-multi/main.go
@@ -71,8 +71,8 @@ func main() {
 
 	logrus.Infof("Starting %q component", component)
 
-	base := base.NewBaseDendrite(cfg, component) // TODO
-	defer base.Close()                           // nolint: errcheck
+	base := base.NewBaseDendrite(cfg, component, base.PolylithMode) // TODO
+	defer base.Close()                                              // nolint: errcheck
 
 	go start(base, cfg)
 	base.WaitForShutdown()
diff --git a/setup/base/base.go b/setup/base/base.go
index d7d5119fc..ef449cc35 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -96,6 +96,7 @@ type BaseDendriteOptions int
 const (
 	NoCacheMetrics BaseDendriteOptions = iota
 	UseHTTPAPIs
+	PolylithMode
 )
 
 // NewBaseDendrite creates a new instance to be used by a component.
@@ -105,17 +106,20 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 	platformSanityChecks()
 	useHTTPAPIs := false
 	cacheMetrics := true
+	isMonolith := true
 	for _, opt := range options {
 		switch opt {
 		case NoCacheMetrics:
 			cacheMetrics = false
 		case UseHTTPAPIs:
 			useHTTPAPIs = true
+		case PolylithMode:
+			isMonolith = false
+			useHTTPAPIs = true
 		}
 	}
 
 	configErrors := &config.ConfigErrors{}
-	isMonolith := componentName == "Monolith" // TODO: better way?
 	cfg.Verify(configErrors, isMonolith)
 	if len(*configErrors) > 0 {
 		for _, err := range *configErrors {

From 6493c0c0f23bb930cc8a2ce071a66b7d4b5ec9eb Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 6 May 2022 15:33:34 +0200
Subject: [PATCH 083/391] Move LL cache (#2429)

---
 internal/caching/cache_lazy_load_members.go | 35 +++++----------------
 internal/caching/caches.go                  |  5 ++-
 internal/caching/impl_inmemorylru.go        | 15 ++++++++-
 syncapi/routing/context.go                  |  4 +--
 syncapi/routing/messages.go                 |  2 +-
 syncapi/routing/routing.go                  |  2 +-
 syncapi/streams/stream_pdu.go               |  2 +-
 syncapi/streams/streams.go                  |  2 +-
 syncapi/syncapi.go                          |  8 ++---
 9 files changed, 34 insertions(+), 41 deletions(-)

diff --git a/internal/caching/cache_lazy_load_members.go b/internal/caching/cache_lazy_load_members.go
index 71a317624..f0d495065 100644
--- a/internal/caching/cache_lazy_load_members.go
+++ b/internal/caching/cache_lazy_load_members.go
@@ -15,33 +15,14 @@ const (
 	LazyLoadCacheMaxAge         = time.Minute * 30
 )
 
-type LazyLoadCache struct {
-	// InMemoryLRUCachePartition containing other InMemoryLRUCachePartitions
-	// with the actual cached members
-	userCaches *InMemoryLRUCachePartition
+type LazyLoadCache interface {
+	StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string)
+	IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool)
 }
 
-// NewLazyLoadCache creates a new LazyLoadCache.
-func NewLazyLoadCache() (*LazyLoadCache, error) {
-	cache, err := NewInMemoryLRUCachePartition(
-		LazyLoadCacheName,
-		LazyLoadCacheMutable,
-		LazyLoadCacheMaxEntries,
-		LazyLoadCacheMaxAge,
-		true,
-	)
-	if err != nil {
-		return nil, err
-	}
-	go cacheCleaner(cache)
-	return &LazyLoadCache{
-		userCaches: cache,
-	}, nil
-}
-
-func (c *LazyLoadCache) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) {
+func (c Caches) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) {
 	cacheName := fmt.Sprintf("%s/%s", device.UserID, device.ID)
-	userCache, ok := c.userCaches.Get(cacheName)
+	userCache, ok := c.LazyLoading.Get(cacheName)
 	if ok && userCache != nil {
 		if cache, ok := userCache.(*InMemoryLRUCachePartition); ok {
 			return cache, nil
@@ -57,12 +38,12 @@ func (c *LazyLoadCache) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryL
 	if err != nil {
 		return nil, err
 	}
-	c.userCaches.Set(cacheName, cache)
+	c.LazyLoading.Set(cacheName, cache)
 	go cacheCleaner(cache)
 	return cache, nil
 }
 
-func (c *LazyLoadCache) StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) {
+func (c Caches) StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) {
 	cache, err := c.lazyLoadCacheForUser(device)
 	if err != nil {
 		return
@@ -71,7 +52,7 @@ func (c *LazyLoadCache) StoreLazyLoadedUser(device *userapi.Device, roomID, user
 	cache.Set(cacheKey, eventID)
 }
 
-func (c *LazyLoadCache) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool) {
+func (c Caches) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool) {
 	cache, err := c.lazyLoadCacheForUser(device)
 	if err != nil {
 		return "", false
diff --git a/internal/caching/caches.go b/internal/caching/caches.go
index 722405de6..173e47e5b 100644
--- a/internal/caching/caches.go
+++ b/internal/caching/caches.go
@@ -1,6 +1,8 @@
 package caching
 
-import "time"
+import (
+	"time"
+)
 
 // Caches contains a set of references to caches. They may be
 // different implementations as long as they satisfy the Cache
@@ -13,6 +15,7 @@ type Caches struct {
 	RoomInfos          Cache // RoomInfoCache
 	FederationEvents   Cache // FederationEventsCache
 	SpaceSummaryRooms  Cache // SpaceSummaryRoomsCache
+	LazyLoading        Cache // LazyLoadCache
 }
 
 // Cache is the interface that an implementation must satisfy.
diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go
index 94fdd1a9b..594760892 100644
--- a/internal/caching/impl_inmemorylru.go
+++ b/internal/caching/impl_inmemorylru.go
@@ -70,9 +70,21 @@ func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	lazyLoadCache, err := NewInMemoryLRUCachePartition(
+		LazyLoadCacheName,
+		LazyLoadCacheMutable,
+		LazyLoadCacheMaxEntries,
+		LazyLoadCacheMaxAge,
+		enablePrometheus,
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	go cacheCleaner(
 		roomVersions, serverKeys, roomServerRoomIDs,
-		roomInfos, federationEvents, spaceRooms,
+		roomInfos, federationEvents, spaceRooms, lazyLoadCache,
 	)
 	return &Caches{
 		RoomVersions:      roomVersions,
@@ -81,6 +93,7 @@ func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
 		RoomInfos:         roomInfos,
 		FederationEvents:  federationEvents,
 		SpaceSummaryRooms: spaceRooms,
+		LazyLoading:       lazyLoadCache,
 	}, nil
 }
 
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index f5f4b2dd0..87cc2aae0 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -45,7 +45,7 @@ func Context(
 	rsAPI roomserver.SyncRoomserverAPI,
 	syncDB storage.Database,
 	roomID, eventID string,
-	lazyLoadCache *caching.LazyLoadCache,
+	lazyLoadCache caching.LazyLoadCache,
 ) util.JSONResponse {
 	filter, err := parseRoomEventFilter(req)
 	if err != nil {
@@ -155,7 +155,7 @@ func applyLazyLoadMembers(
 	filter *gomatrixserverlib.RoomEventFilter,
 	eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent,
 	state []*gomatrixserverlib.HeaderedEvent,
-	lazyLoadCache *caching.LazyLoadCache,
+	lazyLoadCache caching.LazyLoadCache,
 ) []*gomatrixserverlib.HeaderedEvent {
 	if filter == nil || !filter.LazyLoadMembers {
 		return state
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index f19dfaed3..b0c990ec0 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -63,7 +63,7 @@ func OnIncomingMessagesRequest(
 	rsAPI api.SyncRoomserverAPI,
 	cfg *config.SyncAPI,
 	srp *sync.RequestPool,
-	lazyLoadCache *caching.LazyLoadCache,
+	lazyLoadCache caching.LazyLoadCache,
 ) util.JSONResponse {
 	var err error
 
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 245ee5b66..6bc495d8d 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -39,7 +39,7 @@ func Setup(
 	userAPI userapi.SyncUserAPI,
 	rsAPI api.SyncRoomserverAPI,
 	cfg *config.SyncAPI,
-	lazyLoadCache *caching.LazyLoadCache,
+	lazyLoadCache caching.LazyLoadCache,
 ) {
 	v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index f774a1af8..00b3dfe3b 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -32,7 +32,7 @@ type PDUStreamProvider struct {
 	tasks   chan func()
 	workers atomic.Int32
 	// userID+deviceID -> lazy loading cache
-	lazyLoadCache *caching.LazyLoadCache
+	lazyLoadCache caching.LazyLoadCache
 	rsAPI         roomserverAPI.SyncRoomserverAPI
 }
 
diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go
index af2a0387e..1ca4ee8c3 100644
--- a/syncapi/streams/streams.go
+++ b/syncapi/streams/streams.go
@@ -27,7 +27,7 @@ type Streams struct {
 func NewSyncStreamProviders(
 	d storage.Database, userAPI userapi.SyncUserAPI,
 	rsAPI rsapi.SyncRoomserverAPI, keyAPI keyapi.SyncKeyAPI,
-	eduCache *caching.EDUCache, lazyLoadCache *caching.LazyLoadCache, notifier *notifier.Notifier,
+	eduCache *caching.EDUCache, lazyLoadCache caching.LazyLoadCache, notifier *notifier.Notifier,
 ) *Streams {
 	streams := &Streams{
 		PDUStreamProvider: &PDUStreamProvider{
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index 686e2044f..6da8ce6d1 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -53,12 +53,8 @@ func AddPublicRoutes(
 	}
 
 	eduCache := caching.NewTypingCache()
-	lazyLoadCache, err := caching.NewLazyLoadCache()
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to create lazy loading cache")
-	}
 	notifier := notifier.NewNotifier()
-	streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, lazyLoadCache, notifier)
+	streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier)
 	notifier.SetCurrentPosition(streams.Latest(context.Background()))
 	if err = notifier.Load(context.Background(), syncDB); err != nil {
 		logrus.WithError(err).Panicf("failed to load notifier ")
@@ -146,6 +142,6 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		base.PublicClientAPIMux, requestPool, syncDB, userAPI,
-		rsAPI, cfg, lazyLoadCache,
+		rsAPI, cfg, base.Caches,
 	)
 }

From 85c00208c5b804b2d95a6a790e1dd977c33d030e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 6 May 2022 15:41:16 +0100
Subject: [PATCH 084/391] Fix power level event auth bugs (update to
 matrix-org/gomatrixserverlib#306)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 38d28e11e..58cfbba11 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220506114534-f3951a683bad
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220506144035-8958f9dfdffd
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 96ac72637..047bbf5c3 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220506114534-f3951a683bad h1:QhPE4f2fijOSW3QvT/Ucwqz9KJwafKaiKwfhwgIpx3M=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220506114534-f3951a683bad/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220506144035-8958f9dfdffd h1:11Wh+NMPDE5UDEx50RJnxeYj7zH5HEClzGfVMAjUy9U=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220506144035-8958f9dfdffd/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 6bc6184d70614a9ba2cc585c88f66e6a780ddb98 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 6 May 2022 15:52:44 +0100
Subject: [PATCH 085/391] Simplify `calculateLatest` (#2430)

* Simplify `calculateLatest`

* Comments
---
 .../internal/input/input_latest_events.go     | 26 ++++++-------------
 1 file changed, 8 insertions(+), 18 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 7e58ef9d0..9ad8b0422 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -316,40 +316,30 @@ func (u *latestEventsUpdater) calculateLatest(
 	// Then let's see if any of the existing forward extremities now
 	// have entries in the previous events table. If they do then we
 	// will no longer include them as forward extremities.
-	existingPrevs := make(map[string]struct{})
-	for _, l := range existingRefs {
+	for k, l := range existingRefs {
 		referenced, err := u.updater.IsReferenced(l.EventReference)
 		if err != nil {
 			return false, fmt.Errorf("u.updater.IsReferenced: %w", err)
 		} else if referenced {
-			existingPrevs[l.EventID] = struct{}{}
+			delete(existingRefs, k)
 		}
 	}
 
-	// Include our new event in the extremities.
-	newLatest := []types.StateAtEventAndReference{newStateAndRef}
+	// Start off with our new unreferenced event. We're reusing the backing
+	// array here rather than allocating a new one.
+	u.latest = append(u.latest[:0], newStateAndRef)
 
-	// Then run through and see if the other extremities are still valid.
-	// If our new event references them then they are no longer good
-	// candidates.
+	// If our new event references any of the existing forward extremities
+	// then they are no longer forward extremities, so remove them.
 	for _, prevEventID := range newEvent.PrevEventIDs() {
 		delete(existingRefs, prevEventID)
 	}
 
-	// Ensure that we don't add any candidate forward extremities from
-	// the old set that are, themselves, referenced by the old set of
-	// forward extremities. This shouldn't happen but guards against
-	// the possibility anyway.
-	for prevEventID := range existingPrevs {
-		delete(existingRefs, prevEventID)
-	}
-
 	// Then re-add any old extremities that are still valid after all.
 	for _, old := range existingRefs {
-		newLatest = append(newLatest, *old)
+		u.latest = append(u.latest, *old)
 	}
 
-	u.latest = newLatest
 	return true, nil
 }
 

From 633ca06eb9f7652a6c4be04b3ffe8950419a8ee3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 6 May 2022 16:34:52 +0100
Subject: [PATCH 086/391] Version 0.8.3rc1

---
 internal/version.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/version.go b/internal/version.go
index 2477bc9ac..e74548831 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,8 +17,8 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 2
-	VersionTag   = "" // example: "rc1"
+	VersionPatch = 3
+	VersionTag   = "rc1" // example: "rc1"
 )
 
 func VersionString() string {

From 4c15c73b3abfb0cca0c95c7a21305a8329e2c23c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 9 May 2022 11:13:04 +0100
Subject: [PATCH 087/391] Add `(user_id, device_id)` index on OTK table (#2435)

---
 keyserver/storage/postgres/one_time_keys_table.go | 2 ++
 keyserver/storage/sqlite3/one_time_keys_table.go  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/keyserver/storage/postgres/one_time_keys_table.go b/keyserver/storage/postgres/one_time_keys_table.go
index 0b143a1aa..d8c76b49b 100644
--- a/keyserver/storage/postgres/one_time_keys_table.go
+++ b/keyserver/storage/postgres/one_time_keys_table.go
@@ -39,6 +39,8 @@ CREATE TABLE IF NOT EXISTS keyserver_one_time_keys (
 	-- Clobber based on 4-uple of user/device/key/algorithm.
     CONSTRAINT keyserver_one_time_keys_unique UNIQUE (user_id, device_id, key_id, algorithm)
 );
+
+CREATE INDEX IF NOT EXISTS keyserver_one_time_keys_idx ON keyserver_one_time_keys (user_id, device_id);
 `
 
 const upsertKeysSQL = "" +
diff --git a/keyserver/storage/sqlite3/one_time_keys_table.go b/keyserver/storage/sqlite3/one_time_keys_table.go
index 897839aca..d2c0b7b20 100644
--- a/keyserver/storage/sqlite3/one_time_keys_table.go
+++ b/keyserver/storage/sqlite3/one_time_keys_table.go
@@ -38,6 +38,8 @@ CREATE TABLE IF NOT EXISTS keyserver_one_time_keys (
 	-- Clobber based on 4-uple of user/device/key/algorithm.
     UNIQUE (user_id, device_id, key_id, algorithm)
 );
+
+CREATE INDEX IF NOT EXISTS keyserver_one_time_keys_idx ON keyserver_one_time_keys (user_id, device_id);
 `
 
 const upsertKeysSQL = "" +

From 79e2fbc66368d8f4754b9fff8005d3e77969fcc4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 9 May 2022 13:53:51 +0100
Subject: [PATCH 088/391] Update to matrix-org/gomatrixserverlib#307

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 58cfbba11..803272c56 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220506144035-8958f9dfdffd
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 047bbf5c3..5fde89e06 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220506144035-8958f9dfdffd h1:11Wh+NMPDE5UDEx50RJnxeYj7zH5HEClzGfVMAjUy9U=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220506144035-8958f9dfdffd/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c h1:KqzqFWxvs90pcDaW9QEveW+Q5JcEYuNnKyaqXc+ohno=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 09d754cfbf9268044d0f59fbe509640b8d71e011 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 9 May 2022 14:15:24 +0100
Subject: [PATCH 089/391] One NATS instance per `BaseDendrite` (#2438)

* One NATS instance per `BaseDendrite`

* Fix roomserver
---
 appservice/appservice.go                |  3 +--
 clientapi/clientapi.go                  |  2 +-
 federationapi/federationapi.go          |  4 +--
 keyserver/keyserver.go                  |  2 +-
 roomserver/internal/input/input_test.go | 12 ++++-----
 roomserver/roomserver.go                |  2 +-
 setup/base/base.go                      |  3 +++
 setup/jetstream/nats.go                 | 36 ++++++++++---------------
 syncapi/syncapi.go                      |  2 +-
 test/base.go                            | 18 +++++++++++++
 userapi/userapi.go                      |  2 +-
 11 files changed, 49 insertions(+), 37 deletions(-)
 create mode 100644 test/base.go

diff --git a/appservice/appservice.go b/appservice/appservice.go
index bd292767b..c5ae9ceb2 100644
--- a/appservice/appservice.go
+++ b/appservice/appservice.go
@@ -32,7 +32,6 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -55,7 +54,7 @@ func NewInternalAPI(
 		gomatrixserverlib.WithSkipVerify(base.Cfg.AppServiceAPI.DisableTLSValidation),
 	)
 
-	js, _ := jetstream.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 
 	// Create a connection to the appservice postgres DB
 	appserviceDB, err := storage.NewDatabase(base, &base.Cfg.AppServiceAPI.Database)
diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index c1e86114b..f550c29bb 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -44,7 +44,7 @@ func AddPublicRoutes(
 ) {
 	cfg := &base.Cfg.ClientAPI
 	mscCfg := &base.Cfg.MSCs
-	js, natsClient := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index e52377c94..bec9ac777 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -56,7 +56,7 @@ func AddPublicRoutes(
 ) {
 	cfg := &base.Cfg.FederationAPI
 	mscCfg := &base.Cfg.MSCs
-	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 	producer := &producers.SyncAPIProducer{
 		JetStream:              js,
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
@@ -115,7 +115,7 @@ func NewInternalAPI(
 		FailuresUntilBlacklist: cfg.FederationMaxRetries,
 	}
 
-	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	queues := queue.NewOutgoingQueues(
 		federationDB, base.ProcessContext,
diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go
index 007a48a55..47d7f57f9 100644
--- a/keyserver/keyserver.go
+++ b/keyserver/keyserver.go
@@ -39,7 +39,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) {
 func NewInternalAPI(
 	base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient,
 ) api.KeyInternalAPI {
-	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	db, err := storage.NewDatabase(base, &cfg.Database)
 	if err != nil {
diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go
index 5d34842bf..a95c13550 100644
--- a/roomserver/internal/input/input_test.go
+++ b/roomserver/internal/input/input_test.go
@@ -10,9 +10,9 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
 	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 )
@@ -21,11 +21,11 @@ var js nats.JetStreamContext
 var jc *nats.Conn
 
 func TestMain(m *testing.M) {
-	var pc *process.ProcessContext
-	pc, js, jc = jetstream.PrepareForTests()
+	var b *base.BaseDendrite
+	b, js, jc = test.Base(nil)
 	code := m.Run()
-	pc.ShutdownDendrite()
-	pc.WaitForComponentsToFinish()
+	b.ShutdownDendrite()
+	b.WaitForComponentsToFinish()
 	os.Exit(code)
 }
 
diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go
index 46261eb3e..1480e8942 100644
--- a/roomserver/roomserver.go
+++ b/roomserver/roomserver.go
@@ -50,7 +50,7 @@ func NewInternalAPI(
 		logrus.WithError(err).Panicf("failed to connect to room server db")
 	}
 
-	js, nc := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, nc := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	return internal.NewRoomserverAPI(
 		base.ProcessContext, cfg, roomserverDB, js, nc,
diff --git a/setup/base/base.go b/setup/base/base.go
index ef449cc35..0e7528a03 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -41,6 +41,7 @@ import (
 	"golang.org/x/net/http2/h2c"
 
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 
 	"github.com/gorilla/mux"
@@ -77,6 +78,7 @@ type BaseDendrite struct {
 	InternalAPIMux         *mux.Router
 	DendriteAdminMux       *mux.Router
 	SynapseAdminMux        *mux.Router
+	NATS                   *jetstream.NATSInstance
 	UseHTTPAPIs            bool
 	apiHttpClient          *http.Client
 	Cfg                    *config.Dendrite
@@ -240,6 +242,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		InternalAPIMux:         mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
 		DendriteAdminMux:       mux.NewRouter().SkipClean(true).PathPrefix(httputil.DendriteAdminPathPrefix).Subrouter().UseEncodedPath(),
 		SynapseAdminMux:        mux.NewRouter().SkipClean(true).PathPrefix(httputil.SynapseAdminPathPrefix).Subrouter().UseEncodedPath(),
+		NATS:                   &jetstream.NATSInstance{},
 		apiHttpClient:          &apiClient,
 		Database:               db,     // set if monolith with global connection pool only
 		DatabaseWriter:         writer, // set if monolith with global connection pool only
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 8d5289697..426f02bb6 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -13,31 +13,23 @@ import (
 	"github.com/sirupsen/logrus"
 
 	natsserver "github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
 	natsclient "github.com/nats-io/nats.go"
 )
 
-var natsServer *natsserver.Server
-var natsServerMutex sync.Mutex
-
-func PrepareForTests() (*process.ProcessContext, nats.JetStreamContext, *nats.Conn) {
-	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
-	cfg.Global.JetStream.InMemory = true
-	pc := process.NewProcessContext()
-	js, jc := Prepare(pc, &cfg.Global.JetStream)
-	return pc, js, jc
+type NATSInstance struct {
+	*natsserver.Server
+	sync.Mutex
 }
 
-func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) {
+func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) {
 	// check if we need an in-process NATS Server
 	if len(cfg.Addresses) != 0 {
 		return setupNATS(process, cfg, nil)
 	}
-	natsServerMutex.Lock()
-	if natsServer == nil {
+	s.Lock()
+	if s.Server == nil {
 		var err error
-		natsServer, err = natsserver.NewServer(&natsserver.Options{
+		s.Server, err = natsserver.NewServer(&natsserver.Options{
 			ServerName:      "monolith",
 			DontListen:      true,
 			JetStream:       true,
@@ -49,23 +41,23 @@ func Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient
 		if err != nil {
 			panic(err)
 		}
-		natsServer.ConfigureLogger()
+		s.ConfigureLogger()
 		go func() {
 			process.ComponentStarted()
-			natsServer.Start()
+			s.Start()
 		}()
 		go func() {
 			<-process.WaitForShutdown()
-			natsServer.Shutdown()
-			natsServer.WaitForShutdown()
+			s.Shutdown()
+			s.WaitForShutdown()
 			process.ComponentFinished()
 		}()
 	}
-	natsServerMutex.Unlock()
-	if !natsServer.ReadyForConnections(time.Second * 10) {
+	s.Unlock()
+	if !s.ReadyForConnections(time.Second * 10) {
 		logrus.Fatalln("NATS did not start in time")
 	}
-	nc, err := natsclient.Connect("", natsclient.InProcessServer(natsServer))
+	nc, err := natsclient.Connect("", natsclient.InProcessServer(s))
 	if err != nil {
 		logrus.Fatalln("Failed to create NATS client")
 	}
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index 6da8ce6d1..dbc6e240c 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -45,7 +45,7 @@ func AddPublicRoutes(
 ) {
 	cfg := &base.Cfg.SyncAPI
 
-	js, natsClient := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database)
 	if err != nil {
diff --git a/test/base.go b/test/base.go
new file mode 100644
index 000000000..32fc8dc53
--- /dev/null
+++ b/test/base.go
@@ -0,0 +1,18 @@
+package test
+
+import (
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/nats-io/nats.go"
+)
+
+func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) {
+	if cfg == nil {
+		cfg = &config.Dendrite{}
+		cfg.Defaults(true)
+	}
+	cfg.Global.JetStream.InMemory = true
+	base := base.NewBaseDendrite(cfg, "Tests")
+	js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream)
+	return base, js, jc
+}
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 03a46807f..603b416bf 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -47,7 +47,7 @@ func NewInternalAPI(
 	appServices []config.ApplicationService, keyAPI keyapi.UserKeyAPI,
 	rsAPI rsapi.UserRoomserverAPI, pgClient pushgateway.Client,
 ) api.UserInternalAPI {
-	js, _ := jetstream.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
+	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	db, err := storage.NewUserAPIDatabase(
 		base,

From f69ebc6af2dfeeb7af7eaabbe0609976c397a685 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 9 May 2022 15:30:32 +0200
Subject: [PATCH 090/391] Add roomserver tests (1/?) (#2434)

* Add EventJSONTable tests

* Add eventJSON tests

* Add EventStateKeysTable tests

* Add EventTypesTable tests

* Add Events Table tests
Move variable declaration outside loops
Switch to testify/assert for tests

* Move variable declaration outside loop

* Remove random data

* Fix issue where the EventReferenceSHA256 is not set

* Add more tests

* Revert "Fix issue where the EventReferenceSHA256 is not set"

This reverts commit 8ae34c4e5f78584f0edb479f5a893556d2b95d19.

* Update GMSL

* Add tests for duplicate entries

* Test what happens if we select non-existing NIDs

* Add test for non-existing eventType

* Really update GMSL
---
 go.mod                                        |   6 +-
 go.sum                                        |  12 +-
 .../storage/postgres/event_json_table.go      |   6 +-
 .../postgres/event_state_keys_table.go        |  12 +-
 .../storage/postgres/event_types_table.go     |   8 +-
 roomserver/storage/postgres/events_table.go   |  32 ++--
 roomserver/storage/postgres/storage.go        |  16 +-
 .../storage/sqlite3/event_json_table.go       |   6 +-
 .../storage/sqlite3/event_state_keys_table.go |  12 +-
 .../storage/sqlite3/event_types_table.go      |   8 +-
 roomserver/storage/sqlite3/events_table.go    |  35 ++--
 roomserver/storage/sqlite3/storage.go         |  16 +-
 .../storage/tables/event_json_table_test.go   |  95 +++++++++++
 .../tables/event_state_keys_table_test.go     |  79 +++++++++
 .../storage/tables/event_types_table_test.go  |  79 +++++++++
 .../storage/tables/events_table_test.go       | 157 ++++++++++++++++++
 roomserver/storage/tables/interface.go        |   8 +-
 17 files changed, 499 insertions(+), 88 deletions(-)
 create mode 100644 roomserver/storage/tables/event_json_table_test.go
 create mode 100644 roomserver/storage/tables/event_state_keys_table_test.go
 create mode 100644 roomserver/storage/tables/event_types_table_test.go
 create mode 100644 roomserver/storage/tables/events_table_test.go

diff --git a/go.mod b/go.mod
index 803272c56..d14ced5b7 100644
--- a/go.mod
+++ b/go.mod
@@ -48,17 +48,17 @@ require (
 	github.com/prometheus/client_golang v1.12.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	github.com/tidwall/gjson v1.14.0
+	github.com/tidwall/gjson v1.14.1
 	github.com/tidwall/sjson v1.2.4
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.3
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29
+	golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122
 	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
 	golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2
 	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3
-	golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
diff --git a/go.sum b/go.sum
index 5fde89e06..8b518935c 100644
--- a/go.sum
+++ b/go.sum
@@ -1130,8 +1130,8 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
-github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
+github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
@@ -1281,8 +1281,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o=
-golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8=
+golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1543,8 +1543,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 h1:QyVthZKMsyaQwBTJE04jdNN0Pp5Fn9Qga0mrgxyERQM=
-golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go
index b3220effd..5f069ca10 100644
--- a/roomserver/storage/postgres/event_json_table.go
+++ b/roomserver/storage/postgres/event_json_table.go
@@ -59,12 +59,12 @@ type eventJSONStatements struct {
 	bulkSelectEventJSONStmt *sql.Stmt
 }
 
-func createEventJSONTable(db *sql.DB) error {
+func CreateEventJSONTable(db *sql.DB) error {
 	_, err := db.Exec(eventJSONSchema)
 	return err
 }
 
-func prepareEventJSONTable(db *sql.DB) (tables.EventJSON, error) {
+func PrepareEventJSONTable(db *sql.DB) (tables.EventJSON, error) {
 	s := &eventJSONStatements{}
 
 	return s, sqlutil.StatementList{
@@ -97,9 +97,9 @@ func (s *eventJSONStatements) BulkSelectEventJSON(
 	// We might get fewer results than NIDs so we adjust the length of the slice before returning it.
 	results := make([]tables.EventJSONPair, len(eventNIDs))
 	i := 0
+	var eventNID int64
 	for ; rows.Next(); i++ {
 		result := &results[i]
-		var eventNID int64
 		if err := rows.Scan(&eventNID, &result.EventJSON); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go
index 762b3a1fc..338e11b82 100644
--- a/roomserver/storage/postgres/event_state_keys_table.go
+++ b/roomserver/storage/postgres/event_state_keys_table.go
@@ -76,12 +76,12 @@ type eventStateKeyStatements struct {
 	bulkSelectEventStateKeyStmt    *sql.Stmt
 }
 
-func createEventStateKeysTable(db *sql.DB) error {
+func CreateEventStateKeysTable(db *sql.DB) error {
 	_, err := db.Exec(eventStateKeysSchema)
 	return err
 }
 
-func prepareEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) {
+func PrepareEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) {
 	s := &eventStateKeyStatements{}
 
 	return s, sqlutil.StatementList{
@@ -123,9 +123,9 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed")
 
 	result := make(map[string]types.EventStateKeyNID, len(eventStateKeys))
+	var stateKey string
+	var stateKeyNID int64
 	for rows.Next() {
-		var stateKey string
-		var stateKeyNID int64
 		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
 			return nil, err
 		}
@@ -149,9 +149,9 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed")
 
 	result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs))
+	var stateKey string
+	var stateKeyNID int64
 	for rows.Next() {
-		var stateKey string
-		var stateKeyNID int64
 		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go
index 1d5de5822..15ab7fd8e 100644
--- a/roomserver/storage/postgres/event_types_table.go
+++ b/roomserver/storage/postgres/event_types_table.go
@@ -99,12 +99,12 @@ type eventTypeStatements struct {
 	bulkSelectEventTypeNIDStmt *sql.Stmt
 }
 
-func createEventTypesTable(db *sql.DB) error {
+func CreateEventTypesTable(db *sql.DB) error {
 	_, err := db.Exec(eventTypesSchema)
 	return err
 }
 
-func prepareEventTypesTable(db *sql.DB) (tables.EventTypes, error) {
+func PrepareEventTypesTable(db *sql.DB) (tables.EventTypes, error) {
 	s := &eventTypeStatements{}
 
 	return s, sqlutil.StatementList{
@@ -143,9 +143,9 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed")
 
 	result := make(map[string]types.EventTypeNID, len(eventTypes))
+	var eventType string
+	var eventTypeNID int64
 	for rows.Next() {
-		var eventType string
-		var eventTypeNID int64
 		if err := rows.Scan(&eventType, &eventTypeNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index 8012174a0..86d226ce7 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -155,12 +155,12 @@ type eventStatements struct {
 	selectRoomNIDsForEventNIDsStmt         *sql.Stmt
 }
 
-func createEventsTable(db *sql.DB) error {
+func CreateEventsTable(db *sql.DB) error {
 	_, err := db.Exec(eventsSchema)
 	return err
 }
 
-func prepareEventsTable(db *sql.DB) (tables.Events, error) {
+func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 	s := &eventStatements{}
 
 	return s, sqlutil.StatementList{
@@ -380,15 +380,15 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed")
 	results := make([]types.StateAtEventAndReference, len(eventNIDs))
 	i := 0
+	var (
+		eventTypeNID     int64
+		eventStateKeyNID int64
+		eventNID         int64
+		stateSnapshotNID int64
+		eventID          string
+		eventSHA256      []byte
+	)
 	for ; rows.Next(); i++ {
-		var (
-			eventTypeNID     int64
-			eventStateKeyNID int64
-			eventNID         int64
-			stateSnapshotNID int64
-			eventID          string
-			eventSHA256      []byte
-		)
 		if err = rows.Scan(
 			&eventTypeNID, &eventStateKeyNID, &eventNID, &stateSnapshotNID, &eventID, &eventSHA256,
 		); err != nil {
@@ -446,9 +446,9 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, txn *sql.Tx, ev
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed")
 	results := make(map[types.EventNID]string, len(eventNIDs))
 	i := 0
+	var eventNID int64
+	var eventID string
 	for ; rows.Next(); i++ {
-		var eventNID int64
-		var eventID string
 		if err = rows.Scan(&eventNID, &eventID); err != nil {
 			return nil, err
 		}
@@ -491,9 +491,9 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed")
 	results := make(map[string]types.EventNID, len(eventIDs))
+	var eventID string
+	var eventNID int64
 	for rows.Next() {
-		var eventID string
-		var eventNID int64
 		if err = rows.Scan(&eventID, &eventNID); err != nil {
 			return nil, err
 		}
@@ -522,9 +522,9 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomNIDsForEventNIDsStmt: rows.close() failed")
 	result := make(map[types.EventNID]types.RoomNID)
+	var eventNID types.EventNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var eventNID types.EventNID
-		var roomNID types.RoomNID
 		if err = rows.Scan(&eventNID, &roomNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index da8d25848..34e891490 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -68,16 +68,16 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 }
 
 func (d *Database) create(db *sql.DB) error {
-	if err := createEventStateKeysTable(db); err != nil {
+	if err := CreateEventStateKeysTable(db); err != nil {
 		return err
 	}
-	if err := createEventTypesTable(db); err != nil {
+	if err := CreateEventTypesTable(db); err != nil {
 		return err
 	}
-	if err := createEventJSONTable(db); err != nil {
+	if err := CreateEventJSONTable(db); err != nil {
 		return err
 	}
-	if err := createEventsTable(db); err != nil {
+	if err := CreateEventsTable(db); err != nil {
 		return err
 	}
 	if err := createRoomsTable(db); err != nil {
@@ -112,19 +112,19 @@ func (d *Database) create(db *sql.DB) error {
 }
 
 func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.RoomServerCaches) error {
-	eventStateKeys, err := prepareEventStateKeysTable(db)
+	eventStateKeys, err := PrepareEventStateKeysTable(db)
 	if err != nil {
 		return err
 	}
-	eventTypes, err := prepareEventTypesTable(db)
+	eventTypes, err := PrepareEventTypesTable(db)
 	if err != nil {
 		return err
 	}
-	eventJSON, err := prepareEventJSONTable(db)
+	eventJSON, err := PrepareEventJSONTable(db)
 	if err != nil {
 		return err
 	}
-	events, err := prepareEventsTable(db)
+	events, err := PrepareEventsTable(db)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go
index f470ea326..dc26885bb 100644
--- a/roomserver/storage/sqlite3/event_json_table.go
+++ b/roomserver/storage/sqlite3/event_json_table.go
@@ -52,12 +52,12 @@ type eventJSONStatements struct {
 	bulkSelectEventJSONStmt *sql.Stmt
 }
 
-func createEventJSONTable(db *sql.DB) error {
+func CreateEventJSONTable(db *sql.DB) error {
 	_, err := db.Exec(eventJSONSchema)
 	return err
 }
 
-func prepareEventJSONTable(db *sql.DB) (tables.EventJSON, error) {
+func PrepareEventJSONTable(db *sql.DB) (tables.EventJSON, error) {
 	s := &eventJSONStatements{
 		db: db,
 	}
@@ -101,9 +101,9 @@ func (s *eventJSONStatements) BulkSelectEventJSON(
 	// We might get fewer results than NIDs so we adjust the length of the slice before returning it.
 	results := make([]tables.EventJSONPair, len(eventNIDs))
 	i := 0
+	var eventNID int64
 	for ; rows.Next(); i++ {
 		result := &results[i]
-		var eventNID int64
 		if err := rows.Scan(&eventNID, &result.EventJSON); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go
index f97541f4a..347524a81 100644
--- a/roomserver/storage/sqlite3/event_state_keys_table.go
+++ b/roomserver/storage/sqlite3/event_state_keys_table.go
@@ -71,12 +71,12 @@ type eventStateKeyStatements struct {
 	bulkSelectEventStateKeyStmt    *sql.Stmt
 }
 
-func createEventStateKeysTable(db *sql.DB) error {
+func CreateEventStateKeysTable(db *sql.DB) error {
 	_, err := db.Exec(eventStateKeysSchema)
 	return err
 }
 
-func prepareEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) {
+func PrepareEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) {
 	s := &eventStateKeyStatements{
 		db: db,
 	}
@@ -128,9 +128,9 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed")
 	result := make(map[string]types.EventStateKeyNID, len(eventStateKeys))
+	var stateKey string
+	var stateKeyNID int64
 	for rows.Next() {
-		var stateKey string
-		var stateKeyNID int64
 		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
 			return nil, err
 		}
@@ -159,9 +159,9 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed")
 	result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs))
+	var stateKey string
+	var stateKeyNID int64
 	for rows.Next() {
-		var stateKey string
-		var stateKeyNID int64
 		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go
index c49cc509a..0581ec194 100644
--- a/roomserver/storage/sqlite3/event_types_table.go
+++ b/roomserver/storage/sqlite3/event_types_table.go
@@ -79,12 +79,12 @@ type eventTypeStatements struct {
 	bulkSelectEventTypeNIDStmt *sql.Stmt
 }
 
-func createEventTypesTable(db *sql.DB) error {
+func CreateEventTypesTable(db *sql.DB) error {
 	_, err := db.Exec(eventTypesSchema)
 	return err
 }
 
-func prepareEventTypesTable(db *sql.DB) (tables.EventTypes, error) {
+func PrepareEventTypesTable(db *sql.DB) (tables.EventTypes, error) {
 	s := &eventTypeStatements{
 		db: db,
 	}
@@ -139,9 +139,9 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed")
 
 	result := make(map[string]types.EventTypeNID, len(eventTypes))
+	var eventType string
+	var eventTypeNID int64
 	for rows.Next() {
-		var eventType string
-		var eventTypeNID int64
 		if err := rows.Scan(&eventType, &eventTypeNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index 45b49e5cb..feb06150a 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -68,7 +68,8 @@ const bulkSelectStateEventByIDSQL = "" +
 const bulkSelectStateEventByNIDSQL = "" +
 	"SELECT event_type_nid, event_state_key_nid, event_nid FROM roomserver_events" +
 	" WHERE event_nid IN ($1)"
-	// Rest of query is built by BulkSelectStateEventByNID
+
+// Rest of query is built by BulkSelectStateEventByNID
 
 const bulkSelectStateAtEventByIDSQL = "" +
 	"SELECT event_type_nid, event_state_key_nid, event_nid, state_snapshot_nid, is_rejected FROM roomserver_events" +
@@ -126,12 +127,12 @@ type eventStatements struct {
 	//selectRoomNIDsForEventNIDsStmt       *sql.Stmt
 }
 
-func createEventsTable(db *sql.DB) error {
+func CreateEventsTable(db *sql.DB) error {
 	_, err := db.Exec(eventsSchema)
 	return err
 }
 
-func prepareEventsTable(db *sql.DB) (tables.Events, error) {
+func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 	s := &eventStatements{
 		db: db,
 	}
@@ -404,15 +405,15 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed")
 	results := make([]types.StateAtEventAndReference, len(eventNIDs))
 	i := 0
+	var (
+		eventTypeNID     int64
+		eventStateKeyNID int64
+		eventNID         int64
+		stateSnapshotNID int64
+		eventID          string
+		eventSHA256      []byte
+	)
 	for ; rows.Next(); i++ {
-		var (
-			eventTypeNID     int64
-			eventStateKeyNID int64
-			eventNID         int64
-			stateSnapshotNID int64
-			eventID          string
-			eventSHA256      []byte
-		)
 		if err = rows.Scan(
 			&eventTypeNID, &eventStateKeyNID, &eventNID, &stateSnapshotNID, &eventID, &eventSHA256,
 		); err != nil {
@@ -491,9 +492,9 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, txn *sql.Tx, ev
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed")
 	results := make(map[types.EventNID]string, len(eventNIDs))
 	i := 0
+	var eventNID int64
+	var eventID string
 	for ; rows.Next(); i++ {
-		var eventNID int64
-		var eventID string
 		if err = rows.Scan(&eventNID, &eventID); err != nil {
 			return nil, err
 		}
@@ -545,9 +546,9 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed")
 	results := make(map[string]types.EventNID, len(eventIDs))
+	var eventID string
+	var eventNID int64
 	for rows.Next() {
-		var eventID string
-		var eventNID int64
 		if err = rows.Scan(&eventID, &eventNID); err != nil {
 			return nil, err
 		}
@@ -595,9 +596,9 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomNIDsForEventNIDsStmt: rows.close() failed")
 	result := make(map[types.EventNID]types.RoomNID)
+	var eventNID types.EventNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var eventNID types.EventNID
-		var roomNID types.RoomNID
 		if err = rows.Scan(&eventNID, &roomNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index e6cf1a53f..9522d3058 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -77,16 +77,16 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 }
 
 func (d *Database) create(db *sql.DB) error {
-	if err := createEventStateKeysTable(db); err != nil {
+	if err := CreateEventStateKeysTable(db); err != nil {
 		return err
 	}
-	if err := createEventTypesTable(db); err != nil {
+	if err := CreateEventTypesTable(db); err != nil {
 		return err
 	}
-	if err := createEventJSONTable(db); err != nil {
+	if err := CreateEventJSONTable(db); err != nil {
 		return err
 	}
-	if err := createEventsTable(db); err != nil {
+	if err := CreateEventsTable(db); err != nil {
 		return err
 	}
 	if err := createRoomsTable(db); err != nil {
@@ -121,19 +121,19 @@ func (d *Database) create(db *sql.DB) error {
 }
 
 func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.RoomServerCaches) error {
-	eventStateKeys, err := prepareEventStateKeysTable(db)
+	eventStateKeys, err := PrepareEventStateKeysTable(db)
 	if err != nil {
 		return err
 	}
-	eventTypes, err := prepareEventTypesTable(db)
+	eventTypes, err := PrepareEventTypesTable(db)
 	if err != nil {
 		return err
 	}
-	eventJSON, err := prepareEventJSONTable(db)
+	eventJSON, err := PrepareEventJSONTable(db)
 	if err != nil {
 		return err
 	}
-	events, err := prepareEventsTable(db)
+	events, err := PrepareEventsTable(db)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/tables/event_json_table_test.go b/roomserver/storage/tables/event_json_table_test.go
new file mode 100644
index 000000000..b490d0fe8
--- /dev/null
+++ b/roomserver/storage/tables/event_json_table_test.go
@@ -0,0 +1,95 @@
+package tables_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateEventJSONTable(t *testing.T, dbType test.DBType) (tables.EventJSON, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.EventJSON
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateEventJSONTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareEventJSONTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateEventJSONTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareEventJSONTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func Test_EventJSONTable(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateEventJSONTable(t, dbType)
+		defer close()
+
+		// create some dummy data
+		for i := 0; i < 10; i++ {
+			err := tab.InsertEventJSON(
+				context.Background(), nil, types.EventNID(i),
+				[]byte(fmt.Sprintf(`{"value":%d"}`, i)),
+			)
+			assert.NoError(t, err)
+		}
+
+		tests := []struct {
+			name      string
+			args      []types.EventNID
+			wantCount int
+		}{
+			{
+				name:      "select subset of existing NIDs",
+				args:      []types.EventNID{1, 2, 3, 4, 5},
+				wantCount: 5,
+			},
+			{
+				name:      "select subset of existing/non-existing NIDs",
+				args:      []types.EventNID{1, 2, 12, 50},
+				wantCount: 2,
+			},
+			{
+				name:      "select single existing NID",
+				args:      []types.EventNID{1},
+				wantCount: 1,
+			},
+			{
+				name:      "select single non-existing NID",
+				args:      []types.EventNID{13},
+				wantCount: 0,
+			},
+		}
+
+		for _, tc := range tests {
+			t.Run(tc.name, func(t *testing.T) {
+				// select a subset of the data
+				values, err := tab.BulkSelectEventJSON(context.Background(), nil, tc.args)
+				assert.NoError(t, err)
+				assert.Equal(t, tc.wantCount, len(values))
+				for i, v := range values {
+					assert.Equal(t, v.EventNID, types.EventNID(i+1))
+					assert.Equal(t, []byte(fmt.Sprintf(`{"value":%d"}`, i+1)), v.EventJSON)
+				}
+			})
+		}
+	})
+}
diff --git a/roomserver/storage/tables/event_state_keys_table_test.go b/roomserver/storage/tables/event_state_keys_table_test.go
new file mode 100644
index 000000000..a856fe551
--- /dev/null
+++ b/roomserver/storage/tables/event_state_keys_table_test.go
@@ -0,0 +1,79 @@
+package tables_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateEventStateKeysTable(t *testing.T, dbType test.DBType) (tables.EventStateKeys, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.EventStateKeys
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateEventStateKeysTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareEventStateKeysTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateEventStateKeysTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareEventStateKeysTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func Test_EventStateKeysTable(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateEventStateKeysTable(t, dbType)
+		defer close()
+		ctx := context.Background()
+		var stateKeyNID, gotEventStateKey types.EventStateKeyNID
+		var err error
+		// create some dummy data
+		for i := 0; i < 10; i++ {
+			stateKey := fmt.Sprintf("@user%d:localhost", i)
+			stateKeyNID, err = tab.InsertEventStateKeyNID(ctx, nil, stateKey)
+			assert.NoError(t, err)
+			gotEventStateKey, err = tab.SelectEventStateKeyNID(ctx, nil, stateKey)
+			assert.NoError(t, err)
+			assert.Equal(t, stateKeyNID, gotEventStateKey)
+		}
+		// This should fail, since @user0:localhost already exists
+		stateKey := fmt.Sprintf("@user%d:localhost", 0)
+		_, err = tab.InsertEventStateKeyNID(ctx, nil, stateKey)
+		assert.Error(t, err)
+
+		stateKeyNIDsMap, err := tab.BulkSelectEventStateKeyNID(ctx, nil, []string{"@user0:localhost", "@user1:localhost"})
+		assert.NoError(t, err)
+		wantStateKeyNIDs := make([]types.EventStateKeyNID, 0, len(stateKeyNIDsMap))
+		for _, nid := range stateKeyNIDsMap {
+			wantStateKeyNIDs = append(wantStateKeyNIDs, nid)
+		}
+		stateKeyNIDs, err := tab.BulkSelectEventStateKey(ctx, nil, wantStateKeyNIDs)
+		assert.NoError(t, err)
+		// verify that BulkSelectEventStateKeyNID and BulkSelectEventStateKey return the same values
+		for userID, nid := range stateKeyNIDsMap {
+			if v, ok := stateKeyNIDs[nid]; ok {
+				assert.Equal(t, v, userID)
+			} else {
+				t.Fatalf("unable to find %d in result set", nid)
+			}
+		}
+	})
+}
diff --git a/roomserver/storage/tables/event_types_table_test.go b/roomserver/storage/tables/event_types_table_test.go
new file mode 100644
index 000000000..92c57a917
--- /dev/null
+++ b/roomserver/storage/tables/event_types_table_test.go
@@ -0,0 +1,79 @@
+package tables_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateEventTypesTable(t *testing.T, dbType test.DBType) (tables.EventTypes, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.EventTypes
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateEventTypesTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareEventTypesTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateEventTypesTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareEventTypesTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func Test_EventTypesTable(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateEventTypesTable(t, dbType)
+		defer close()
+		ctx := context.Background()
+		var eventTypeNID, gotEventTypeNID types.EventTypeNID
+		var err error
+		// create some dummy data
+		eventTypeMap := make(map[string]types.EventTypeNID)
+		for i := 0; i < 10; i++ {
+			eventType := fmt.Sprintf("dummyEventType%d", i)
+			eventTypeNID, err = tab.InsertEventTypeNID(ctx, nil, eventType)
+			assert.NoError(t, err)
+			eventTypeMap[eventType] = eventTypeNID
+			gotEventTypeNID, err = tab.SelectEventTypeNID(ctx, nil, eventType)
+			assert.NoError(t, err)
+			assert.Equal(t, eventTypeNID, gotEventTypeNID)
+		}
+		// This should fail, since the dummyEventType0 already exists
+		eventType := fmt.Sprintf("dummyEventType%d", 0)
+		_, err = tab.InsertEventTypeNID(ctx, nil, eventType)
+		assert.Error(t, err)
+
+		// This should return an error, as this eventType does not exist
+		_, err = tab.SelectEventTypeNID(ctx, nil, "dummyEventType13")
+		assert.Error(t, err)
+
+		eventTypeNIDs, err := tab.BulkSelectEventTypeNID(ctx, nil, []string{"dummyEventType0", "dummyEventType3"})
+		assert.NoError(t, err)
+		// verify that BulkSelectEventTypeNID and InsertEventTypeNID return the same values
+		for eventType, nid := range eventTypeNIDs {
+			if v, ok := eventTypeMap[eventType]; ok {
+				assert.Equal(t, v, nid)
+			} else {
+				t.Fatalf("unable to find %d in result set", nid)
+			}
+		}
+	})
+}
diff --git a/roomserver/storage/tables/events_table_test.go b/roomserver/storage/tables/events_table_test.go
new file mode 100644
index 000000000..d5d699c4c
--- /dev/null
+++ b/roomserver/storage/tables/events_table_test.go
@@ -0,0 +1,157 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateEventsTable(t *testing.T, dbType test.DBType) (tables.Events, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.Events
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateEventsTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareEventsTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateEventsTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareEventsTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func Test_EventsTable(t *testing.T) {
+	alice := test.NewUser()
+	room := test.NewRoom(t, alice)
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateEventsTable(t, dbType)
+		defer close()
+		// create some dummy data
+		eventIDs := make([]string, 0, len(room.Events()))
+		wantStateAtEvent := make([]types.StateAtEvent, 0, len(room.Events()))
+		wantEventReferences := make([]gomatrixserverlib.EventReference, 0, len(room.Events()))
+		wantStateAtEventAndRefs := make([]types.StateAtEventAndReference, 0, len(room.Events()))
+		for _, ev := range room.Events() {
+			eventNID, snapNID, err := tab.InsertEvent(ctx, nil, 1, 1, 1, ev.EventID(), ev.EventReference().EventSHA256, nil, ev.Depth(), false)
+			assert.NoError(t, err)
+			gotEventNID, gotSnapNID, err := tab.SelectEvent(ctx, nil, ev.EventID())
+			assert.NoError(t, err)
+			assert.Equal(t, eventNID, gotEventNID)
+			assert.Equal(t, snapNID, gotSnapNID)
+			eventID, err := tab.SelectEventID(ctx, nil, eventNID)
+			assert.NoError(t, err)
+			assert.Equal(t, eventID, ev.EventID())
+
+			// The events shouldn't be sent to output yet
+			sentToOutput, err := tab.SelectEventSentToOutput(ctx, nil, gotEventNID)
+			assert.NoError(t, err)
+			assert.False(t, sentToOutput)
+
+			err = tab.UpdateEventSentToOutput(ctx, nil, gotEventNID)
+			assert.NoError(t, err)
+
+			// Now they should be sent to output
+			sentToOutput, err = tab.SelectEventSentToOutput(ctx, nil, gotEventNID)
+			assert.NoError(t, err)
+			assert.True(t, sentToOutput)
+
+			eventIDs = append(eventIDs, ev.EventID())
+			wantEventReferences = append(wantEventReferences, ev.EventReference())
+
+			// Set the stateSnapshot to 2 for some events to verify they are returned later
+			stateSnapshot := 0
+			if eventNID < 3 {
+				stateSnapshot = 2
+				err = tab.UpdateEventState(ctx, nil, eventNID, 2)
+				assert.NoError(t, err)
+			}
+			stateAtEvent := types.StateAtEvent{
+				Overwrite:              false,
+				BeforeStateSnapshotNID: types.StateSnapshotNID(stateSnapshot),
+				IsRejected:             false,
+				StateEntry: types.StateEntry{
+					EventNID: eventNID,
+					StateKeyTuple: types.StateKeyTuple{
+						EventTypeNID:     1,
+						EventStateKeyNID: 1,
+					},
+				},
+			}
+			wantStateAtEvent = append(wantStateAtEvent, stateAtEvent)
+			wantStateAtEventAndRefs = append(wantStateAtEventAndRefs, types.StateAtEventAndReference{
+				StateAtEvent:   stateAtEvent,
+				EventReference: ev.EventReference(),
+			})
+		}
+
+		stateEvents, err := tab.BulkSelectStateEventByID(ctx, nil, eventIDs)
+		assert.NoError(t, err)
+		assert.Equal(t, len(stateEvents), len(eventIDs))
+		nids := make([]types.EventNID, 0, len(stateEvents))
+		for _, ev := range stateEvents {
+			nids = append(nids, ev.EventNID)
+		}
+		stateEvents2, err := tab.BulkSelectStateEventByNID(ctx, nil, nids, nil)
+		assert.NoError(t, err)
+		// somehow SQLite doesn't return the values ordered as requested by the query
+		assert.ElementsMatch(t, stateEvents, stateEvents2)
+
+		roomNIDs, err := tab.SelectRoomNIDsForEventNIDs(ctx, nil, nids)
+		assert.NoError(t, err)
+		// We only inserted one room, so the RoomNID should be the same for all evendNIDs
+		for _, roomNID := range roomNIDs {
+			assert.Equal(t, types.RoomNID(1), roomNID)
+		}
+
+		stateAtEvent, err := tab.BulkSelectStateAtEventByID(ctx, nil, eventIDs)
+		assert.NoError(t, err)
+		assert.Equal(t, len(eventIDs), len(stateAtEvent))
+
+		assert.ElementsMatch(t, wantStateAtEvent, stateAtEvent)
+
+		evendNIDMap, err := tab.BulkSelectEventID(ctx, nil, nids)
+		assert.NoError(t, err)
+		t.Logf("%+v", evendNIDMap)
+		assert.Equal(t, len(evendNIDMap), len(nids))
+
+		nidMap, err := tab.BulkSelectEventNID(ctx, nil, eventIDs)
+		assert.NoError(t, err)
+		// check that we got all expected eventNIDs
+		for _, eventID := range eventIDs {
+			_, ok := nidMap[eventID]
+			assert.True(t, ok)
+		}
+
+		references, err := tab.BulkSelectEventReference(ctx, nil, nids)
+		assert.NoError(t, err)
+		assert.Equal(t, wantEventReferences, references)
+
+		stateAndRefs, err := tab.BulkSelectStateAtEventAndReference(ctx, nil, nids)
+		assert.NoError(t, err)
+		assert.Equal(t, wantStateAtEventAndRefs, stateAndRefs)
+
+		// check we get the expected event depth
+		maxDepth, err := tab.SelectMaxEventDepth(ctx, nil, nids)
+		assert.NoError(t, err)
+		assert.Equal(t, int64(len(room.Events())+1), maxDepth)
+	})
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 97e4afcff..95609787a 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -10,9 +10,8 @@ import (
 )
 
 type EventJSONPair struct {
-	EventNID    types.EventNID
-	RoomVersion gomatrixserverlib.RoomVersion
-	EventJSON   []byte
+	EventNID  types.EventNID
+	EventJSON []byte
 }
 
 type EventJSON interface {
@@ -36,7 +35,8 @@ type EventStateKeys interface {
 
 type Events interface {
 	InsertEvent(
-		ctx context.Context, txn *sql.Tx, i types.RoomNID, j types.EventTypeNID, k types.EventStateKeyNID, eventID string,
+		ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventTypeNID types.EventTypeNID,
+		eventStateKeyNID types.EventStateKeyNID, eventID string,
 		referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, isRejected bool,
 	) (types.EventNID, types.StateSnapshotNID, error)
 	SelectEvent(ctx context.Context, txn *sql.Tx, eventID string) (types.EventNID, types.StateSnapshotNID, error)

From 1a7f4c8aa978d7e2f6046b6628ecf523460eee28 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 9 May 2022 15:22:33 +0100
Subject: [PATCH 091/391] Don't try to re-fetch the event if it is listed in
 `adds_state_event_ids` (#2437)

* Don't try to re-fetch the event in the output message

* Try that again

* Add the initial event into the set
---
 syncapi/consumers/roomserver.go | 75 ++++++++++++++++++++++++---------
 1 file changed, 55 insertions(+), 20 deletions(-)

diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 7712c8403..e1c2ea823 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -154,41 +154,76 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 	ctx context.Context, msg api.OutputNewRoomEvent,
 ) error {
 	ev := msg.Event
-
 	addsStateEvents := []*gomatrixserverlib.HeaderedEvent{}
-	foundEventIDs := map[string]bool{}
-	if len(msg.AddsStateEventIDs) > 0 {
-		for _, eventID := range msg.AddsStateEventIDs {
-			foundEventIDs[eventID] = false
+
+	// Work out the list of events we need to find out about. Either
+	// they will be the event supplied in the request, we will find it
+	// in the sync API database or we'll need to ask the roomserver.
+	knownEventIDs := make(map[string]bool, len(msg.AddsStateEventIDs))
+	for _, eventID := range msg.AddsStateEventIDs {
+		if eventID == ev.EventID() {
+			knownEventIDs[eventID] = true
+			addsStateEvents = append(addsStateEvents, ev)
+		} else {
+			knownEventIDs[eventID] = false
 		}
-		foundEvents, err := s.db.Events(ctx, msg.AddsStateEventIDs)
+	}
+
+	// Work out which events we want to look up in the sync API database.
+	// At this stage the only event that should be excluded is the event
+	// supplied in the request, if it appears in the adds_state_event_ids.
+	missingEventIDs := make([]string, 0, len(msg.AddsStateEventIDs))
+	for eventID, known := range knownEventIDs {
+		if !known {
+			missingEventIDs = append(missingEventIDs, eventID)
+		}
+	}
+
+	// Look the events up in the database. If we know them, add them into
+	// the set of adds state events.
+	if len(missingEventIDs) > 0 {
+		alreadyKnown, err := s.db.Events(ctx, msg.AddsStateEventIDs)
 		if err != nil {
 			return fmt.Errorf("s.db.Events: %w", err)
 		}
-		for _, event := range foundEvents {
-			foundEventIDs[event.EventID()] = true
+		for _, knownEvent := range alreadyKnown {
+			knownEventIDs[knownEvent.EventID()] = true
+			addsStateEvents = append(addsStateEvents, knownEvent)
+		}
+	}
+
+	// Now work out if there are any remaining events we don't know. For
+	// these we will need to ask the roomserver for help.
+	missingEventIDs = missingEventIDs[:0]
+	for eventID, known := range knownEventIDs {
+		if !known {
+			missingEventIDs = append(missingEventIDs, eventID)
+		}
+	}
+
+	// Ask the roomserver and add in the rest of the results into the set.
+	// Finally, work out if there are any more events missing.
+	if len(missingEventIDs) > 0 {
+		eventsReq := &api.QueryEventsByIDRequest{
+			EventIDs: missingEventIDs,
 		}
-		eventsReq := &api.QueryEventsByIDRequest{}
 		eventsRes := &api.QueryEventsByIDResponse{}
-		for eventID, found := range foundEventIDs {
-			if !found {
-				eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
-			}
-		}
-		if err = s.rsAPI.QueryEventsByID(ctx, eventsReq, eventsRes); err != nil {
+		if err := s.rsAPI.QueryEventsByID(ctx, eventsReq, eventsRes); err != nil {
 			return fmt.Errorf("s.rsAPI.QueryEventsByID: %w", err)
 		}
 		for _, event := range eventsRes.Events {
-			eventID := event.EventID()
-			foundEvents = append(foundEvents, event)
-			foundEventIDs[eventID] = true
+			addsStateEvents = append(addsStateEvents, event)
+			knownEventIDs[event.EventID()] = true
 		}
-		for eventID, found := range foundEventIDs {
+
+		// This should never happen because this would imply that the
+		// roomserver has sent us adds_state_event_ids for events that it
+		// also doesn't know about, but let's just be sure.
+		for eventID, found := range knownEventIDs {
 			if !found {
 				return fmt.Errorf("event %s is missing", eventID)
 			}
 		}
-		addsStateEvents = foundEvents
 	}
 
 	ev, err := s.updateStateEvent(ev)

From 79da75d483d3ee554722000975e13776e4e8a656 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 9 May 2022 16:19:35 +0100
Subject: [PATCH 092/391] Federation consumer `adds_state_event_ids` tweak
 (#2441)

* Don't ask roomserver for events we already have in federation API

* Check number of events returned is as expected

* Preallocate array

* Improve shape a bit
---
 federationapi/consumers/roomserver.go | 25 +++++++++++--------------
 roomserver/api/output.go              | 13 +++++++++++++
 syncapi/consumers/roomserver.go       | 23 ++++-------------------
 3 files changed, 28 insertions(+), 33 deletions(-)

diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index ff2c8e5d4..80317ee69 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -146,28 +146,25 @@ func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPee
 // processMessage updates the list of currently joined hosts in the room
 // and then sends the event to the hosts that were joined before the event.
 func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error {
-	eventsRes := &api.QueryEventsByIDResponse{}
-	if len(ore.AddsStateEventIDs) > 0 {
+	addsStateEvents, missingEventIDs := ore.NeededStateEventIDs()
+
+	// Ask the roomserver and add in the rest of the results into the set.
+	// Finally, work out if there are any more events missing.
+	if len(missingEventIDs) > 0 {
 		eventsReq := &api.QueryEventsByIDRequest{
-			EventIDs: ore.AddsStateEventIDs,
+			EventIDs: missingEventIDs,
 		}
+		eventsRes := &api.QueryEventsByIDResponse{}
 		if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
 			return fmt.Errorf("s.rsAPI.QueryEventsByID: %w", err)
 		}
-
-		found := false
-		for _, event := range eventsRes.Events {
-			if event.EventID() == ore.Event.EventID() {
-				found = true
-				break
-			}
-		}
-		if !found {
-			eventsRes.Events = append(eventsRes.Events, ore.Event)
+		if len(eventsRes.Events) != len(missingEventIDs) {
+			return fmt.Errorf("missing state events")
 		}
+		addsStateEvents = append(addsStateEvents, eventsRes.Events...)
 	}
 
-	addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(eventsRes.Events))
+	addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents))
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/api/output.go b/roomserver/api/output.go
index 767611ec4..a82bf8701 100644
--- a/roomserver/api/output.go
+++ b/roomserver/api/output.go
@@ -163,6 +163,19 @@ type OutputNewRoomEvent struct {
 	TransactionID *TransactionID `json:"transaction_id,omitempty"`
 }
 
+func (o *OutputNewRoomEvent) NeededStateEventIDs() ([]*gomatrixserverlib.HeaderedEvent, []string) {
+	addsStateEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, 1)
+	missingEventIDs := make([]string, 0, len(o.AddsStateEventIDs))
+	for _, eventID := range o.AddsStateEventIDs {
+		if eventID != o.Event.EventID() {
+			missingEventIDs = append(missingEventIDs, eventID)
+		} else {
+			addsStateEvents = append(addsStateEvents, o.Event)
+		}
+	}
+	return addsStateEvents, missingEventIDs
+}
+
 // An OutputOldRoomEvent is written when the roomserver receives an old event.
 // This will typically happen as a result of getting either missing events
 // or backfilling. Downstream components may wish to send these events to
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index e1c2ea823..63bde8166 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -154,35 +154,20 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 	ctx context.Context, msg api.OutputNewRoomEvent,
 ) error {
 	ev := msg.Event
-	addsStateEvents := []*gomatrixserverlib.HeaderedEvent{}
+	addsStateEvents, missingEventIDs := msg.NeededStateEventIDs()
 
 	// Work out the list of events we need to find out about. Either
 	// they will be the event supplied in the request, we will find it
 	// in the sync API database or we'll need to ask the roomserver.
 	knownEventIDs := make(map[string]bool, len(msg.AddsStateEventIDs))
-	for _, eventID := range msg.AddsStateEventIDs {
-		if eventID == ev.EventID() {
-			knownEventIDs[eventID] = true
-			addsStateEvents = append(addsStateEvents, ev)
-		} else {
-			knownEventIDs[eventID] = false
-		}
-	}
-
-	// Work out which events we want to look up in the sync API database.
-	// At this stage the only event that should be excluded is the event
-	// supplied in the request, if it appears in the adds_state_event_ids.
-	missingEventIDs := make([]string, 0, len(msg.AddsStateEventIDs))
-	for eventID, known := range knownEventIDs {
-		if !known {
-			missingEventIDs = append(missingEventIDs, eventID)
-		}
+	for _, eventID := range missingEventIDs {
+		knownEventIDs[eventID] = false
 	}
 
 	// Look the events up in the database. If we know them, add them into
 	// the set of adds state events.
 	if len(missingEventIDs) > 0 {
-		alreadyKnown, err := s.db.Events(ctx, msg.AddsStateEventIDs)
+		alreadyKnown, err := s.db.Events(ctx, missingEventIDs)
 		if err != nil {
 			return fmt.Errorf("s.db.Events: %w", err)
 		}

From a443d1e5f3796942f68067741f4bdd482548bfd7 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 9 May 2022 16:25:22 +0100
Subject: [PATCH 093/391] Don't store invites in sync API that aren't relevant
 to local users (#2439)

---
 syncapi/consumers/roomserver.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 63bde8166..f0ca2106f 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -347,9 +347,11 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent(
 	ctx context.Context, msg api.OutputNewInviteEvent,
 ) {
 	if msg.Event.StateKey() == nil {
-		log.WithFields(log.Fields{
-			"event": string(msg.Event.JSON()),
-		}).Panicf("roomserver output log: invite has no state key")
+		return
+	}
+	if _, serverName, err := gomatrixserverlib.SplitID('@', *msg.Event.StateKey()); err != nil {
+		return
+	} else if serverName != s.cfg.Matrix.ServerName {
 		return
 	}
 	pduPos, err := s.db.AddInviteEvent(ctx, msg.Event)

From 236b16aa6c97bc0894388dce7f6b420ef7a1fd88 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Mon, 9 May 2022 17:23:02 +0100
Subject: [PATCH 094/391] Begin adding syncapi component tests (#2442)

* Add very basic syncapi tests

* Add a way to inject jetstream messages

* implement add_state_ids

* bugfixes

* Unbreak tests

* Remove now un-needed API call

* Linting
---
 federationapi/federationapi_keys_test.go |   2 +-
 setup/base/base.go                       |  12 +-
 setup/jetstream/nats.go                  |   8 ++
 syncapi/sync/requestpool.go              |  10 +-
 syncapi/syncapi.go                       |   2 +-
 syncapi/syncapi_test.go                  | 162 +++++++++++++++++++++++
 test/base.go                             |  72 ++++++++++
 test/http.go                             |  45 +++++++
 test/jetstream.go                        |  35 +++++
 9 files changed, 337 insertions(+), 11 deletions(-)
 create mode 100644 syncapi/syncapi_test.go
 create mode 100644 test/http.go
 create mode 100644 test/jetstream.go

diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go
index 4774c8820..31e9a4c73 100644
--- a/federationapi/federationapi_keys_test.go
+++ b/federationapi/federationapi_keys_test.go
@@ -102,7 +102,7 @@ func TestMain(m *testing.M) {
 		)
 
 		// Finally, build the server key APIs.
-		sbase := base.NewBaseDendrite(cfg, "Monolith", base.NoCacheMetrics)
+		sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
 		s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
 	}
 
diff --git a/setup/base/base.go b/setup/base/base.go
index 0e7528a03..5cbd7da9c 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -86,6 +86,7 @@ type BaseDendrite struct {
 	DNSCache               *gomatrixserverlib.DNSCache
 	Database               *sql.DB
 	DatabaseWriter         sqlutil.Writer
+	EnableMetrics          bool
 }
 
 const NoListener = ""
@@ -96,7 +97,7 @@ const HTTPClientTimeout = time.Second * 30
 type BaseDendriteOptions int
 
 const (
-	NoCacheMetrics BaseDendriteOptions = iota
+	DisableMetrics BaseDendriteOptions = iota
 	UseHTTPAPIs
 	PolylithMode
 )
@@ -107,12 +108,12 @@ const (
 func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...BaseDendriteOptions) *BaseDendrite {
 	platformSanityChecks()
 	useHTTPAPIs := false
-	cacheMetrics := true
+	enableMetrics := true
 	isMonolith := true
 	for _, opt := range options {
 		switch opt {
-		case NoCacheMetrics:
-			cacheMetrics = false
+		case DisableMetrics:
+			enableMetrics = false
 		case UseHTTPAPIs:
 			useHTTPAPIs = true
 		case PolylithMode:
@@ -160,7 +161,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		}
 	}
 
-	cache, err := caching.NewInMemoryLRUCache(cacheMetrics)
+	cache, err := caching.NewInMemoryLRUCache(enableMetrics)
 	if err != nil {
 		logrus.WithError(err).Warnf("Failed to create cache")
 	}
@@ -246,6 +247,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		apiHttpClient:          &apiClient,
 		Database:               db,     // set if monolith with global connection pool only
 		DatabaseWriter:         writer, // set if monolith with global connection pool only
+		EnableMetrics:          enableMetrics,
 	}
 }
 
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 426f02bb6..248b0e656 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -13,6 +13,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	natsserver "github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
 	natsclient "github.com/nats-io/nats.go"
 )
 
@@ -21,6 +22,13 @@ type NATSInstance struct {
 	sync.Mutex
 }
 
+func DeleteAllStreams(js nats.JetStreamContext, cfg *config.JetStream) {
+	for _, stream := range streams { // streams are defined in streams.go
+		name := cfg.Prefixed(stream.Name)
+		_ = js.DeleteStream(name)
+	}
+}
+
 func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) {
 	// check if we need an in-process NATS Server
 	if len(cfg.Addresses) != 0 {
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 99d1e40c3..8ab130911 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -65,11 +65,13 @@ func NewRequestPool(
 	userAPI userapi.SyncUserAPI, keyAPI keyapi.SyncKeyAPI,
 	rsAPI roomserverAPI.SyncRoomserverAPI,
 	streams *streams.Streams, notifier *notifier.Notifier,
-	producer PresencePublisher,
+	producer PresencePublisher, enableMetrics bool,
 ) *RequestPool {
-	prometheus.MustRegister(
-		activeSyncRequests, waitingSyncRequests,
-	)
+	if enableMetrics {
+		prometheus.MustRegister(
+			activeSyncRequests, waitingSyncRequests,
+		)
+	}
 	rp := &RequestPool{
 		db:       db,
 		cfg:      cfg,
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index dbc6e240c..d8bacb2da 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -65,7 +65,7 @@ func AddPublicRoutes(
 		JetStream: js,
 	}
 
-	requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer)
+	requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, base.EnableMetrics)
 
 	userAPIStreamEventProducer := &producers.UserAPIStreamEventProducer{
 		JetStream: js,
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
new file mode 100644
index 000000000..12b5178d8
--- /dev/null
+++ b/syncapi/syncapi_test.go
@@ -0,0 +1,162 @@
+package syncapi
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	keyapi "github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/test"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/nats-io/nats.go"
+)
+
+type syncRoomserverAPI struct {
+	rsapi.SyncRoomserverAPI
+	rooms []*test.Room
+}
+
+func (s *syncRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error {
+	var room *test.Room
+	for _, r := range s.rooms {
+		if r.ID == req.RoomID {
+			room = r
+			break
+		}
+	}
+	if room == nil {
+		res.RoomExists = false
+		return nil
+	}
+	res.RoomVersion = room.Version
+	return nil // TODO: return state
+}
+
+type syncUserAPI struct {
+	userapi.SyncUserAPI
+	accounts []userapi.Device
+}
+
+func (s *syncUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error {
+	for _, acc := range s.accounts {
+		if acc.AccessToken == req.AccessToken {
+			res.Device = &acc
+			return nil
+		}
+	}
+	res.Err = "unknown user"
+	return nil
+}
+
+func (s *syncUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error {
+	return nil
+}
+
+type syncKeyAPI struct {
+	keyapi.KeyInternalAPI
+}
+
+func TestSyncAPI(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testSync(t, dbType)
+	})
+}
+
+func testSync(t *testing.T, dbType test.DBType) {
+	user := test.NewUser()
+	room := test.NewRoom(t, user)
+	alice := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      user.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "Alice",
+		AccountType: userapi.AccountTypeUser,
+	}
+
+	base, close := test.CreateBaseDendrite(t, dbType)
+	defer close()
+
+	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+	var msgs []*nats.Msg
+	for _, ev := range room.Events() {
+		var addsStateIDs []string
+		if ev.StateKey() != nil {
+			addsStateIDs = append(addsStateIDs, ev.EventID())
+		}
+		msgs = append(msgs, test.NewOutputEventMsg(t, base, room.ID, api.OutputEvent{
+			Type: rsapi.OutputTypeNewRoomEvent,
+			NewRoomEvent: &rsapi.OutputNewRoomEvent{
+				Event:             ev,
+				AddsStateEventIDs: addsStateIDs,
+			},
+		}))
+	}
+	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
+	test.MustPublishMsgs(t, jsctx, msgs...)
+
+	testCases := []struct {
+		name            string
+		req             *http.Request
+		wantCode        int
+		wantJoinedRooms []string
+	}{
+		{
+			name: "missing access token",
+			req: test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+				"timeout": "0",
+			})),
+			wantCode: 401,
+		},
+		{
+			name: "unknown access token",
+			req: test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+				"access_token": "foo",
+				"timeout":      "0",
+			})),
+			wantCode: 401,
+		},
+		{
+			name: "valid access token",
+			req: test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+				"access_token": alice.AccessToken,
+				"timeout":      "0",
+			})),
+			wantCode:        200,
+			wantJoinedRooms: []string{room.ID},
+		},
+	}
+	// TODO: find a better way
+	time.Sleep(500 * time.Millisecond)
+
+	for _, tc := range testCases {
+		w := httptest.NewRecorder()
+		base.PublicClientAPIMux.ServeHTTP(w, tc.req)
+		if w.Code != tc.wantCode {
+			t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode)
+		}
+		if tc.wantJoinedRooms != nil {
+			var res types.Response
+			if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+				t.Fatalf("%s: failed to decode response body: %s", tc.name, err)
+			}
+			if len(res.Rooms.Join) != len(tc.wantJoinedRooms) {
+				t.Errorf("%s: got %v joined rooms, want %v.\nResponse: %+v", tc.name, len(res.Rooms.Join), len(tc.wantJoinedRooms), res)
+			}
+			t.Logf("res: %+v", res.Rooms.Join[room.ID])
+
+			gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events))
+			for i, ev := range res.Rooms.Join[room.ID].Timeline.Events {
+				gotEventIDs[i] = ev.EventID
+			}
+			test.AssertEventIDsEqual(t, gotEventIDs, room.Events())
+		}
+	}
+}
diff --git a/test/base.go b/test/base.go
index 32fc8dc53..664442c03 100644
--- a/test/base.go
+++ b/test/base.go
@@ -1,11 +1,83 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package test
 
 import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"strings"
+	"testing"
+
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/nats-io/nats.go"
 )
 
+func CreateBaseDendrite(t *testing.T, dbType DBType) (*base.BaseDendrite, func()) {
+	var cfg config.Dendrite
+	cfg.Defaults(false)
+	cfg.Global.JetStream.InMemory = true
+
+	switch dbType {
+	case DBTypePostgres:
+		cfg.Global.Defaults(true)   // autogen a signing key
+		cfg.MediaAPI.Defaults(true) // autogen a media path
+		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
+		// the file system event with InMemory=true :(
+		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
+		connStr, close := PrepareDBConnectionString(t, dbType)
+		cfg.Global.DatabaseOptions = config.DatabaseOptions{
+			ConnectionString:       config.DataSource(connStr),
+			MaxOpenConnections:     10,
+			MaxIdleConnections:     2,
+			ConnMaxLifetimeSeconds: 60,
+		}
+		return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), close
+	case DBTypeSQLite:
+		cfg.Defaults(true) // sets a sqlite db per component
+		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
+		// the file system event with InMemory=true :(
+		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
+		return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), func() {
+			// cleanup db files. This risks getting out of sync as we add more database strings :(
+			dbFiles := []config.DataSource{
+				cfg.AppServiceAPI.Database.ConnectionString,
+				cfg.FederationAPI.Database.ConnectionString,
+				cfg.KeyServer.Database.ConnectionString,
+				cfg.MSCs.Database.ConnectionString,
+				cfg.MediaAPI.Database.ConnectionString,
+				cfg.RoomServer.Database.ConnectionString,
+				cfg.SyncAPI.Database.ConnectionString,
+				cfg.UserAPI.AccountDatabase.ConnectionString,
+			}
+			for _, fileURI := range dbFiles {
+				path := strings.TrimPrefix(string(fileURI), "file:")
+				err := os.Remove(path)
+				if err != nil && !errors.Is(err, fs.ErrNotExist) {
+					t.Fatalf("failed to cleanup sqlite db '%s': %s", fileURI, err)
+				}
+			}
+		}
+	default:
+		t.Fatalf("unknown db type: %v", dbType)
+	}
+	return nil, nil
+}
+
 func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) {
 	if cfg == nil {
 		cfg = &config.Dendrite{}
diff --git a/test/http.go b/test/http.go
new file mode 100644
index 000000000..a458a3385
--- /dev/null
+++ b/test/http.go
@@ -0,0 +1,45 @@
+package test
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+type HTTPRequestOpt func(req *http.Request)
+
+func WithJSONBody(t *testing.T, body interface{}) HTTPRequestOpt {
+	t.Helper()
+	b, err := json.Marshal(body)
+	if err != nil {
+		t.Fatalf("WithJSONBody: %s", err)
+	}
+	return func(req *http.Request) {
+		req.Body = io.NopCloser(bytes.NewBuffer(b))
+	}
+}
+
+func WithQueryParams(qps map[string]string) HTTPRequestOpt {
+	var vals url.Values = map[string][]string{}
+	for k, v := range qps {
+		vals.Set(k, v)
+	}
+	return func(req *http.Request) {
+		req.URL.RawQuery = vals.Encode()
+	}
+}
+
+func NewRequest(t *testing.T, method, path string, opts ...HTTPRequestOpt) *http.Request {
+	t.Helper()
+	req, err := http.NewRequest(method, "http://localhost"+path, nil)
+	if err != nil {
+		t.Fatalf("failed to make new HTTP request %v %v : %v", method, path, err)
+	}
+	for _, o := range opts {
+		o(req)
+	}
+	return req
+}
diff --git a/test/jetstream.go b/test/jetstream.go
new file mode 100644
index 000000000..488c22beb
--- /dev/null
+++ b/test/jetstream.go
@@ -0,0 +1,35 @@
+package test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/nats-io/nats.go"
+)
+
+func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Msg) {
+	t.Helper()
+	for _, msg := range msgs {
+		if _, err := jsctx.PublishMsg(msg); err != nil {
+			t.Fatalf("MustPublishMsgs: failed to publish message: %s", err)
+		}
+	}
+}
+
+func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg {
+	t.Helper()
+	msg := &nats.Msg{
+		Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
+		Header:  nats.Header{},
+	}
+	msg.Header.Set(jetstream.RoomID, roomID)
+	var err error
+	msg.Data, err = json.Marshal(update)
+	if err != nil {
+		t.Fatalf("failed to marshal update: %s", err)
+	}
+	return msg
+}

From 6b3c183396233a6d03102535b238b713617ae2ac Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 9 May 2022 17:31:14 +0100
Subject: [PATCH 095/391] Version 0.8.3 (#2431)

* Version 0.8.3

* Update changelog
---
 CHANGES.md          | 36 ++++++++++++++++++++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 6278bcba4..b13908f73 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,41 @@
 # Changelog
 
+## Dendrite 0.8.3 (2022-05-09)
+
+### Features
+
+* Open registration is now harder to enable, which should reduce the chance that Dendrite servers will be used to conduct spam or abuse attacks
+  * Dendrite will only enable open registration if you pass the `--really-enable-open-registration` command line flag at startup
+  * If open registration is enabled but this command line flag is not passed, Dendrite will fail to start up
+* Dendrite now supports phone-home statistic reporting
+  * These statistics include things like the number of registered and active users, some configuration options and platform/environment details, to help us to understand how Dendrite is used
+  * This is not enabled by default — it must be enabled in the `global.report_stats` section of the config file
+* Monolith installations can now be configured with a single global database connection pool (in `global.database` in the config) rather than having to configure each component separately
+  * This also means that you no longer need to balance connection counts between different components, as they will share the same larger pool
+  * Specific components can override the global database settings by specifying their own `database` block
+  * To use only the global pool, you must configure `global.database` and then remove the `database` block from all of the component sections of the config file
+* A new admin API endpoint `/_dendrite/admin/evacuateRoom/{roomID}` has been added, allowing server admins to forcefully part all local users from a given room
+* The sync notifier now only loads members for the relevant rooms, which should reduce CPU usage and load on the database
+* A number of component interfaces have been refactored for cleanliness and developer ease
+* Event auth errors in the log should now be much more useful, including the reason for the event failures
+* The forward extremity calculation in the roomserver has been simplified
+* A new index has been added to the one-time keys table in the keyserver which should speed up key count lookups
+
+### Fixes
+
+* Dendrite will no longer process events for rooms where there are no local users joined, which should help to reduce CPU and RAM usage
+* A bug has been fixed in event auth when changing the user levels in `m.room.power_levels` events
+* Usernames should no longer be duplicated when no room name is set
+* Device display names should now be correctly propagated over federation
+* A panic when uploading cross-signing signatures has been fixed
+* Presence is now correctly limited in `/sync` based on the filters
+* The presence stream position returned by `/sync` will now be correct if no presence events were returned
+* The media `/config` endpoint will no longer return a maximum upload size field if it is configured to be unlimited in the Dendrite config
+* The server notices room will no longer produce "User is already joined to the room" errors
+* Consumer errors will no longer flood the logs during a graceful shutdown
+* Sync API and federation API consumers will no longer unnecessarily query added state events matching the one in the output event
+* The Sync API will no longer unnecessarily track invites for remote users
+
 ## Dendrite 0.8.2 (2022-04-27)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index e74548831..08c02cfcd 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -18,7 +18,7 @@ const (
 	VersionMajor = 0
 	VersionMinor = 8
 	VersionPatch = 3
-	VersionTag   = "rc1" // example: "rc1"
+	VersionTag   = "" // example: "rc1"
 )
 
 func VersionString() string {

From 1b3fa9689ca28de2337b67253089e286694c60e9 Mon Sep 17 00:00:00 2001
From: database64128 <free122448@hotmail.com>
Date: Tue, 10 May 2022 00:51:30 +0800
Subject: [PATCH 096/391] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20mediaapi/thumbn?=
 =?UTF-8?q?ailer:=20fix=20build=20with=20bimg=20(#2440)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: kegsay <kegan@matrix.org>
---
 mediaapi/thumbnailer/thumbnailer_bimg.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mediaapi/thumbnailer/thumbnailer_bimg.go b/mediaapi/thumbnailer/thumbnailer_bimg.go
index 6ca533176..fa1acbf08 100644
--- a/mediaapi/thumbnailer/thumbnailer_bimg.go
+++ b/mediaapi/thumbnailer/thumbnailer_bimg.go
@@ -37,7 +37,7 @@ func GenerateThumbnails(
 	mediaMetadata *types.MediaMetadata,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	logger *log.Entry,
 ) (busy bool, errorReturn error) {
 	buffer, err := bimg.Read(string(src))
@@ -49,7 +49,7 @@ func GenerateThumbnails(
 	for _, config := range configs {
 		// Note: createThumbnail does locking based on activeThumbnailGeneration
 		busy, err = createThumbnail(
-			ctx, src, img, config, mediaMetadata, activeThumbnailGeneration,
+			ctx, src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration,
 			maxThumbnailGenerators, db, logger,
 		)
 		if err != nil {
@@ -71,7 +71,7 @@ func GenerateThumbnail(
 	mediaMetadata *types.MediaMetadata,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	logger *log.Entry,
 ) (busy bool, errorReturn error) {
 	buffer, err := bimg.Read(string(src))
@@ -109,7 +109,7 @@ func createThumbnail(
 	mediaMetadata *types.MediaMetadata,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	logger *log.Entry,
 ) (busy bool, errorReturn error) {
 	logger = logger.WithFields(log.Fields{

From 77722c5a4f5330f6fe517edc2d11bcba8c1fc274 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 10 May 2022 11:08:10 +0100
Subject: [PATCH 097/391] Back out matrix-org/dendrite#2421 by restoring
 `http.Client`s

This creates problems with non-HTTPS endpoints and should fix #2444.
---
 appservice/appservice.go                    | 23 +++++++++++++--------
 appservice/query/query.go                   |  8 +++----
 appservice/workers/transaction_scheduler.go |  8 +++----
 clientapi/threepid/invites.go               |  4 ++--
 internal/pushgateway/client.go              | 22 ++++++++++++--------
 userapi/util/phonehomestats.go              | 12 ++++++-----
 6 files changed, 44 insertions(+), 33 deletions(-)

diff --git a/appservice/appservice.go b/appservice/appservice.go
index c5ae9ceb2..8fe1b2fc4 100644
--- a/appservice/appservice.go
+++ b/appservice/appservice.go
@@ -16,6 +16,8 @@ package appservice
 
 import (
 	"context"
+	"crypto/tls"
+	"net/http"
 	"sync"
 	"time"
 
@@ -33,7 +35,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddInternalRoutes registers HTTP handlers for internal API calls
@@ -45,15 +46,19 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInte
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite,
-	userAPI userapi.AppserviceUserAPI,
-	rsAPI roomserverAPI.AppserviceRoomserverAPI,
+	userAPI userapi.UserInternalAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
-	client := gomatrixserverlib.NewClient(
-		gomatrixserverlib.WithTimeout(time.Second*30),
-		gomatrixserverlib.WithKeepAlives(false),
-		gomatrixserverlib.WithSkipVerify(base.Cfg.AppServiceAPI.DisableTLSValidation),
-	)
-
+	client := &http.Client{
+		Timeout: time.Second * 30,
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
+			},
+			Proxy: http.ProxyFromEnvironment,
+		},
+	}
 	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 
 	// Create a connection to the appservice postgres DB
diff --git a/appservice/query/query.go b/appservice/query/query.go
index b7b0b335a..dacd3caa8 100644
--- a/appservice/query/query.go
+++ b/appservice/query/query.go
@@ -23,7 +23,6 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
 	opentracing "github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 )
@@ -33,7 +32,7 @@ const userIDExistsPath = "/users/"
 
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
-	HTTPClient *gomatrixserverlib.Client
+	HTTPClient *http.Client
 	Cfg        *config.Dendrite
 }
 
@@ -65,8 +64,9 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
+			req = req.WithContext(ctx)
 
-			resp, err := a.HTTPClient.DoHTTPRequest(ctx, req)
+			resp, err := a.HTTPClient.Do(req)
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -130,7 +130,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
-			resp, err := a.HTTPClient.DoHTTPRequest(ctx, req)
+			resp, err := a.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go
index 47d447c2c..4dab00bd7 100644
--- a/appservice/workers/transaction_scheduler.go
+++ b/appservice/workers/transaction_scheduler.go
@@ -42,7 +42,7 @@ var (
 // size), then send that off to the AS's /transactions/{txnID} endpoint. It also
 // handles exponentially backing off in case the AS isn't currently available.
 func SetupTransactionWorkers(
-	client *gomatrixserverlib.Client,
+	client *http.Client,
 	appserviceDB storage.Database,
 	workerStates []types.ApplicationServiceWorkerState,
 ) error {
@@ -58,7 +58,7 @@ func SetupTransactionWorkers(
 
 // worker is a goroutine that sends any queued events to the application service
 // it is given.
-func worker(client *gomatrixserverlib.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
+func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
 	log.WithFields(log.Fields{
 		"appservice": ws.AppService.ID,
 	}).Info("Starting application service")
@@ -200,7 +200,7 @@ func createTransaction(
 // send sends events to an application service. Returns an error if an OK was not
 // received back from the application service or the request timed out.
 func send(
-	client *gomatrixserverlib.Client,
+	client *http.Client,
 	appservice config.ApplicationService,
 	txnID int,
 	transaction []byte,
@@ -213,7 +213,7 @@ func send(
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.DoHTTPRequest(context.TODO(), req)
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go
index 6e7426a7f..9670fecad 100644
--- a/clientapi/threepid/invites.go
+++ b/clientapi/threepid/invites.go
@@ -231,7 +231,7 @@ func queryIDServerStoreInvite(
 		profile = &authtypes.Profile{}
 	}
 
-	client := gomatrixserverlib.NewClient()
+	client := http.Client{}
 
 	data := url.Values{}
 	data.Add("medium", body.Medium)
@@ -253,7 +253,7 @@ func queryIDServerStoreInvite(
 	}
 
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	resp, err := client.DoHTTPRequest(ctx, req)
+	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/pushgateway/client.go b/internal/pushgateway/client.go
index 231327a1e..95f5afd90 100644
--- a/internal/pushgateway/client.go
+++ b/internal/pushgateway/client.go
@@ -3,28 +3,32 @@ package pushgateway
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/opentracing/opentracing-go"
 )
 
 type httpClient struct {
-	hc *gomatrixserverlib.Client
+	hc *http.Client
 }
 
 // NewHTTPClient creates a new Push Gateway client.
 func NewHTTPClient(disableTLSValidation bool) Client {
-	return &httpClient{
-		hc: gomatrixserverlib.NewClient(
-			gomatrixserverlib.WithTimeout(time.Second*30),
-			gomatrixserverlib.WithKeepAlives(false),
-			gomatrixserverlib.WithSkipVerify(disableTLSValidation),
-		),
+	hc := &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: disableTLSValidation,
+			},
+			Proxy: http.ProxyFromEnvironment,
+		},
 	}
+	return &httpClient{hc: hc}
 }
 
 func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest, resp *NotifyResponse) error {
@@ -41,7 +45,7 @@ func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest,
 	}
 	hreq.Header.Set("Content-Type", "application/json")
 
-	hresp, err := h.hc.DoHTTPRequest(ctx, hreq)
+	hresp, err := h.hc.Do(hreq)
 	if err != nil {
 		return err
 	}
diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go
index e24daba6b..ad93a50e3 100644
--- a/userapi/util/phonehomestats.go
+++ b/userapi/util/phonehomestats.go
@@ -39,7 +39,7 @@ type phoneHomeStats struct {
 	cfg        *config.Dendrite
 	db         storage.Statistics
 	isMonolith bool
-	client     *gomatrixserverlib.Client
+	client     *http.Client
 }
 
 type timestampToRUUsage struct {
@@ -55,9 +55,10 @@ func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB
 		cfg:        cfg,
 		db:         statsDB,
 		isMonolith: cfg.IsMonolith,
-		client: gomatrixserverlib.NewClient(
-			gomatrixserverlib.WithTimeout(time.Second * 30),
-		),
+		client: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: http.DefaultTransport,
+		},
 	}
 
 	// start initial run after 5min
@@ -151,7 +152,8 @@ func (p *phoneHomeStats) collect() {
 	}
 	request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
 
-	if _, err = p.client.DoHTTPRequest(ctx, request); err != nil {
+	_, err = p.client.Do(request)
+	if err != nil {
 		logrus.WithError(err).Error("unable to send anonymous stats")
 		return
 	}

From e2a932ec0b4b1568b48726c9a855485f596d07ce Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 10 May 2022 11:23:36 +0100
Subject: [PATCH 098/391] Add indexes to `syncapi_output_room_events` table
 that satisfy the filters (#2446)

---
 syncapi/storage/postgres/output_room_events_table.go | 5 +++++
 syncapi/storage/sqlite3/output_room_events_table.go  | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 17e2feab6..d84d0cfa2 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -69,6 +69,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
   -- were emitted.
   exclude_from_sync BOOL DEFAULT FALSE
 );
+
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
 `
 
 const insertEventSQL = "" +
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 188f7582b..f9961a9d1 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -49,6 +49,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
   transaction_id TEXT,
   exclude_from_sync BOOL NOT NULL DEFAULT FALSE
 );
+
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
 `
 
 const insertEventSQL = "" +

From 1897e2f1c07f1b06a540aa2b4ccedfc67008e52a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 10 May 2022 12:44:29 +0100
Subject: [PATCH 099/391] Version 0.8.4

---
 CHANGES.md          | 8 ++++++++
 internal/version.go | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index b13908f73..c058da6a1 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,13 @@
 # Changelog
 
+## Dendrite 0.8.4 (2022-05-10)
+
+### Fixes
+
+* Fixes a regression introduced in the previous version where appservices, push and phone-home statistics would not work over plain HTTP
+* Adds missing indexes to the sync API output events table, which should significantly improve `/sync` performance and reduce database CPU usage
+* Building Dendrite with the `bimg` thumbnailer should now work again (contributed by [database64128](https://github.com/database64128))
+
 ## Dendrite 0.8.3 (2022-05-09)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index 08c02cfcd..5097bb2a6 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 3
+	VersionPatch = 4
 	VersionTag   = "" // example: "rc1"
 )
 

From 6db08b2874307c516b10ef9c9e996807fbfdb1ff Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 10 May 2022 14:41:12 +0200
Subject: [PATCH 100/391] Add roomserver tests (2/?) (#2445)

* Add invite table tests; move variable declarations

* Add Membership table tests

* Move variable declarations

* Add PrevEvents table tests

* Add Published table test

* Add Redactions tests
Fix bug in SQLite markRedactionValidatedSQL

* PR comments, better readability for invite tests
---
 roomserver/storage/postgres/invite_table.go   |  10 +-
 .../storage/postgres/membership_table.go      |  20 +--
 .../storage/postgres/previous_events_table.go |   4 +-
 .../storage/postgres/published_table.go       |   6 +-
 .../storage/postgres/redactions_table.go      |   4 +-
 roomserver/storage/postgres/storage.go        |  20 +--
 roomserver/storage/sqlite3/invite_table.go    |  10 +-
 .../storage/sqlite3/membership_table.go       |  20 +--
 .../storage/sqlite3/previous_events_table.go  |   4 +-
 roomserver/storage/sqlite3/published_table.go |   6 +-
 .../storage/sqlite3/redactions_table.go       |   8 +-
 roomserver/storage/sqlite3/storage.go         |  20 +--
 .../storage/tables/invite_table_test.go       |  92 +++++++++++++
 .../storage/tables/membership_table_test.go   | 130 ++++++++++++++++++
 .../tables/previous_events_table_test.go      |  61 ++++++++
 .../storage/tables/published_table_test.go    |  79 +++++++++++
 .../storage/tables/redactions_table_test.go   |  89 ++++++++++++
 17 files changed, 517 insertions(+), 66 deletions(-)
 create mode 100644 roomserver/storage/tables/invite_table_test.go
 create mode 100644 roomserver/storage/tables/membership_table_test.go
 create mode 100644 roomserver/storage/tables/previous_events_table_test.go
 create mode 100644 roomserver/storage/tables/published_table_test.go
 create mode 100644 roomserver/storage/tables/redactions_table_test.go

diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go
index 176c16e48..4cddfe2e9 100644
--- a/roomserver/storage/postgres/invite_table.go
+++ b/roomserver/storage/postgres/invite_table.go
@@ -81,12 +81,12 @@ type inviteStatements struct {
 	updateInviteRetiredStmt             *sql.Stmt
 }
 
-func createInvitesTable(db *sql.DB) error {
+func CreateInvitesTable(db *sql.DB) error {
 	_, err := db.Exec(inviteSchema)
 	return err
 }
 
-func prepareInvitesTable(db *sql.DB) (tables.Invites, error) {
+func PrepareInvitesTable(db *sql.DB) (tables.Invites, error) {
 	s := &inviteStatements{}
 
 	return s, sqlutil.StatementList{
@@ -127,8 +127,8 @@ func (s *inviteStatements) UpdateInviteRetired(
 	defer internal.CloseAndLogIfError(ctx, rows, "updateInviteRetired: rows.close() failed")
 
 	var eventIDs []string
+	var inviteEventID string
 	for rows.Next() {
-		var inviteEventID string
 		if err = rows.Scan(&inviteEventID); err != nil {
 			return nil, err
 		}
@@ -152,9 +152,9 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
 	var eventIDs []string
+	var inviteEventID string
+	var senderUserNID int64
 	for rows.Next() {
-		var inviteEventID string
-		var senderUserNID int64
 		if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil {
 			return nil, nil, err
 		}
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index 6ed5293e4..c01753c3a 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -160,12 +160,12 @@ type membershipStatements struct {
 	selectServerInRoomStmt                          *sql.Stmt
 }
 
-func createMembershipTable(db *sql.DB) error {
+func CreateMembershipTable(db *sql.DB) error {
 	_, err := db.Exec(membershipSchema)
 	return err
 }
 
-func prepareMembershipTable(db *sql.DB) (tables.Membership, error) {
+func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 	s := &membershipStatements{}
 
 	return s, sqlutil.StatementList{
@@ -234,8 +234,8 @@ func (s *membershipStatements) SelectMembershipsFromRoom(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed")
 
+	var eNID types.EventNID
 	for rows.Next() {
-		var eNID types.EventNID
 		if err = rows.Scan(&eNID); err != nil {
 			return
 		}
@@ -262,8 +262,8 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed")
 
+	var eNID types.EventNID
 	for rows.Next() {
-		var eNID types.EventNID
 		if err = rows.Scan(&eNID); err != nil {
 			return
 		}
@@ -298,8 +298,8 @@ func (s *membershipStatements) SelectRoomsWithMembership(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithMembership: rows.close() failed")
 	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var roomNID types.RoomNID
 		if err := rows.Scan(&roomNID); err != nil {
 			return nil, err
 		}
@@ -320,9 +320,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsersSetForRooms: rows.close() failed")
 	result := make(map[types.EventStateKeyNID]int)
+	var userID types.EventStateKeyNID
+	var count int
 	for rows.Next() {
-		var userID types.EventStateKeyNID
-		var count int
 		if err := rows.Scan(&userID, &count); err != nil {
 			return nil, err
 		}
@@ -342,12 +342,12 @@ func (s *membershipStatements) SelectKnownUsers(
 	}
 	result := []string{}
 	defer internal.CloseAndLogIfError(ctx, rows, "SelectKnownUsers: rows.close() failed")
+	var resUserID string
 	for rows.Next() {
-		var userID string
-		if err := rows.Scan(&userID); err != nil {
+		if err := rows.Scan(&resUserID); err != nil {
 			return nil, err
 		}
-		result = append(result, userID)
+		result = append(result, resUserID)
 	}
 	return result, rows.Err()
 }
diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go
index bd4e853eb..26999a290 100644
--- a/roomserver/storage/postgres/previous_events_table.go
+++ b/roomserver/storage/postgres/previous_events_table.go
@@ -64,12 +64,12 @@ type previousEventStatements struct {
 	selectPreviousEventExistsStmt *sql.Stmt
 }
 
-func createPrevEventsTable(db *sql.DB) error {
+func CreatePrevEventsTable(db *sql.DB) error {
 	_, err := db.Exec(previousEventSchema)
 	return err
 }
 
-func preparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) {
+func PreparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) {
 	s := &previousEventStatements{}
 
 	return s, sqlutil.StatementList{
diff --git a/roomserver/storage/postgres/published_table.go b/roomserver/storage/postgres/published_table.go
index 15985fcd6..56fa02f7b 100644
--- a/roomserver/storage/postgres/published_table.go
+++ b/roomserver/storage/postgres/published_table.go
@@ -49,12 +49,12 @@ type publishedStatements struct {
 	selectPublishedStmt    *sql.Stmt
 }
 
-func createPublishedTable(db *sql.DB) error {
+func CreatePublishedTable(db *sql.DB) error {
 	_, err := db.Exec(publishedSchema)
 	return err
 }
 
-func preparePublishedTable(db *sql.DB) (tables.Published, error) {
+func PreparePublishedTable(db *sql.DB) (tables.Published, error) {
 	s := &publishedStatements{}
 
 	return s, sqlutil.StatementList{
@@ -94,8 +94,8 @@ func (s *publishedStatements) SelectAllPublishedRooms(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
 
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/redactions_table.go b/roomserver/storage/postgres/redactions_table.go
index 5614f2bd8..6e2f6712d 100644
--- a/roomserver/storage/postgres/redactions_table.go
+++ b/roomserver/storage/postgres/redactions_table.go
@@ -59,12 +59,12 @@ type redactionStatements struct {
 	markRedactionValidatedStmt                  *sql.Stmt
 }
 
-func createRedactionsTable(db *sql.DB) error {
+func CreateRedactionsTable(db *sql.DB) error {
 	_, err := db.Exec(redactionsSchema)
 	return err
 }
 
-func prepareRedactionsTable(db *sql.DB) (tables.Redactions, error) {
+func PrepareRedactionsTable(db *sql.DB) (tables.Redactions, error) {
 	s := &redactionStatements{}
 
 	return s, sqlutil.StatementList{
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index 34e891490..88df72009 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -89,22 +89,22 @@ func (d *Database) create(db *sql.DB) error {
 	if err := createStateSnapshotTable(db); err != nil {
 		return err
 	}
-	if err := createPrevEventsTable(db); err != nil {
+	if err := CreatePrevEventsTable(db); err != nil {
 		return err
 	}
 	if err := createRoomAliasesTable(db); err != nil {
 		return err
 	}
-	if err := createInvitesTable(db); err != nil {
+	if err := CreateInvitesTable(db); err != nil {
 		return err
 	}
-	if err := createMembershipTable(db); err != nil {
+	if err := CreateMembershipTable(db); err != nil {
 		return err
 	}
-	if err := createPublishedTable(db); err != nil {
+	if err := CreatePublishedTable(db); err != nil {
 		return err
 	}
-	if err := createRedactionsTable(db); err != nil {
+	if err := CreateRedactionsTable(db); err != nil {
 		return err
 	}
 
@@ -140,7 +140,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	prevEvents, err := preparePrevEventsTable(db)
+	prevEvents, err := PreparePrevEventsTable(db)
 	if err != nil {
 		return err
 	}
@@ -148,19 +148,19 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	invites, err := prepareInvitesTable(db)
+	invites, err := PrepareInvitesTable(db)
 	if err != nil {
 		return err
 	}
-	membership, err := prepareMembershipTable(db)
+	membership, err := PrepareMembershipTable(db)
 	if err != nil {
 		return err
 	}
-	published, err := preparePublishedTable(db)
+	published, err := PreparePublishedTable(db)
 	if err != nil {
 		return err
 	}
-	redactions, err := prepareRedactionsTable(db)
+	redactions, err := PrepareRedactionsTable(db)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go
index d54d313a9..e051d63af 100644
--- a/roomserver/storage/sqlite3/invite_table.go
+++ b/roomserver/storage/sqlite3/invite_table.go
@@ -69,12 +69,12 @@ type inviteStatements struct {
 	selectInvitesAboutToRetireStmt      *sql.Stmt
 }
 
-func createInvitesTable(db *sql.DB) error {
+func CreateInvitesTable(db *sql.DB) error {
 	_, err := db.Exec(inviteSchema)
 	return err
 }
 
-func prepareInvitesTable(db *sql.DB) (tables.Invites, error) {
+func PrepareInvitesTable(db *sql.DB) (tables.Invites, error) {
 	s := &inviteStatements{
 		db: db,
 	}
@@ -119,8 +119,8 @@ func (s *inviteStatements) UpdateInviteRetired(
 		return
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "UpdateInviteRetired: rows.close() failed")
+	var inviteEventID string
 	for rows.Next() {
-		var inviteEventID string
 		if err = rows.Scan(&inviteEventID); err != nil {
 			return
 		}
@@ -147,9 +147,9 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
 	var eventIDs []string
+	var eventID string
+	var senderUserNID int64
 	for rows.Next() {
-		var eventID string
-		var senderUserNID int64
 		if err := rows.Scan(&eventID, &senderUserNID); err != nil {
 			return nil, nil, err
 		}
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index 7ed86b612..6f0fe8b64 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -136,12 +136,12 @@ type membershipStatements struct {
 	selectServerInRoomStmt                          *sql.Stmt
 }
 
-func createMembershipTable(db *sql.DB) error {
+func CreateMembershipTable(db *sql.DB) error {
 	_, err := db.Exec(membershipSchema)
 	return err
 }
 
-func prepareMembershipTable(db *sql.DB) (tables.Membership, error) {
+func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 	s := &membershipStatements{
 		db: db,
 	}
@@ -212,8 +212,8 @@ func (s *membershipStatements) SelectMembershipsFromRoom(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed")
 
+	var eNID types.EventNID
 	for rows.Next() {
-		var eNID types.EventNID
 		if err = rows.Scan(&eNID); err != nil {
 			return
 		}
@@ -239,8 +239,8 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed")
 
+	var eNID types.EventNID
 	for rows.Next() {
-		var eNID types.EventNID
 		if err = rows.Scan(&eNID); err != nil {
 			return
 		}
@@ -275,8 +275,8 @@ func (s *membershipStatements) SelectRoomsWithMembership(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithMembership: rows.close() failed")
 	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var roomNID types.RoomNID
 		if err := rows.Scan(&roomNID); err != nil {
 			return nil, err
 		}
@@ -307,9 +307,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context,
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsersSetForRooms: rows.close() failed")
 	result := make(map[types.EventStateKeyNID]int)
+	var userID types.EventStateKeyNID
+	var count int
 	for rows.Next() {
-		var userID types.EventStateKeyNID
-		var count int
 		if err := rows.Scan(&userID, &count); err != nil {
 			return nil, err
 		}
@@ -326,12 +326,12 @@ func (s *membershipStatements) SelectKnownUsers(ctx context.Context, txn *sql.Tx
 	}
 	result := []string{}
 	defer internal.CloseAndLogIfError(ctx, rows, "SelectKnownUsers: rows.close() failed")
+	var resUserID string
 	for rows.Next() {
-		var userID string
-		if err := rows.Scan(&userID); err != nil {
+		if err := rows.Scan(&resUserID); err != nil {
 			return nil, err
 		}
-		result = append(result, userID)
+		result = append(result, resUserID)
 	}
 	return result, rows.Err()
 }
diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go
index 7304bf0d5..2a146ef64 100644
--- a/roomserver/storage/sqlite3/previous_events_table.go
+++ b/roomserver/storage/sqlite3/previous_events_table.go
@@ -70,12 +70,12 @@ type previousEventStatements struct {
 	selectPreviousEventExistsStmt *sql.Stmt
 }
 
-func createPrevEventsTable(db *sql.DB) error {
+func CreatePrevEventsTable(db *sql.DB) error {
 	_, err := db.Exec(previousEventSchema)
 	return err
 }
 
-func preparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) {
+func PreparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) {
 	s := &previousEventStatements{
 		db: db,
 	}
diff --git a/roomserver/storage/sqlite3/published_table.go b/roomserver/storage/sqlite3/published_table.go
index 9e416ace3..50dfa5492 100644
--- a/roomserver/storage/sqlite3/published_table.go
+++ b/roomserver/storage/sqlite3/published_table.go
@@ -49,12 +49,12 @@ type publishedStatements struct {
 	selectPublishedStmt    *sql.Stmt
 }
 
-func createPublishedTable(db *sql.DB) error {
+func CreatePublishedTable(db *sql.DB) error {
 	_, err := db.Exec(publishedSchema)
 	return err
 }
 
-func preparePublishedTable(db *sql.DB) (tables.Published, error) {
+func PreparePublishedTable(db *sql.DB) (tables.Published, error) {
 	s := &publishedStatements{
 		db: db,
 	}
@@ -96,8 +96,8 @@ func (s *publishedStatements) SelectAllPublishedRooms(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
 
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/sqlite3/redactions_table.go b/roomserver/storage/sqlite3/redactions_table.go
index aed190b1e..db6f57a1b 100644
--- a/roomserver/storage/sqlite3/redactions_table.go
+++ b/roomserver/storage/sqlite3/redactions_table.go
@@ -48,7 +48,7 @@ const selectRedactionInfoByEventBeingRedactedSQL = "" +
 	" WHERE redacts_event_id = $1"
 
 const markRedactionValidatedSQL = "" +
-	" UPDATE roomserver_redactions SET validated = $2 WHERE redaction_event_id = $1"
+	" UPDATE roomserver_redactions SET validated = $1 WHERE redaction_event_id = $2"
 
 type redactionStatements struct {
 	db                                          *sql.DB
@@ -58,12 +58,12 @@ type redactionStatements struct {
 	markRedactionValidatedStmt                  *sql.Stmt
 }
 
-func createRedactionsTable(db *sql.DB) error {
+func CreateRedactionsTable(db *sql.DB) error {
 	_, err := db.Exec(redactionsSchema)
 	return err
 }
 
-func prepareRedactionsTable(db *sql.DB) (tables.Redactions, error) {
+func PrepareRedactionsTable(db *sql.DB) (tables.Redactions, error) {
 	s := &redactionStatements{
 		db: db,
 	}
@@ -118,6 +118,6 @@ func (s *redactionStatements) MarkRedactionValidated(
 	ctx context.Context, txn *sql.Tx, redactionEventID string, validated bool,
 ) error {
 	stmt := sqlutil.TxStmt(txn, s.markRedactionValidatedStmt)
-	_, err := stmt.ExecContext(ctx, redactionEventID, validated)
+	_, err := stmt.ExecContext(ctx, validated, redactionEventID)
 	return err
 }
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 9522d3058..a4e32d528 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -98,22 +98,22 @@ func (d *Database) create(db *sql.DB) error {
 	if err := createStateSnapshotTable(db); err != nil {
 		return err
 	}
-	if err := createPrevEventsTable(db); err != nil {
+	if err := CreatePrevEventsTable(db); err != nil {
 		return err
 	}
 	if err := createRoomAliasesTable(db); err != nil {
 		return err
 	}
-	if err := createInvitesTable(db); err != nil {
+	if err := CreateInvitesTable(db); err != nil {
 		return err
 	}
-	if err := createMembershipTable(db); err != nil {
+	if err := CreateMembershipTable(db); err != nil {
 		return err
 	}
-	if err := createPublishedTable(db); err != nil {
+	if err := CreatePublishedTable(db); err != nil {
 		return err
 	}
-	if err := createRedactionsTable(db); err != nil {
+	if err := CreateRedactionsTable(db); err != nil {
 		return err
 	}
 
@@ -149,7 +149,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	prevEvents, err := preparePrevEventsTable(db)
+	prevEvents, err := PreparePrevEventsTable(db)
 	if err != nil {
 		return err
 	}
@@ -157,19 +157,19 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	invites, err := prepareInvitesTable(db)
+	invites, err := PrepareInvitesTable(db)
 	if err != nil {
 		return err
 	}
-	membership, err := prepareMembershipTable(db)
+	membership, err := PrepareMembershipTable(db)
 	if err != nil {
 		return err
 	}
-	published, err := preparePublishedTable(db)
+	published, err := PreparePublishedTable(db)
 	if err != nil {
 		return err
 	}
-	redactions, err := prepareRedactionsTable(db)
+	redactions, err := PrepareRedactionsTable(db)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/tables/invite_table_test.go b/roomserver/storage/tables/invite_table_test.go
new file mode 100644
index 000000000..8df3faa2d
--- /dev/null
+++ b/roomserver/storage/tables/invite_table_test.go
@@ -0,0 +1,92 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateInviteTable(t *testing.T, dbType test.DBType) (tables.Invites, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.Invites
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateInvitesTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareInvitesTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateInvitesTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareInvitesTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestInviteTable(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateInviteTable(t, dbType)
+		defer close()
+		eventID1 := util.RandomString(16)
+		roomNID := types.RoomNID(1)
+		targetUserNID, senderUserNID := types.EventStateKeyNID(1), types.EventStateKeyNID(2)
+		newInvite, err := tab.InsertInviteEvent(ctx, nil, eventID1, roomNID, targetUserNID, senderUserNID, []byte(""))
+		assert.NoError(t, err)
+		assert.True(t, newInvite)
+
+		// Try adding the same invite again
+		newInvite, err = tab.InsertInviteEvent(ctx, nil, eventID1, roomNID, targetUserNID, senderUserNID, []byte(""))
+		assert.NoError(t, err)
+		assert.False(t, newInvite)
+
+		// Add another invite for this room
+		eventID2 := util.RandomString(16)
+		newInvite, err = tab.InsertInviteEvent(ctx, nil, eventID2, roomNID, targetUserNID, senderUserNID, []byte(""))
+		assert.NoError(t, err)
+		assert.True(t, newInvite)
+
+		// Add another invite for a different user
+		eventID := util.RandomString(16)
+		newInvite, err = tab.InsertInviteEvent(ctx, nil, eventID, types.RoomNID(3), targetUserNID, senderUserNID, []byte(""))
+		assert.NoError(t, err)
+		assert.True(t, newInvite)
+
+		stateKeyNIDs, eventIDs, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{eventID1, eventID2}, eventIDs)
+		assert.Equal(t, []types.EventStateKeyNID{2, 2}, stateKeyNIDs)
+
+		// retire the invite
+		retiredEventIDs, err := tab.UpdateInviteRetired(ctx, nil, roomNID, targetUserNID)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{eventID1, eventID2}, retiredEventIDs)
+
+		// This should now be empty
+		stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
+		assert.NoError(t, err)
+		assert.Empty(t, eventIDs)
+		assert.Empty(t, stateKeyNIDs)
+
+		// Non-existent targetUserNID
+		stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID)
+		assert.NoError(t, err)
+		assert.Empty(t, stateKeyNIDs)
+		assert.Empty(t, eventIDs)
+	})
+}
diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go
new file mode 100644
index 000000000..14e8ce50a
--- /dev/null
+++ b/roomserver/storage/tables/membership_table_test.go
@@ -0,0 +1,130 @@
+package tables_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateMembershipTable(t *testing.T, dbType test.DBType) (tab tables.Membership, stateKeyTab tables.EventStateKeys, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateEventStateKeysTable(db)
+		assert.NoError(t, err)
+		err = postgres.CreateMembershipTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareMembershipTable(db)
+		assert.NoError(t, err)
+		stateKeyTab, err = postgres.PrepareEventStateKeysTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateEventStateKeysTable(db)
+		assert.NoError(t, err)
+		err = sqlite3.CreateMembershipTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareMembershipTable(db)
+		assert.NoError(t, err)
+		stateKeyTab, err = sqlite3.PrepareEventStateKeysTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, stateKeyTab, close
+}
+
+func TestMembershipTable(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, stateKeyTab, close := mustCreateMembershipTable(t, dbType)
+		defer close()
+		_ = close
+
+		userNIDs := make([]types.EventStateKeyNID, 0, 10)
+		for i := 0; i < 10; i++ {
+			stateKeyNID, err := stateKeyTab.InsertEventStateKeyNID(ctx, nil, fmt.Sprintf("@dummy%d:localhost", i))
+			assert.NoError(t, err)
+			userNIDs = append(userNIDs, stateKeyNID)
+			// This inserts a left user to the room
+			err = tab.InsertMembership(ctx, nil, 1, stateKeyNID, true)
+			assert.NoError(t, err)
+		}
+
+		// ... so this should be false
+		inRoom, err := tab.SelectLocalServerInRoom(ctx, nil, 1)
+		assert.NoError(t, err)
+		assert.False(t, inRoom)
+
+		changed, err := tab.UpdateMembership(ctx, nil, 1, userNIDs[0], userNIDs[0], tables.MembershipStateJoin, 1, false)
+		assert.NoError(t, err)
+		assert.True(t, changed)
+
+		// ... should now be true
+		inRoom, err = tab.SelectLocalServerInRoom(ctx, nil, 1)
+		assert.NoError(t, err)
+		assert.True(t, inRoom)
+
+		userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(userJoinedToRooms))
+
+		// Get all left/banned users
+		eventNIDs, err := tab.SelectMembershipsFromRoomAndMembership(ctx, nil, 1, tables.MembershipStateLeaveOrBan, true)
+		assert.NoError(t, err)
+		assert.Equal(t, 9, len(eventNIDs))
+
+		_, membershipState, forgotten, err := tab.SelectMembershipFromRoomAndTarget(ctx, nil, 1, userNIDs[5])
+		assert.NoError(t, err)
+		assert.False(t, forgotten)
+		assert.Equal(t, tables.MembershipStateLeaveOrBan, membershipState)
+
+		// Get all members, regardless of state
+		members, err := tab.SelectMembershipsFromRoom(ctx, nil, 1, true)
+		assert.NoError(t, err)
+		assert.Equal(t, 10, len(members))
+
+		// Get correct user
+		roomNIDs, err := tab.SelectRoomsWithMembership(ctx, nil, userNIDs[1], tables.MembershipStateLeaveOrBan)
+		assert.NoError(t, err)
+		assert.Equal(t, []types.RoomNID{1}, roomNIDs)
+
+		// User is not joined to room
+		roomNIDs, err = tab.SelectRoomsWithMembership(ctx, nil, userNIDs[5], tables.MembershipStateJoin)
+		assert.NoError(t, err)
+		assert.Equal(t, 0, len(roomNIDs))
+
+		// Forget room
+		err = tab.UpdateForgetMembership(ctx, nil, 1, userNIDs[0], true)
+		assert.NoError(t, err)
+
+		// should now return true
+		_, _, forgotten, err = tab.SelectMembershipFromRoomAndTarget(ctx, nil, 1, userNIDs[0])
+		assert.NoError(t, err)
+		assert.True(t, forgotten)
+
+		serverInRoom, err := tab.SelectServerInRoom(ctx, nil, 1, "localhost")
+		assert.NoError(t, err)
+		assert.True(t, serverInRoom)
+
+		serverInRoom, err = tab.SelectServerInRoom(ctx, nil, 1, "notJoined")
+		assert.NoError(t, err)
+		assert.False(t, serverInRoom)
+
+		// get all users we know about; should be only one user, since no other user joined the room
+		knownUsers, err := tab.SelectKnownUsers(ctx, nil, userNIDs[0], "localhost", 2)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(knownUsers))
+	})
+}
diff --git a/roomserver/storage/tables/previous_events_table_test.go b/roomserver/storage/tables/previous_events_table_test.go
new file mode 100644
index 000000000..96d7bfed0
--- /dev/null
+++ b/roomserver/storage/tables/previous_events_table_test.go
@@ -0,0 +1,61 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreatePreviousEventsTable(t *testing.T, dbType test.DBType) (tab tables.PreviousEvents, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreatePrevEventsTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PreparePrevEventsTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreatePrevEventsTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PreparePrevEventsTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestPreviousEventsTable(t *testing.T) {
+	ctx := context.Background()
+	alice := test.NewUser()
+	room := test.NewRoom(t, alice)
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreatePreviousEventsTable(t, dbType)
+		defer close()
+
+		for _, x := range room.Events() {
+			for _, prevEvent := range x.PrevEvents() {
+				err := tab.InsertPreviousEvent(ctx, nil, prevEvent.EventID, prevEvent.EventSHA256, 1)
+				assert.NoError(t, err)
+
+				err = tab.SelectPreviousEventExists(ctx, nil, prevEvent.EventID, prevEvent.EventSHA256)
+				assert.NoError(t, err)
+			}
+		}
+
+		// RandomString with a correct EventSHA256 should fail and return sql.ErrNoRows
+		err := tab.SelectPreviousEventExists(ctx, nil, util.RandomString(16), room.Events()[0].EventReference().EventSHA256)
+		assert.Error(t, err)
+	})
+}
diff --git a/roomserver/storage/tables/published_table_test.go b/roomserver/storage/tables/published_table_test.go
new file mode 100644
index 000000000..87662ed4c
--- /dev/null
+++ b/roomserver/storage/tables/published_table_test.go
@@ -0,0 +1,79 @@
+package tables_test
+
+import (
+	"context"
+	"sort"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreatePublishedTable(t *testing.T, dbType test.DBType) (tab tables.Published, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreatePublishedTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PreparePublishedTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreatePublishedTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PreparePublishedTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestPublishedTable(t *testing.T) {
+	ctx := context.Background()
+	alice := test.NewUser()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreatePublishedTable(t, dbType)
+		defer close()
+
+		// Publish some rooms
+		publishedRooms := []string{}
+		for i := 0; i < 10; i++ {
+			room := test.NewRoom(t, alice)
+			published := i%2 == 0
+			err := tab.UpsertRoomPublished(ctx, nil, room.ID, published)
+			assert.NoError(t, err)
+			if published {
+				publishedRooms = append(publishedRooms, room.ID)
+			}
+			publishedRes, err := tab.SelectPublishedFromRoomID(ctx, nil, room.ID)
+			assert.NoError(t, err)
+			assert.Equal(t, published, publishedRes)
+		}
+		sort.Strings(publishedRooms)
+
+		// check that we get the expected published rooms
+		roomIDs, err := tab.SelectAllPublishedRooms(ctx, nil, true)
+		assert.NoError(t, err)
+		assert.Equal(t, publishedRooms, roomIDs)
+
+		// test an actual upsert
+		room := test.NewRoom(t, alice)
+		err = tab.UpsertRoomPublished(ctx, nil, room.ID, true)
+		assert.NoError(t, err)
+		err = tab.UpsertRoomPublished(ctx, nil, room.ID, false)
+		assert.NoError(t, err)
+		// should now be false, due to the upsert
+		publishedRes, err := tab.SelectPublishedFromRoomID(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.False(t, publishedRes)
+	})
+}
diff --git a/roomserver/storage/tables/redactions_table_test.go b/roomserver/storage/tables/redactions_table_test.go
new file mode 100644
index 000000000..ea48dc22f
--- /dev/null
+++ b/roomserver/storage/tables/redactions_table_test.go
@@ -0,0 +1,89 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateRedactionsTable(t *testing.T, dbType test.DBType) (tab tables.Redactions, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateRedactionsTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareRedactionsTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateRedactionsTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareRedactionsTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestRedactionsTable(t *testing.T) {
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateRedactionsTable(t, dbType)
+		defer close()
+
+		// insert and verify some redactions
+		for i := 0; i < 10; i++ {
+			redactionEventID, redactsEventID := util.RandomString(16), util.RandomString(16)
+			wantRedactionInfo := tables.RedactionInfo{
+				Validated:        false,
+				RedactsEventID:   redactsEventID,
+				RedactionEventID: redactionEventID,
+			}
+			err := tab.InsertRedaction(ctx, nil, wantRedactionInfo)
+			assert.NoError(t, err)
+
+			// verify the redactions are inserted as expected
+			redactionInfo, err := tab.SelectRedactionInfoByRedactionEventID(ctx, nil, redactionEventID)
+			assert.NoError(t, err)
+			assert.Equal(t, &wantRedactionInfo, redactionInfo)
+
+			redactionInfo, err = tab.SelectRedactionInfoByEventBeingRedacted(ctx, nil, redactsEventID)
+			assert.NoError(t, err)
+			assert.Equal(t, &wantRedactionInfo, redactionInfo)
+
+			// redact event
+			err = tab.MarkRedactionValidated(ctx, nil, redactionEventID, true)
+			assert.NoError(t, err)
+
+			wantRedactionInfo.Validated = true
+			redactionInfo, err = tab.SelectRedactionInfoByRedactionEventID(ctx, nil, redactionEventID)
+			assert.NoError(t, err)
+			assert.Equal(t, &wantRedactionInfo, redactionInfo)
+		}
+
+		// Should not fail, it just updates 0 rows
+		err := tab.MarkRedactionValidated(ctx, nil, "iDontExist", true)
+		assert.NoError(t, err)
+
+		// Should also not fail, but return a nil redactionInfo
+		redactionInfo, err := tab.SelectRedactionInfoByRedactionEventID(ctx, nil, "iDontExist")
+		assert.NoError(t, err)
+		assert.Nil(t, redactionInfo)
+
+		redactionInfo, err = tab.SelectRedactionInfoByEventBeingRedacted(ctx, nil, "iDontExist")
+		assert.NoError(t, err)
+		assert.Nil(t, redactionInfo)
+	})
+}

From c15bfefd0dbbd9619c2606b59b784f2a7926ca20 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Wed, 11 May 2022 11:29:23 +0100
Subject: [PATCH 101/391] Add RoomExists flag to QueryMembershipForUser (#2450)

Fixes https://github.com/matrix-org/complement/pull/369
---
 clientapi/routing/membership.go    | 12 ++++++++++++
 clientapi/routing/state.go         |  6 ++++++
 roomserver/api/query.go            |  1 +
 roomserver/internal/query/query.go |  4 +++-
 syncapi/routing/context.go         |  6 ++++++
 syncapi/routing/messages.go        | 14 ++++++++++----
 6 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go
index cfdf6f2de..77f627eb2 100644
--- a/clientapi/routing/membership.go
+++ b/clientapi/routing/membership.go
@@ -188,6 +188,12 @@ func SendUnban(
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
+	if !queryRes.RoomExists {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("room does not exist"),
+		}
+	}
 	// unban is only valid if the user is currently banned
 	if queryRes.Membership != "ban" {
 		return util.JSONResponse{
@@ -471,6 +477,12 @@ func SendForget(
 		logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user")
 		return jsonerror.InternalServerError()
 	}
+	if !membershipRes.RoomExists {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("room does not exist"),
+		}
+	}
 	if membershipRes.IsInRoom {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go
index c6e9e91d0..12984c39a 100644
--- a/clientapi/routing/state.go
+++ b/clientapi/routing/state.go
@@ -56,6 +56,12 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}
+	if !stateRes.RoomExists {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("room does not exist"),
+		}
+	}
 
 	// Look at the room state and see if we have a history visibility event
 	// that marks the room as world-readable. If we don't then we assume that
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index ef2e6bb57..afafb87c3 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -122,6 +122,7 @@ type QueryMembershipForUserResponse struct {
 	Membership string `json:"membership"`
 	// True if the user asked to forget this room.
 	IsRoomForgotten bool `json:"is_room_forgotten"`
+	RoomExists      bool `json:"room_exists"`
 }
 
 // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 5b33ec3c3..d25bdc378 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -169,8 +169,10 @@ func (r *Queryer) QueryMembershipForUser(
 		return err
 	}
 	if info == nil {
-		return fmt.Errorf("QueryMembershipForUser: unknown room %s", request.RoomID)
+		response.RoomExists = false
+		return nil
 	}
+	response.RoomExists = true
 
 	membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
 	if err != nil {
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 87cc2aae0..96438e184 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -73,6 +73,12 @@ func Context(
 		logrus.WithError(err).Error("unable to query membership")
 		return jsonerror.InternalServerError()
 	}
+	if !membershipRes.RoomExists {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("room does not exist"),
+		}
+	}
 
 	stateFilter := gomatrixserverlib.StateFilter{
 		Limit:                   100,
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index b0c990ec0..e55c661d6 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -68,10 +68,16 @@ func OnIncomingMessagesRequest(
 	var err error
 
 	// check if the user has already forgotten about this room
-	isForgotten, err := checkIsRoomForgotten(req.Context(), roomID, device.UserID, rsAPI)
+	isForgotten, roomExists, err := checkIsRoomForgotten(req.Context(), roomID, device.UserID, rsAPI)
 	if err != nil {
 		return jsonerror.InternalServerError()
 	}
+	if !roomExists {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("room does not exist"),
+		}
+	}
 
 	if isForgotten {
 		return util.JSONResponse{
@@ -244,17 +250,17 @@ func OnIncomingMessagesRequest(
 	}
 }
 
-func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (bool, error) {
+func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (forgotten bool, exists bool, err error) {
 	req := api.QueryMembershipForUserRequest{
 		RoomID: roomID,
 		UserID: userID,
 	}
 	resp := api.QueryMembershipForUserResponse{}
 	if err := rsAPI.QueryMembershipForUser(ctx, &req, &resp); err != nil {
-		return false, err
+		return false, false, err
 	}
 
-	return resp.IsRoomForgotten, nil
+	return resp.IsRoomForgotten, resp.RoomExists, nil
 }
 
 // retrieveEvents retrieves events from the local database for a request on

From 9599b3686e02356e48a537a820b075523252ac64 Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Wed, 11 May 2022 13:44:32 +0100
Subject: [PATCH 102/391] More syncapi tests (#2451)

* WIP tests for flakey create event

* Uncomment all database test
---
 syncapi/syncapi_test.go | 148 +++++++++++++++++++++++++++++++++++-----
 test/event.go           |   3 +-
 2 files changed, 132 insertions(+), 19 deletions(-)

diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 12b5178d8..7809cdaba 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -11,10 +11,12 @@ import (
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/test"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 )
 
@@ -39,6 +41,14 @@ func (s *syncRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *
 	return nil // TODO: return state
 }
 
+func (s *syncRoomserverAPI) QuerySharedUsers(ctx context.Context, req *rsapi.QuerySharedUsersRequest, res *rsapi.QuerySharedUsersResponse) error {
+	res.UserIDsToCount = make(map[string]int)
+	return nil
+}
+func (s *syncRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *rsapi.QueryBulkStateContentRequest, res *rsapi.QueryBulkStateContentResponse) error {
+	return nil
+}
+
 type syncUserAPI struct {
 	userapi.SyncUserAPI
 	accounts []userapi.Device
@@ -60,16 +70,22 @@ func (s *syncUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.Pe
 }
 
 type syncKeyAPI struct {
-	keyapi.KeyInternalAPI
+	keyapi.SyncKeyAPI
 }
 
-func TestSyncAPI(t *testing.T) {
+func (s *syncKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) {
+}
+func (s *syncKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) {
+
+}
+
+func TestSyncAPIAccessTokens(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		testSync(t, dbType)
+		testSyncAccessTokens(t, dbType)
 	})
 }
 
-func testSync(t *testing.T, dbType test.DBType) {
+func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
 	user := test.NewUser()
 	room := test.NewRoom(t, user)
 	alice := userapi.Device{
@@ -85,20 +101,7 @@ func testSync(t *testing.T, dbType test.DBType) {
 
 	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
-	var msgs []*nats.Msg
-	for _, ev := range room.Events() {
-		var addsStateIDs []string
-		if ev.StateKey() != nil {
-			addsStateIDs = append(addsStateIDs, ev.EventID())
-		}
-		msgs = append(msgs, test.NewOutputEventMsg(t, base, room.ID, api.OutputEvent{
-			Type: rsapi.OutputTypeNewRoomEvent,
-			NewRoomEvent: &rsapi.OutputNewRoomEvent{
-				Event:             ev,
-				AddsStateEventIDs: addsStateIDs,
-			},
-		}))
-	}
+	msgs := toNATSMsgs(t, base, room.Events())
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
 	test.MustPublishMsgs(t, jsctx, msgs...)
 
@@ -160,3 +163,112 @@ func testSync(t *testing.T, dbType test.DBType) {
 		}
 	}
 }
+
+// Tests what happens when we create a room and then /sync before all events from /createRoom have
+// been sent to the syncapi
+func TestSyncAPICreateRoomSyncEarly(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testSyncAPICreateRoomSyncEarly(t, dbType)
+	})
+}
+
+func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
+	user := test.NewUser()
+	room := test.NewRoom(t, user)
+	alice := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      user.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "Alice",
+		AccountType: userapi.AccountTypeUser,
+	}
+
+	base, close := test.CreateBaseDendrite(t, dbType)
+	defer close()
+
+	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+	// order is:
+	// m.room.create
+	// m.room.member
+	// m.room.power_levels
+	// m.room.join_rules
+	// m.room.history_visibility
+	msgs := toNATSMsgs(t, base, room.Events())
+	sinceTokens := make([]string, len(msgs))
+	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
+	for i, msg := range msgs {
+		test.MustPublishMsgs(t, jsctx, msg)
+		time.Sleep(50 * time.Millisecond)
+		w := httptest.NewRecorder()
+		base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+			"access_token": alice.AccessToken,
+			"timeout":      "0",
+		})))
+		if w.Code != 200 {
+			t.Errorf("got HTTP %d want 200", w.Code)
+			continue
+		}
+		var res types.Response
+		if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+			t.Errorf("failed to decode response body: %s", err)
+		}
+		sinceTokens[i] = res.NextBatch.String()
+		if i == 0 { // create event does not produce a room section
+			if len(res.Rooms.Join) != 0 {
+				t.Fatalf("i=%v got %d joined rooms, want 0", i, len(res.Rooms.Join))
+			}
+		} else { // we should have that room somewhere
+			if len(res.Rooms.Join) != 1 {
+				t.Fatalf("i=%v got %d joined rooms, want 1", i, len(res.Rooms.Join))
+			}
+		}
+	}
+
+	// sync with no token "" and with the penultimate token and this should neatly return room events in the timeline block
+	sinceTokens = append([]string{""}, sinceTokens[:len(sinceTokens)-1]...)
+
+	t.Logf("waited for events to be consumed; syncing with %v", sinceTokens)
+	for i, since := range sinceTokens {
+		w := httptest.NewRecorder()
+		base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+			"access_token": alice.AccessToken,
+			"timeout":      "0",
+			"since":        since,
+		})))
+		if w.Code != 200 {
+			t.Errorf("since=%s got HTTP %d want 200", since, w.Code)
+		}
+		var res types.Response
+		if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+			t.Errorf("failed to decode response body: %s", err)
+		}
+		if len(res.Rooms.Join) != 1 {
+			t.Fatalf("since=%s got %d joined rooms, want 1", since, len(res.Rooms.Join))
+		}
+		t.Logf("since=%s res state:%+v res timeline:%+v", since, res.Rooms.Join[room.ID].State.Events, res.Rooms.Join[room.ID].Timeline.Events)
+		gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events))
+		for j, ev := range res.Rooms.Join[room.ID].Timeline.Events {
+			gotEventIDs[j] = ev.EventID
+		}
+		test.AssertEventIDsEqual(t, gotEventIDs, room.Events()[i:])
+	}
+}
+
+func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
+	result := make([]*nats.Msg, len(input))
+	for i, ev := range input {
+		var addsStateIDs []string
+		if ev.StateKey() != nil {
+			addsStateIDs = append(addsStateIDs, ev.EventID())
+		}
+		result[i] = test.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{
+			Type: rsapi.OutputTypeNewRoomEvent,
+			NewRoomEvent: &rsapi.OutputNewRoomEvent{
+				Event:             ev,
+				AddsStateEventIDs: addsStateIDs,
+			},
+		})
+	}
+	return result
+}
diff --git a/test/event.go b/test/event.go
index b2e2805ba..40cb8f0e1 100644
--- a/test/event.go
+++ b/test/event.go
@@ -64,7 +64,8 @@ func Reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.Header
 func AssertEventIDsEqual(t *testing.T, gotEventIDs []string, wants []*gomatrixserverlib.HeaderedEvent) {
 	t.Helper()
 	if len(gotEventIDs) != len(wants) {
-		t.Fatalf("length mismatch: got %d events, want %d", len(gotEventIDs), len(wants))
+		t.Errorf("length mismatch: got %d events, want %d", len(gotEventIDs), len(wants))
+		return
 	}
 	for i := range wants {
 		w := wants[i].EventID()

From 19a9166eb0de86b643c17a3b96c770635468b4f5 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 11 May 2022 15:39:36 +0100
Subject: [PATCH 103/391] New documentation:
 https://matrix-org.github.io/dendrite/

---
 .gitignore                               |   7 +
 README.md                                |   3 +-
 docs/CODE_STYLE.md                       |  60 -----
 docs/CONTRIBUTING.md                     | 114 ++++++---
 docs/DESIGN.md                           | 140 -----------
 docs/FAQ.md                              |  73 +++---
 docs/Gemfile                             |   5 +
 docs/Gemfile.lock                        | 283 +++++++++++++++++++++
 docs/INSTALL.md                          | 298 ++---------------------
 docs/PROFILING.md                        |  10 +-
 docs/WIRING-Current.md                   |  71 ------
 docs/WIRING.md                           | 229 -----------------
 docs/_config.yml                         |  19 ++
 docs/_sass/custom/custom.scss            |   3 +
 docs/administration.md                   |  10 +
 docs/administration/1_createusers.md     |  53 ++++
 docs/administration/2_registration.md    |  53 ++++
 docs/administration/3_presence.md        |  39 +++
 docs/administration/4_adminapi.md        |  25 ++
 docs/development.md                      |  10 +
 docs/index.md                            |  24 ++
 docs/installation.md                     |  10 +
 docs/installation/1_planning.md          | 110 +++++++++
 docs/installation/2_domainname.md        |  93 +++++++
 docs/installation/3_database.md          | 106 ++++++++
 docs/installation/4_signingkey.md        |  79 ++++++
 docs/installation/5_install_monolith.md  |  21 ++
 docs/installation/6_install_polylith.md  |  33 +++
 docs/installation/7_configuration.md     | 145 +++++++++++
 docs/installation/8_starting_monolith.md |  41 ++++
 docs/installation/9_starting_polylith.md |  73 ++++++
 docs/{ => other}/p2p.md                  |  49 ++--
 docs/other/peeking.md                    |  33 +++
 docs/peeking.md                          |  26 --
 docs/serverkeyformat.md                  |  29 ---
 docs/sytest.md                           |  10 +-
 docs/tracing/jaeger.png                  | Bin 264127 -> 0 bytes
 docs/tracing/opentracing.md              |  18 +-
 docs/tracing/setup.md                    |  22 +-
 39 files changed, 1483 insertions(+), 944 deletions(-)
 delete mode 100644 docs/CODE_STYLE.md
 delete mode 100644 docs/DESIGN.md
 create mode 100644 docs/Gemfile
 create mode 100644 docs/Gemfile.lock
 delete mode 100644 docs/WIRING-Current.md
 delete mode 100644 docs/WIRING.md
 create mode 100644 docs/_config.yml
 create mode 100644 docs/_sass/custom/custom.scss
 create mode 100644 docs/administration.md
 create mode 100644 docs/administration/1_createusers.md
 create mode 100644 docs/administration/2_registration.md
 create mode 100644 docs/administration/3_presence.md
 create mode 100644 docs/administration/4_adminapi.md
 create mode 100644 docs/development.md
 create mode 100644 docs/index.md
 create mode 100644 docs/installation.md
 create mode 100644 docs/installation/1_planning.md
 create mode 100644 docs/installation/2_domainname.md
 create mode 100644 docs/installation/3_database.md
 create mode 100644 docs/installation/4_signingkey.md
 create mode 100644 docs/installation/5_install_monolith.md
 create mode 100644 docs/installation/6_install_polylith.md
 create mode 100644 docs/installation/7_configuration.md
 create mode 100644 docs/installation/8_starting_monolith.md
 create mode 100644 docs/installation/9_starting_polylith.md
 rename docs/{ => other}/p2p.md (71%)
 create mode 100644 docs/other/peeking.md
 delete mode 100644 docs/peeking.md
 delete mode 100644 docs/serverkeyformat.md
 delete mode 100644 docs/tracing/jaeger.png

diff --git a/.gitignore b/.gitignore
index 2a8c2cf55..e4f0112c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,6 +41,10 @@ _testmain.go
 *.test
 *.prof
 *.wasm
+*.aar
+*.jar
+*.framework
+*.xcframework
 
 # Generated keys
 *.pem
@@ -65,4 +69,7 @@ test/wasm/node_modules
 # Ignore complement folder when running locally
 complement/
 
+# Stuff from GitHub Pages
+docs/_site
+
 media_store/
diff --git a/README.md b/README.md
index 2a8c36508..9c38dee90 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
 # Dendrite
+
 [![Build status](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml/badge.svg?event=push)](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org)
 
 Dendrite is a second-generation Matrix homeserver written in Go.
@@ -52,7 +53,7 @@ The [Federation Tester](https://federationtester.matrix.org) can be used to veri
 
 ## Get started
 
-If you wish to build a fully-federating Dendrite instance, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker).
+If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](docs/installation). For running in Docker, see [build/docker](build/docker).
 
 The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases:
 
diff --git a/docs/CODE_STYLE.md b/docs/CODE_STYLE.md
deleted file mode 100644
index 8096ae27c..000000000
--- a/docs/CODE_STYLE.md
+++ /dev/null
@@ -1,60 +0,0 @@
-# Code Style
-
-In addition to standard Go code style (`gofmt`, `goimports`), we use `golangci-lint`
-to run a number of linters, the exact list can be found under linters in [.golangci.yml](.golangci.yml).
-[Installation](https://github.com/golangci/golangci-lint#install-golangci-lint) and [Editor
-Integration](https://golangci-lint.run/usage/integrations/#editor-integration) for
-it can be found in the readme of golangci-lint.
-
-For rare cases where a linter is giving a spurious warning, it can be disabled
-for that line or statement using a [comment
-directive](https://golangci-lint.run/usage/false-positives/#nolint), e.g.  `var
-bad_name int //nolint:golint,unused`. This should be used sparingly and only
-when its clear that the lint warning is spurious.
-
-The linters can be run using [build/scripts/find-lint.sh](/build/scripts/find-lint.sh)
-(see file for docs) or as part of a build/test/lint cycle using
-[build/scripts/build-test-lint.sh](/build/scripts/build-test-lint.sh).
-
-
-## Labels
-
-In addition to `TODO` and `FIXME` we also use `NOTSPEC` to identify deviations
-from the Matrix specification.
-
-## Logging
-
-We generally prefer to log with static log messages and include any dynamic
-information in fields.
-
-```golang
-logger := util.GetLogger(ctx)
-
-// Not recommended
-logger.Infof("Finished processing keys for %s, number of keys %d", name, numKeys)
-
-// Recommended
-logger.WithFields(logrus.Fields{
-    "numberOfKeys": numKeys,
-    "entityName":   name,
-}).Info("Finished processing keys")
-```
-
-This is useful when logging to systems that natively understand log fields, as
-it allows people to search and process the fields without having to parse the
-log message.
-
-
-## Visual Studio Code
-
-If you use VSCode then the following is an example of a workspace setting that
-sets up linting correctly:
-
-```json
-{
-    "go.lintTool":"golangci-lint",
-    "go.lintFlags": [
-      "--fast"
-    ]
-}
-```
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 116adfae6..5a89e6841 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -1,55 +1,103 @@
+---
+title: Contributing
+parent: Development
+permalink: /development/contributing
+---
+
 # Contributing to Dendrite
 
 Everyone is welcome to contribute to Dendrite! We aim to make it as easy as
 possible to get started.
 
-Please ensure that you sign off your contributions! See [Sign Off](#sign-off)
-section below.
+## Sign off
+
+We ask that everyone who contributes to the project signs off their contributions
+in accordance with the [DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off).
+In effect, this means adding a statement to your pull requests or commit messages
+along the lines of:
+
+```
+Signed-off-by: Full Name <email address>
+```
+
+Unfortunately we can't accept contributions without it.
 
 ## Getting up and running
 
-See [INSTALL.md](INSTALL.md) for instructions on setting up a running dev
-instance of dendrite, and [CODE_STYLE.md](CODE_STYLE.md) for the code style
-guide.
+See the [Installation](INSTALL.md) section for information on how to build an
+instance of Dendrite. You will likely need this in order to test your changes.
 
-We use [golangci-lint](https://github.com/golangci/golangci-lint) to lint
-Dendrite which can be executed via:
+## Code style
 
+On the whole, the format as prescribed by `gofmt`, `goimports` etc. is exactly
+what we use and expect. Please make sure that you run one of these formatters before
+submitting your contribution.
+
+## Comments
+
+Please make sure that the comments adequately explain *why* your code does what it
+does. If there are statements that are not obvious, please comment what they do.
+
+We also have some special tags which we use for searchability. These are:
+
+* `// TODO:` for places where a future review, rewrite or refactor is likely required;
+* `// FIXME:` for places where we know there is an outstanding bug that needs a fix;
+* `// NOTSPEC:` for places where the behaviour specifically does not match what the
+  [Matrix Specification](https://spec.matrix.org/) prescribes, along with a description
+  of *why* that is the case.
+
+## Linting
+
+We use [golangci-lint](https://github.com/golangci/golangci-lint) to lint Dendrite
+which can be executed via:
+
+```bash
+golangci-lint run
 ```
-$ golangci-lint run
-```
+
+If you are receiving linter warnings that you are certain are spurious and want to
+silence them, you can annotate the relevant lines or methods with a `// nolint:`
+comment. Please avoid doing this if you can.
+
+## Unit tests
 
 We also have unit tests which we run via:
 
-```
-$ go test ./...
+```bash
+go test ./...
 ```
 
-## Continuous Integration
+In general, we like submissions that come with tests. Anything that proves that the
+code is functioning as intended is great, and to ensure that we will find out quickly
+in the future if any regressions happen.
 
-When a Pull Request is submitted, continuous integration jobs are run
-automatically to ensure the code builds and is relatively well-written. The jobs
-are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/), and the
-Buildkite pipeline configuration can be found in Matrix.org's [pipelines
-repository](https://github.com/matrix-org/pipelines).
+We use the standard [Go testing package](https://gobyexample.com/testing) for this,
+alongside some helper functions in our own [`test` package](https://pkg.go.dev/github.com/matrix-org/dendrite/test).
 
-If a job fails, click the "details" button and you should be taken to the job's
-logs.
+## Continuous integration
 
-![Click the details button on the failing build
-step](https://raw.githubusercontent.com/matrix-org/dendrite/main/docs/images/details-button-location.jpg)
+When a Pull Request is submitted, continuous integration jobs are run automatically
+by GitHub actions to ensure that the code builds and works in a number of configurations,
+such as different Go versions, using full HTTP APIs and both database engines.
+CI will automatically run the unit tests (as above) as well as both of our integration
+test suites ([Complement](https://github.com/matrix-org/complement) and
+[SyTest](https://github.com/matrix-org/sytest)).
 
-Scroll down to the failing step and you should see some log output. Scan the
-logs until you find what it's complaining about, fix it, submit a new commit,
-then rinse and repeat until CI passes.
+You can see the progress of any CI jobs at the bottom of the Pull Request page, or by
+looking at the [Actions](https://github.com/matrix-org/dendrite/actions) tab of the Dendrite
+repository.
 
-### Running CI Tests Locally
+We generally won't accept a submission unless all of the CI jobs are passing. We
+do understand though that sometimes the tests get things wrong — if that's the case,
+please also raise a pull request to fix the relevant tests!
+
+### Running CI tests locally
 
 To save waiting for CI to finish after every commit, it is ideal to run the
 checks locally before pushing, fixing errors first. This also saves other people
 time as only so many PRs can be tested at a given time.
 
-To execute what Buildkite tests, first run `./build/scripts/build-test-lint.sh`; this
+To execute what CI tests, first run `./build/scripts/build-test-lint.sh`; this
 script will build the code, lint it, and run `go test ./...` with race condition
 checking enabled. If something needs to be changed, fix it and then run the
 script again until it no longer complains. Be warned that the linting can take a
@@ -64,8 +112,7 @@ passing tests.
 If these two steps report no problems, the code should be able to pass the CI
 tests.
 
-
-## Picking Things To Do
+## Picking things to do
 
 If you're new then feel free to pick up an issue labelled [good first
 issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue).
@@ -81,17 +128,10 @@ We ask people who are familiar with Dendrite to leave the [good first
 issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue)
 issues so that there is always a way for new people to come and get involved.
 
-## Getting Help
+## Getting help
 
 For questions related to developing on Dendrite we have a dedicated room on
 Matrix [#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)
 where we're happy to help.
 
-For more general questions please use
-[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org).
-
-## Sign off
-
-We ask that everyone who contributes to the project signs off their
-contributions, in accordance with the
-[DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off).
+For more general questions please use [#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org).
diff --git a/docs/DESIGN.md b/docs/DESIGN.md
deleted file mode 100644
index 80e251c5e..000000000
--- a/docs/DESIGN.md
+++ /dev/null
@@ -1,140 +0,0 @@
-# Design
-
-## Log Based Architecture
-
-### Decomposition and Decoupling
-
-A matrix homeserver can be built around append-only event logs built from the
-messages, receipts, presence, typing notifications, device messages and other
-events sent by users on the homeservers or by other homeservers.
-
-The server would then decompose into two categories: writers that add new
-entries to the logs and readers that read those entries.
-
-The event logs then serve to decouple the two components, the writers and
-readers need only agree on the format of the entries in the event log.
-This format could be largely derived from the wire format of the events used
-in the client and federation protocols:
-
-
-     C-S API   +---------+    Event Log    +---------+   C-S API
-    ---------> |         |+  (e.g. kafka)  |         |+ --------->
-               | Writers || =============> | Readers ||
-    ---------> |         ||                |         || --------->
-     S-S API   +---------+|                +---------+|   S-S API
-                +---------+                 +---------+
-
-However the way matrix handles state events in a room creates a few
-complications for this model.
-
- 1) Writers require the room state at an event to check if it is allowed.
- 2) Readers require the room state at an event to determine the users and
-    servers that are allowed to see the event.
- 3) A client can query the current state of the room from a reader.
-
-The writers and readers cannot extract the necessary information directly from
-the event logs because it would take too long to extract the information as the
-state is built up by collecting individual state events from the event history.
-
-The writers and readers therefore need access to something that stores copies
-of the event state in a form that can be efficiently queried. One possibility
-would be for the readers and writers to maintain copies of the current state
-in local databases. A second possibility would be to add a dedicated component
-that maintained the state of the room and exposed an API that the readers and
-writers could query to get the state. The second has the advantage that the
-state is calculated and stored in a single location.
-
-
-     C-S API   +---------+    Log   +--------+   Log   +---------+   C-S API
-    ---------> |         |+ ======> |        | ======> |         |+ --------->
-               | Writers ||         |  Room  |         | Readers ||
-    ---------> |         || <------ | Server | ------> |         || --------->
-     S-S API   +---------+|  Query  |        |  Query  +---------+|  S-S API
-                +---------+         +--------+          +---------+
-
-
-The room server can annotate the events it logs to the readers with room state
-so that the readers can avoid querying the room server unnecessarily.
-
-[This architecture can be extended to cover most of the APIs.](WIRING.md)
-
-## How things are supposed to work.
-
-### Local client sends an event in an existing room.
-
-  0) The client sends a PUT `/_matrix/client/r0/rooms/{roomId}/send` request
-    and an HTTP loadbalancer routes the request to a ClientAPI.
-
-  1) The ClientAPI:
-
-    * Authenticates the local user using the `access_token` sent in the HTTP
-      request.
-    * Checks if it has already processed or is processing a request with the
-      same `txnID`.
-    * Calculates which state events are needed to auth the request.
-    * Queries the necessary state events and the latest events in the room
-      from the RoomServer.
-    * Confirms that the room exists and checks whether the event is allowed by
-      the auth checks.
-    * Builds and signs the events.
-    * Writes the event to a "InputRoomEvent" kafka topic.
-    * Send a `200 OK` response to the client.
-
-  2) The RoomServer reads the event from "InputRoomEvent" kafka topic:
-
-    * Checks if it has already has a copy of the event.
-    * Checks if the event is allowed by the auth checks using the auth events
-      at the event.
-    * Calculates the room state at the event.
-    * Works out what the latest events in the room after processing this event
-      are.
-    * Calculate how the changes in the latest events affect the current state
-      of the room.
-    * TODO: Workout what events determine the visibility of this event to other
-      users
-    * Writes the event along with the changes in current state to an
-      "OutputRoomEvent" kafka topic. It writes all the events for a room to
-      the same kafka partition.
-
-  3a) The ClientSync reads the event from the "OutputRoomEvent" kafka topic:
-
-    * Updates its copy of the current state for the room.
-    * Works out which users need to be notified about the event.
-    * Wakes up any pending `/_matrix/client/r0/sync` requests for those users.
-    * Adds the event to the recent timeline events for the room.
-
-  3b) The FederationSender reads the event from the "OutputRoomEvent" kafka topic:
-
-    * Updates its copy of the current state for the room.
-    * Works out which remote servers need to be notified about the event.
-    * Sends a `/_matrix/federation/v1/send` request to those servers.
-    * Or if there is a request in progress then add the event to a queue to be
-      sent when the previous request finishes.
-
-### Remote server sends an event in an existing room.
-
-  0) The remote server sends a `PUT /_matrix/federation/v1/send` request and an
-    HTTP loadbalancer routes the request to a FederationReceiver.
-
-  1) The FederationReceiver:
-
-    * Authenticates the remote server using the "X-Matrix" authorisation header.
-    * Checks if it has already processed or is processing a request with the
-      same `txnID`.
-    * Checks the signatures for the events.
-      Fetches the ed25519 keys for the event senders if necessary.
-    * Queries the RoomServer for a copy of the state of the room at each event.
-    * If the RoomServer doesn't know the state of the room at an event then
-      query the state of the room at the event from the remote server using
-      `GET /_matrix/federation/v1/state_ids` falling back to
-      `GET /_matrix/federation/v1/state` if necessary.
-    * Once the state at each event is known check whether the events are
-      allowed by the auth checks against the state at each event.
-    * For each event that is allowed write the event to the "InputRoomEvent"
-      kafka topic.
-    * Send a 200 OK response to the remote server listing which events were
-      successfully processed and which events failed
-
-  2) The RoomServer processes the event the same as it would a local event.
-
-  3a) The ClientSync processes the event the same as it would a local event.
diff --git a/docs/FAQ.md b/docs/FAQ.md
index 47eaecf0f..571726d61 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -1,26 +1,34 @@
-# Frequently Asked Questions
+---
+title: FAQ
+nav_order: 1
+permalink: /faq
+---
 
-### Is Dendrite stable?
+# FAQ
+
+## Is Dendrite stable?
 
 Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead.
 
-### Is Dendrite feature-complete?
+## Is Dendrite feature-complete?
 
 No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information.
 
-### Is there a migration path from Synapse to Dendrite?
+## Is there a migration path from Synapse to Dendrite?
 
-No, not at present. There will be in the future when Dendrite reaches version 1.0.
+No, not at present. There will be in the future when Dendrite reaches version 1.0. For now it is not
+possible to migrate an existing Synapse deployment to Dendrite.
 
-### Can I use Dendrite with an existing Synapse database?
+## Can I use Dendrite with an existing Synapse database?
 
 No, Dendrite has a very different database schema to Synapse and the two are not interchangeable.
 
-### Should I run a monolith or a polylith deployment?
+## Should I run a monolith or a polylith deployment?
 
-Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines.
+Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't
+recommend.
 
-### I've installed Dendrite but federation isn't working
+## I've installed Dendrite but federation isn't working
 
 Check the [Federation Tester](https://federationtester.matrix.org). You need at least:
 
@@ -28,54 +36,57 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at
 * A valid TLS certificate for that DNS name
 * Either DNS SRV records or well-known files
 
-### Does Dendrite work with my favourite client?
+## Does Dendrite work with my favourite client?
 
 It should do, although we are aware of some minor issues:
 
 * **Element Android**: registration does not work, but logging in with an existing account does
 * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
 
-### Does Dendrite support push notifications?
+## Does Dendrite support push notifications?
 
 Yes, we have experimental support for push notifications. Configure them in the usual way in your Matrix client.
 
-### Does Dendrite support application services/bridges?
+## Does Dendrite support application services/bridges?
 
 Possibly - Dendrite does have some application service support but it is not well tested. Please let us know by raising a GitHub issue if you try it and run into problems.
 
 Bridges known to work (as of v0.5.1):
-- [Telegram](https://docs.mau.fi/bridges/python/telegram/index.html)
-- [WhatsApp](https://docs.mau.fi/bridges/go/whatsapp/index.html)
-- [Signal](https://docs.mau.fi/bridges/python/signal/index.html)
-- [probably all other mautrix bridges](https://docs.mau.fi/bridges/)
+
+* [Telegram](https://docs.mau.fi/bridges/python/telegram/index.html)
+* [WhatsApp](https://docs.mau.fi/bridges/go/whatsapp/index.html)
+* [Signal](https://docs.mau.fi/bridges/python/signal/index.html)
+* [probably all other mautrix bridges](https://docs.mau.fi/bridges/)
 
 Remember to add the config file(s) to the `app_service_api` [config](https://github.com/matrix-org/dendrite/blob/de38be469a23813921d01bef3e14e95faab2a59e/dendrite-config.yaml#L130-L131).
 
-### Is it possible to prevent communication with the outside world?
+## Is it possible to prevent communication with the outside world?
 
-Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. 
+Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file.
 
-### Should I use PostgreSQL or SQLite for my databases?
+## Should I use PostgreSQL or SQLite for my databases?
 
-Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. 
+Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users.
 
-### Dendrite is using a lot of CPU
+## Dendrite is using a lot of CPU
 
-Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue.
+Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the
+CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](PROFILING.md) then that would
+be a huge help too, as that will help us to understand where the CPU time is going.
 
-### Dendrite is using a lot of RAM
+## Dendrite is using a lot of RAM
 
-A lot of users report that Dendrite is using a lot of RAM, sometimes even gigabytes of it. This is usually due to Go's allocator behaviour, which tries to hold onto allocated memory until the operating system wants to reclaim it for something else. This can make the memory usage look significantly inflated in tools like `top`/`htop` when actually most of that memory is not really in use at all.
+As above with CPU usage, some memory spikes are expected if Dendrite is doing particularly heavy work
+at a given instant. However, if it is using more RAM than you expect for a long time, that's probably
+not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doing when the memory usage
+ballooned, or file a GitHub issue if you can. If you can take a [memory profile](PROFILING.md) then that
+would be a huge help too, as that will help us to understand where the memory usage is happening.
 
-If you want to prevent this behaviour so that the Go runtime releases memory normally, start Dendrite using the `GODEBUG=madvdontneed=1` environment variable. It is also expected that the allocator behaviour will be changed again in Go 1.16 so that it does not hold onto memory unnecessarily in this way.
-
-If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated memory usage then that's quite possibly a bug - please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue.
-
-### Dendrite is running out of PostgreSQL database connections
+## Dendrite is running out of PostgreSQL database connections
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!
 
-### What is being reported when enabling anonymous stats?
+## What is being reported when enabling anonymous stats?
 
 If anonymous stats reporting is enabled, the following data is send to the defined endpoint.
 
@@ -116,4 +127,4 @@ If anonymous stats reporting is enabled, the following data is send to the defin
     "uptime_seconds": 30,
     "version": "0.8.2"
 }
-```
\ No newline at end of file
+```
diff --git a/docs/Gemfile b/docs/Gemfile
new file mode 100644
index 000000000..a6aa152a2
--- /dev/null
+++ b/docs/Gemfile
@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+gem "github-pages", "~> 226", group: :jekyll_plugins
+group :jekyll_plugins do
+  gem "jekyll-feed", "~> 0.15.1"
+end
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
new file mode 100644
index 000000000..e62aa4ce3
--- /dev/null
+++ b/docs/Gemfile.lock
@@ -0,0 +1,283 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.5)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    coffee-script (2.4.1)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.11.1)
+    colorator (1.1.0)
+    commonmarker (0.23.4)
+    concurrent-ruby (1.1.10)
+    dnsruby (1.61.9)
+      simpleidn (~> 0.1)
+    em-websocket (0.5.3)
+      eventmachine (>= 0.12.9)
+      http_parser.rb (~> 0)
+    ethon (0.15.0)
+      ffi (>= 1.15.0)
+    eventmachine (1.2.7)
+    execjs (2.8.1)
+    faraday (1.10.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.0)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      faraday-retry (~> 1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
+      multipart-post (>= 1.2, < 3)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    faraday-retry (1.0.3)
+    ffi (1.15.5)
+    forwardable-extended (2.6.0)
+    gemoji (3.0.1)
+    github-pages (226)
+      github-pages-health-check (= 1.17.9)
+      jekyll (= 3.9.2)
+      jekyll-avatar (= 0.7.0)
+      jekyll-coffeescript (= 1.1.1)
+      jekyll-commonmark-ghpages (= 0.2.0)
+      jekyll-default-layout (= 0.1.4)
+      jekyll-feed (= 0.15.1)
+      jekyll-gist (= 1.5.0)
+      jekyll-github-metadata (= 2.13.0)
+      jekyll-include-cache (= 0.2.1)
+      jekyll-mentions (= 1.6.0)
+      jekyll-optional-front-matter (= 0.3.2)
+      jekyll-paginate (= 1.1.0)
+      jekyll-readme-index (= 0.3.0)
+      jekyll-redirect-from (= 0.16.0)
+      jekyll-relative-links (= 0.6.1)
+      jekyll-remote-theme (= 0.4.3)
+      jekyll-sass-converter (= 1.5.2)
+      jekyll-seo-tag (= 2.8.0)
+      jekyll-sitemap (= 1.4.0)
+      jekyll-swiss (= 1.0.0)
+      jekyll-theme-architect (= 0.2.0)
+      jekyll-theme-cayman (= 0.2.0)
+      jekyll-theme-dinky (= 0.2.0)
+      jekyll-theme-hacker (= 0.2.0)
+      jekyll-theme-leap-day (= 0.2.0)
+      jekyll-theme-merlot (= 0.2.0)
+      jekyll-theme-midnight (= 0.2.0)
+      jekyll-theme-minimal (= 0.2.0)
+      jekyll-theme-modernist (= 0.2.0)
+      jekyll-theme-primer (= 0.6.0)
+      jekyll-theme-slate (= 0.2.0)
+      jekyll-theme-tactile (= 0.2.0)
+      jekyll-theme-time-machine (= 0.2.0)
+      jekyll-titles-from-headings (= 0.5.3)
+      jemoji (= 0.12.0)
+      kramdown (= 2.3.2)
+      kramdown-parser-gfm (= 1.1.0)
+      liquid (= 4.0.3)
+      mercenary (~> 0.3)
+      minima (= 2.5.1)
+      nokogiri (>= 1.13.4, < 2.0)
+      rouge (= 3.26.0)
+      terminal-table (~> 1.4)
+    github-pages-health-check (1.17.9)
+      addressable (~> 2.3)
+      dnsruby (~> 1.60)
+      octokit (~> 4.0)
+      public_suffix (>= 3.0, < 5.0)
+      typhoeus (~> 1.3)
+    html-pipeline (2.14.1)
+      activesupport (>= 2)
+      nokogiri (>= 1.4)
+    http_parser.rb (0.8.0)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    jekyll (3.9.2)
+      addressable (~> 2.4)
+      colorator (~> 1.0)
+      em-websocket (~> 0.5)
+      i18n (~> 0.7)
+      jekyll-sass-converter (~> 1.0)
+      jekyll-watch (~> 2.0)
+      kramdown (>= 1.17, < 3)
+      liquid (~> 4.0)
+      mercenary (~> 0.3.3)
+      pathutil (~> 0.9)
+      rouge (>= 1.7, < 4)
+      safe_yaml (~> 1.0)
+    jekyll-avatar (0.7.0)
+      jekyll (>= 3.0, < 5.0)
+    jekyll-coffeescript (1.1.1)
+      coffee-script (~> 2.2)
+      coffee-script-source (~> 1.11.1)
+    jekyll-commonmark (1.4.0)
+      commonmarker (~> 0.22)
+    jekyll-commonmark-ghpages (0.2.0)
+      commonmarker (~> 0.23.4)
+      jekyll (~> 3.9.0)
+      jekyll-commonmark (~> 1.4.0)
+      rouge (>= 2.0, < 4.0)
+    jekyll-default-layout (0.1.4)
+      jekyll (~> 3.0)
+    jekyll-feed (0.15.1)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-gist (1.5.0)
+      octokit (~> 4.2)
+    jekyll-github-metadata (2.13.0)
+      jekyll (>= 3.4, < 5.0)
+      octokit (~> 4.0, != 4.4.0)
+    jekyll-include-cache (0.2.1)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-mentions (1.6.0)
+      html-pipeline (~> 2.3)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-optional-front-matter (0.3.2)
+      jekyll (>= 3.0, < 5.0)
+    jekyll-paginate (1.1.0)
+    jekyll-readme-index (0.3.0)
+      jekyll (>= 3.0, < 5.0)
+    jekyll-redirect-from (0.16.0)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-relative-links (0.6.1)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-remote-theme (0.4.3)
+      addressable (~> 2.0)
+      jekyll (>= 3.5, < 5.0)
+      jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0)
+      rubyzip (>= 1.3.0, < 3.0)
+    jekyll-sass-converter (1.5.2)
+      sass (~> 3.4)
+    jekyll-seo-tag (2.8.0)
+      jekyll (>= 3.8, < 5.0)
+    jekyll-sitemap (1.4.0)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-swiss (1.0.0)
+    jekyll-theme-architect (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-cayman (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-dinky (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-hacker (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-leap-day (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-merlot (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-midnight (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-minimal (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-modernist (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-primer (0.6.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-github-metadata (~> 2.9)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-slate (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-tactile (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-time-machine (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-titles-from-headings (0.5.3)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-watch (2.2.1)
+      listen (~> 3.0)
+    jemoji (0.12.0)
+      gemoji (~> 3.0)
+      html-pipeline (~> 2.2)
+      jekyll (>= 3.0, < 5.0)
+    kramdown (2.3.2)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    liquid (4.0.3)
+    listen (3.7.1)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    mercenary (0.3.6)
+    minima (2.5.1)
+      jekyll (>= 3.5, < 5.0)
+      jekyll-feed (~> 0.9)
+      jekyll-seo-tag (~> 2.1)
+    minitest (5.15.0)
+    multipart-post (2.1.1)
+    nokogiri (1.13.6-arm64-darwin)
+      racc (~> 1.4)
+    octokit (4.22.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
+    pathutil (0.16.2)
+      forwardable-extended (~> 2.6)
+    public_suffix (4.0.7)
+    racc (1.6.0)
+    rb-fsevent (0.11.1)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    rexml (3.2.5)
+    rouge (3.26.0)
+    ruby2_keywords (0.0.5)
+    rubyzip (2.3.2)
+    safe_yaml (1.0.5)
+    sass (3.7.4)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    simpleidn (0.2.1)
+      unf (~> 0.1.4)
+    terminal-table (1.8.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
+    thread_safe (0.3.6)
+    typhoeus (1.4.0)
+      ethon (>= 0.9.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8.1)
+    unicode-display_width (1.8.0)
+    zeitwerk (2.5.4)
+
+PLATFORMS
+  arm64-darwin-21
+
+DEPENDENCIES
+  github-pages (~> 226)
+  jekyll-feed (~> 0.15.1)
+  minima (~> 2.5.1)
+
+BUNDLED WITH
+   2.3.7
diff --git a/docs/INSTALL.md b/docs/INSTALL.md
index ca1316aca..add822108 100644
--- a/docs/INSTALL.md
+++ b/docs/INSTALL.md
@@ -1,283 +1,15 @@
-# Installing Dendrite
-
-Dendrite can be run in one of two configurations:
-
-* **Monolith mode**: All components run in the same process. In this mode,
-   it is possible to run an in-process [NATS Server](https://github.com/nats-io/nats-server)
-   instead of running a standalone deployment. This will usually be the preferred model for
-   low-to-mid volume deployments, providing the best balance between performance and resource usage.
-
-* **Polylith mode**: A cluster of individual components running in their own processes, dealing
-  with different aspects of the Matrix protocol (see [WIRING.md](WIRING-Current.md)). Components
-  communicate with each other using internal HTTP APIs and [NATS Server](https://github.com/nats-io/nats-server).
-  This will almost certainly be the preferred model for very large deployments but scalability
-  comes with a cost. API calls are expensive and therefore a polylith deployment may end up using
-  disproportionately more resources for a smaller number of users compared to a monolith deployment.
-
-In almost all cases, it is **recommended to run in monolith mode with PostgreSQL databases**.
-
-Regardless of whether you are running in polylith or monolith mode, each Dendrite component that
-requires storage has its own database connections. Both Postgres and SQLite are supported and can
-be mixed-and-matched across components as needed in the configuration file.
-
-Be advised that Dendrite is still in development and it's not recommended for
-use in production environments just yet!
-
-## Requirements
-
-Dendrite requires:
-
-* Go 1.16 or higher
-* PostgreSQL 12 or higher (if using PostgreSQL databases, not needed for SQLite)
-
-If you want to run a polylith deployment, you also need:
-
-* A standalone [NATS Server](https://github.com/nats-io/nats-server) deployment with JetStream enabled
-
-If you want to build it on Windows, you need `gcc` in the path:
-
-* [MinGW-w64](https://www.mingw-w64.org/)
-
-## Building Dendrite
-
-Start by cloning the code:
-
-```bash
-git clone https://github.com/matrix-org/dendrite
-cd dendrite
-```
-
-Then build it:
-
-* Linux or UNIX-like systems:
-  ```bash
-  ./build.sh
-  ```
-
-* Windows:
-  ```dos
-  build.cmd
-  ```
-
-## Install NATS Server
-
-Follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running).
-
-JetStream must be enabled, either by passing the `-js` flag to `nats-server`,
-or by specifying the `store_dir` option in the the `jetstream` configuration.
-
-## Configuration
-
-### PostgreSQL database setup
-
-Assuming that PostgreSQL 12 (or later) is installed:
-
-* Create role, choosing a new password when prompted:
-
-  ```bash
-  sudo -u postgres createuser -P dendrite
-  ```
-
-At this point you have a choice on whether to run all of the Dendrite
-components from a single database, or for each component to have its
-own database. For most deployments, running from a single database will
-be sufficient, although you may wish to separate them if you plan to
-split out the databases across multiple machines in the future.
-
-On macOS, omit `sudo -u postgres` from the below commands.
-
-* If you want to run all Dendrite components from a single database:
-
-  ```bash
-    sudo -u postgres createdb -O dendrite dendrite
-  ```
-
-  ... in which case your connection string will look like `postgres://user:pass@database/dendrite`.
-
-* If you want to run each Dendrite component with its own database:
-
-  ```bash
-  for i in mediaapi syncapi roomserver federationapi appservice keyserver userapi_accounts; do
-      sudo -u postgres createdb -O dendrite dendrite_$i
-  done
-  ```
-
-  ... in which case your connection string will look like `postgres://user:pass@database/dendrite_componentname`.
-
-### SQLite database setup
-
-**WARNING:** SQLite is suitable for small experimental deployments only and should not be used in production - use PostgreSQL instead for any user-facing federating installation!
-
-Dendrite can use the built-in SQLite database engine for small setups.
-The SQLite databases do not need to be pre-built - Dendrite will
-create them automatically at startup.
-
-### Server key generation
-
-Each Dendrite installation requires:
-
-* A unique Matrix signing private key
-* A valid and trusted TLS certificate and private key
-
-To generate a Matrix signing private key:
-
-```bash
-./bin/generate-keys --private-key matrix_key.pem
-```
-
-**WARNING:** Make sure take a safe backup of this key! You will likely need it if you want to reinstall Dendrite, or
-any other Matrix homeserver, on the same domain name in the future. If you lose this key, you may have trouble joining
-federated rooms.
-
-For testing, you can generate a self-signed certificate and key, although this will not work for public federation:
-
-```bash
-./bin/generate-keys --tls-cert server.crt --tls-key server.key
-```
-
-If you have server keys from an older Synapse instance,
-[convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM
-format and configure them as `old_private_keys` in your config.
-
-### Configuration file
-
-Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Things that will need editing include *at least*:
-
-* The `server_name` entry to reflect the hostname of your Dendrite server
-* The `database` lines with an updated connection string based on your
-  desired setup, e.g. replacing `database` with the name of the database:
-  * For Postgres: `postgres://dendrite:password@localhost/database`, e.g.
-    * `postgres://dendrite:password@localhost/dendrite_userapi_account` to connect to PostgreSQL with SSL/TLS
-    * `postgres://dendrite:password@localhost/dendrite_userapi_account?sslmode=disable` to connect to PostgreSQL without SSL/TLS
-  * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`, e.g. `file:userapi_account.db`
-  * Postgres and SQLite can be mixed and matched on different components as desired.
-* Either one of the following in the `jetstream` configuration section:
-  * The `addresses` option — a list of one or more addresses of an external standalone
-    NATS Server deployment
-  * The `storage_path` — where on the filesystem the built-in NATS server should
-    store durable queues, if using the built-in NATS server
-
-There are other options which may be useful so review them all. In particular,
-if you are trying to federate from your Dendrite instance into public rooms
-then configuring `key_perspectives` (like `matrix.org` in the sample) can
-help to improve reliability considerably by allowing your homeserver to fetch
-public keys for dead homeservers from somewhere else.
-
-**WARNING:** Dendrite supports running all components from the same database in
-PostgreSQL mode, but this is **NOT** a supported configuration with SQLite. When
-using SQLite, all components **MUST** use their own database file.
-
-## Starting a monolith server
-
-The monolith server can be started as shown below. By default it listens for
-HTTP connections on port 8008, so you can configure your Matrix client to use
-`http://servername:8008` as the server:
-
-```bash
-./bin/dendrite-monolith-server
-```
-
-If you set `--tls-cert` and `--tls-key` as shown below, it will also listen
-for HTTPS connections on port 8448:
-
-```bash
-./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key
-```
-
-If the `jetstream` section of the configuration contains no `addresses` but does
-contain a `store_dir`, Dendrite will start up a built-in NATS JetStream node
-automatically, eliminating the need to run a separate NATS server.
-
-## Starting a polylith deployment
-
-The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite.
-
-### nginx (or other reverse proxy)
-
-This is what your clients and federated hosts will talk to. It must forward
-requests onto the correct API server based on URL:
-
-* `/_matrix/client` to the client API server
-* `/_matrix/federation` to the federation API server
-* `/_matrix/key` to the federation API server
-* `/_matrix/media` to the media API server
-
-See `docs/nginx/polylith-sample.conf` for a sample configuration.
-
-### Client API server
-
-This is what implements CS API endpoints. Clients talk to this via the proxy in
-order to send messages, create and join rooms, etc.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi
-```
-
-### Sync server
-
-This is what implements `/sync` requests. Clients talk to this via the proxy
-in order to receive messages.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi
-```
-
-### Media server
-
-This implements `/media` requests. Clients talk to this via the proxy in
-order to upload and retrieve media.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi
-```
-
-### Federation API server
-
-This implements the federation API. Servers talk to this via the proxy in
-order to send transactions.  This is only required if you want to support
-federation.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi
-```
-
-### Internal components
-
-This refers to components that are not directly spoken to by clients. They are only
-contacted by other components. This includes the following components.
-
-#### Room server
-
-This is what implements the room DAG. Clients do not talk to this.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver
-```
-
-#### Appservice server
-
-This sends events from the network to [application
-services](https://matrix.org/docs/spec/application_service/unstable.html)
-running locally.  This is only required if you want to support running
-application services on your homeserver.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml appservice
-```
-
-#### Key server
-
-This manages end-to-end encryption keys for users.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver
-```
-
-#### User server
-
-This manages user accounts, device access tokens and user account data,
-amongst other things.
-
-```bash
-./bin/dendrite-polylith-multi --config=dendrite.yaml userapi
-```
+# Installation
+
+Please note that new installation instructions can be found
+on the [new documentation site](https://matrix-org.github.io/dendrite/),
+or alternatively, in the [installation](installation/) folder:
+
+1. [Planning your deployment](installation/1_planning.md)
+2. [Setting up the domain](installation/2_domainname.md)
+3. [Preparing database storage](installation/3_database.md)
+4. [Generating signing keys](installation/4_signingkey.md)
+5. [Installing as a monolith](installation/5_install_monolith.md)
+6. [Installing as a polylith](installation/6_install_polylith.md)
+7. [Populate the configuration](installation/7_configuration.md)
+8. [Starting the monolith](installation/8_starting_monolith.md)
+9. [Starting the polylith](installation/9_starting_polylith.md)
diff --git a/docs/PROFILING.md b/docs/PROFILING.md
index b026a8aed..f3b573472 100644
--- a/docs/PROFILING.md
+++ b/docs/PROFILING.md
@@ -1,8 +1,14 @@
+---
+title: Profiling
+parent: Development
+permalink: /development/profiling
+---
+
 # Profiling Dendrite
 
 If you are running into problems with Dendrite using excessive resources (e.g. CPU or RAM) then you can use the profiler to work out what is happening.
 
-Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain. 
+Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain.
 
 ## Enable the profiler
 
@@ -16,7 +22,7 @@ If pprof has been enabled successfully, a log line at startup will show that ppr
 
 ```
 WARN[2020-12-03T13:32:33.669405000Z] [/Users/neilalexander/Desktop/dendrite/internal/log.go:87] SetupPprof
-	 Starting pprof on localhost:65432
+  Starting pprof on localhost:65432
 ```
 
 All examples from this point forward assume `PPROFLISTEN=localhost:65432` but you may need to adjust as necessary for your setup.
diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md
deleted file mode 100644
index b74f341e5..000000000
--- a/docs/WIRING-Current.md
+++ /dev/null
@@ -1,71 +0,0 @@
-This document details how various components communicate with each other. There are two kinds of components:
- - Public-facing: exposes CS/SS API endpoints and need to be routed to via client-api-proxy or equivalent.
- - Internal-only: exposes internal APIs and produces Kafka events.
-
-## Internal HTTP APIs
-
-Not everything can be done using Kafka logs. For example, requesting the latest events in a room is much better suited to
-a request/response model like HTTP or RPC. Therefore, components can expose "internal APIs" which sit outside of Kafka logs.
-Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests.
-
-```
-   Tier 1            Sync                         FederationAPI      ClientAPI    MediaAPI
-Public Facing        |                             | | |    | |        | | | |
-                     2   .-------3-----------------` | |    | `--------|-|-|-|--11--------------------.
-                     |   | .--------4----------------------------------` | | |                        |
-                     |   | |         .---5-----------` |    |            | | |                        |
-                     |   | |         |  .---6----------------------------` | |                        |
-                     |   | |         |  |              |  .-----7----------` |                        |
-                     |   | |         |  |              8  | |                10                       |
-                     |   | |         |  |              |  | `---9----.       |                        |
-                     V   V V         V  V              V  V          V       V                        V
-   Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
-Internal only               |                               `------------------------12----------^   ^
-                            `------------------------------------------------------------13----------`
-
- Client ---> Server
-```
-- 2 (Sync -> Roomserver): When making backfill requests
-- 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests
-- 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests
-- 5 (FedAPI -> EDUServer): Sending typing/send-to-device events
-- 6 (ClientAPI -> EDUServer): Sending typing/send-to-device events
-- 7 (ClientAPI -> FedSender): Handling directory lookups
-- 8 (FedAPI -> FedSender): Resetting backoffs when receiving traffic from a server. Querying joined hosts when handling alias lookup requests
-- 9 (FedAPI -> AppService): Working out if the client is an appservice user
-- 10 (ClientAPI -> AppService): Working out if the client is an appservice user
-- 11 (FedAPI -> ServerKeyAPI): Verifying incoming event signatures
-- 12 (FedSender -> ServerKeyAPI): Verifying event signatures of responses (e.g from send_join)
-- 13 (Roomserver -> ServerKeyAPI): Verifying event signatures of backfilled events
-
-In addition to this, all public facing components (Tier 1) talk to the `UserAPI` to verify access tokens and extract profile information where needed.
-
-## Kafka logs
-
-```
-                       .----1--------------------------------------------.
-                       V                                                 |
-   Tier 1            Sync                         FederationAPI      ClientAPI    MediaAPI
-Public Facing        ^   ^                                             ^  
-                     |   |                                             |
-                     2   |                                             |
-                     |   `-3------------.                              |
-                     |                  |                              |
-                     |                  |                              |
-                     |                  |                              |
-                     |   .--------4-----|------------------------------`             
-                     |   |              |           
-   Tier 2           Roomserver     EDUServer         FedSender       AppService    KeyServer   ServerKeyAPI
-Internal only              |          |                ^ ^
-                           |          `-----5----------` |
-                           `--------------------6--------`
-
-
-Producer ----> Consumer
-```
-- 1 (ClientAPI -> Sync): For tracking account data
-- 2 (Roomserver -> Sync): For all data to send to clients
-- 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients
-- 4 (Roomserver -> ClientAPI): For tracking memberships for profile updates.
-- 5 (EDUServer -> FedSender): For sending EDUs over federation
-- 6 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts.
diff --git a/docs/WIRING.md b/docs/WIRING.md
deleted file mode 100644
index 8ec5b0432..000000000
--- a/docs/WIRING.md
+++ /dev/null
@@ -1,229 +0,0 @@
-# Wiring
-
-The diagram is incomplete. The following things aren't shown on the diagram:
-
-* Device Messages
-* User Profiles
-* Notification Counts
-* Sending federation.
-* Querying federation.
-* Other things that aren't shown on the diagram.
-
-Diagram:
-
-
-    W -> Writer
-    S -> Server/Store/Service/Something/Stuff
-    R -> Reader
-
-               +---+                                                    +---+                              +---+
-    +----------| W |                                         +----------| S |                     +--------| R |
-    |          +---+                                         | Receipts +---+                     | Client +---+
-    | Federation |>=========================================>| Server     |>=====================>| Sync     |
-    | Receiver   |                                           |            |                       |          |
-    |            |                                 +---+     |            |                       |          |
-    |            |                        +--------| W |     |            |                       |          |
-    |            |                        | Client +---+     |            |                       |          |
-    |            |                        | Receipt  |>=====>|            |                       |          |
-    |            |                        | Updater  |       |            |                       |          |
-    |            |                        +----------+       |            |                       |          |
-    |            |                                           |            |                       |          |
-    |            |                +---+            +---+     |            |                +---+  |          |
-    |            |   +------------| W |     +------| S |     |            |       +--------| R |  |          |
-    |            |   | Federation +---+     | Room +---+     |            |       | Client +---+  |          |
-    |            |   | Backfill     |>=====>| Server |>=====>|            |>=====>| Push     |    |          |
-    |            |   +--------------+       |        |       +------------+       |          |    |          |
-    |            |                          |        |                            |          |    |          |
-    |            |                          |        |>==========================>|          |    |          |
-    |            |                          |        |                            +----------+    |          |
-    |            |                          |        |                     +---+                  |          |
-    |            |                          |        |       +-------------| R |                  |          |
-    |            |                          |        |>=====>| Application +---+                  |          |
-    |            |                          |        |       | Services     |                     |          |
-    |            |                          |        |       +--------------+                     |          |
-    |            |                          |        |                                     +---+  |          |
-    |            |                          |        |                            +--------| R |  |          |
-    |            |                          |        |                            | Client +---+  |          |
-    |            |>========================>|        |>==========================>| Search   |    |          |
-    |            |                          |        |                            |          |    |          |
-    |            |                          |        |                            +----------+    |          |
-    |            |                          |        |                                            |          |
-    |            |                          |        |>==========================================>|          |
-    |            |                          |        |                                            |          |
-    |            |                +---+     |        |                  +---+                     |          |
-    |            |       +--------| W |     |        |       +----------| S |                     |          |
-    |            |       | Client +---+     |        |       | Presence +---+                     |          |
-    |            |       | API      |>=====>|        |>=====>| Server     |>=====================>|          |
-    |            |       | /send    |       +--------+       |            |                       |          |
-    |            |       |          |                        |            |                       |          |
-    |            |       |          |>======================>|            |<=====================<|          |
-    |            |       +----------+                        |            |                       |          |
-    |            |                                           |            |                       |          |
-    |            |                                 +---+     |            |                       |          |
-    |            |                        +--------| W |     |            |                       |          |
-    |            |                        | Client +---+     |            |                       |          |
-    |            |                        | Presence |>=====>|            |                       |          |
-    |            |                        | Setter   |       |            |                       |          |
-    |            |                        +----------+       |            |                       |          |
-    |            |                                           |            |                       |          |
-    |            |                                           |            |                       |          |
-    |            |>=========================================>|            |                       |          |
-    |            |                                           +------------+                       |          |
-    |            |                                                                                |          |
-    |            |                                                      +---+                     |          |
-    |            |                                           +----------| S |                     |          |
-    |            |                                           | EDU      +---+                     |          |
-    |            |>=========================================>| Server     |>=====================>|          |
-    +------------+                                           |            |                       +----------+
-                                                   +---+     |            |
-                                          +--------| W |     |            |
-                                          | Client +---+     |            |
-                                          | Typing   |>=====>|            |
-                                          | Setter   |       |            |
-                                          +----------+       +------------+
-
-
-# Component Descriptions
-
-Many of the components are logical rather than physical. For example it is
-possible that all of the client API writers will end up being glued together
-and always deployed as a single unit.
-
-Outbound federation requests will probably need to be funnelled through a
-choke-point to implement ratelimiting and backoff correctly.
-
-## Federation Send
-
- * Handles `/federation/v1/send/` requests.
- * Fetches missing ``prev_events`` from the remote server if needed.
- * Fetches missing room state from the remote server if needed.
- * Checks signatures on remote events, downloading keys if needed.
- * Queries information needed to process events from the Room Server.
- * Writes room events to logs.
- * Writes presence updates to logs.
- * Writes receipt updates to logs.
- * Writes typing updates to logs.
- * Writes other updates to logs.
-
-## Client API /send
-
- * Handles puts to `/client/v1/rooms/` that create room events.
- * Queries information needed to process events from the Room Server.
- * Talks to remote servers if needed for joins and invites.
- * Writes room event pdus.
- * Writes presence updates to logs.
-
-## Client Presence Setter
-
- * Handles puts to the [client API presence paths](https://matrix.org/docs/spec/client_server/unstable.html#id41).
- * Writes presence updates to logs.
-
-## Client Typing Setter
-
- * Handles puts to the [client API typing paths](https://matrix.org/docs/spec/client_server/unstable.html#id32).
- * Writes typing updates to logs.
-
-## Client Receipt Updater
-
- * Handles puts to the [client API receipt paths](https://matrix.org/docs/spec/client_server/unstable.html#id36).
- * Writes receipt updates to logs.
-
-## Federation Backfill
-
- * Backfills events from other servers
- * Writes the resulting room events to logs.
- * Is a different component from the room server itself cause it'll
-   be easier if the room server component isn't making outbound HTTP requests
-   to remote servers
-
-## Room Server
-
- * Reads new and backfilled room events from the logs written by FS, FB and CRS.
- * Tracks the current state of the room and the state at each event.
- * Probably does auth checks on the incoming events.
- * Handles state resolution as part of working out the current state and the
-   state at each event.
- * Writes updates to the current state and new events to logs.
- * Shards by room ID.
-
-## Receipt Server
-
- * Reads new updates to receipts from the logs written by the FS and CRU.
- * Somehow learns enough information from the room server to workout how the
-   current receipt markers move with each update.
- * Writes the new marker positions to logs
- * Shards by room ID?
- * It may be impossible to implement without folding it into the Room Server
-   forever coupling the components together.
-
-## EDU Server
-
- * Reads new updates to typing from the logs written by the FS and CTS.
- * Updates the current list of people typing in a room.
- * Writes the current list of people typing in a room to the logs.
- * Shards by room ID?
-
-## Presence Server
-
- * Reads the current state of the rooms from the logs to track the intersection
-   of room membership between users.
- * Reads updates to presence from the logs written by the FS and the CPS.
- * Reads when clients sync from the logs from the Client Sync.
- * Tracks any timers for users.
- * Writes the changes to presence state to the logs.
- * Shards by user ID somehow?
-
-## Client Sync
-
- * Handle /client/v2/sync requests.
- * Reads new events and the current state of the rooms from logs written by the Room Server.
- * Reads new receipts positions from the logs written by the Receipts Server.
- * Reads changes to presence from the logs written by the Presence Server.
- * Reads changes to typing from the logs written by the EDU Server.
- * Writes when a client starts and stops syncing to the logs.
-
-## Client Search
-
- * Handle whatever the client API path for event search is?
- * Reads new events and the current state of the rooms from logs writeen by the Room Server.
- * Maintains a full text search index of somekind.
-
-## Client Push
-
- * Pushes unread messages to remote push servers.
- * Reads new events and the current state of the rooms from logs writeen by the Room Server.
- * Reads the position of the read marker from the Receipts Server.
- * Makes outbound HTTP hits to the push server for the client device.
-
-## Application Service
-
- * Receives events from the Room Server.
- * Filters events and sends them to each registered application service.
- * Runs a separate goroutine for each application service.
-
-# Internal Component API
-
-Some dendrite components use internal APIs to communicate information back
-and forth between each other. There are two implementations of each API, one
-that uses HTTP requests and one that does not. The HTTP implementation is
-used in multi-process mode, so processes on separate computers may still
-communicate, whereas in single-process or Monolith mode, the direct
-implementation is used. HTTP is preferred here to kafka streams as it allows
-for request responses.
-
-Running `dendrite-monolith-server` will set up direct connections between
-components, whereas running each individual component (which are only run in
-multi-process mode) will set up HTTP-based connections.
-
-The functions that make HTTP requests to internal APIs of a component are
-located in `/<component name>/api/<name>.go`, named according to what
-functionality they cover. Each of these requests are handled in `/<component
-name>/<name>/<name>.go`.
-
-As an example, the `appservices` component allows other Dendrite components
-to query external application services via its internal API. A component
-would call the desired function in `/appservices/api/query.go`. In
-multi-process mode, this would send an internal HTTP request, which would
-be handled by a function in `/appservices/query/query.go`. In single-process
-mode, no internal HTTP request occurs, instead functions are simply called
-directly, thus requiring no changes on the calling component's end.
diff --git a/docs/_config.yml b/docs/_config.yml
new file mode 100644
index 000000000..ed93fd796
--- /dev/null
+++ b/docs/_config.yml
@@ -0,0 +1,19 @@
+title: Dendrite
+description: >-
+  Second-generation Matrix homeserver written in Go!
+baseurl: "/dendrite" # the subpath of your site, e.g. /blog
+url: ""
+twitter_username: matrixdotorg
+github_username: matrix-org
+remote_theme: just-the-docs/just-the-docs
+plugins:
+  - jekyll-feed
+aux_links:
+  "GitHub":
+    - "//github.com/matrix-org/dendrite"
+aux_links_new_tab: true
+sass:
+  sass_dir: _sass
+  style: compressed
+exclude:
+  - INSTALL.md
diff --git a/docs/_sass/custom/custom.scss b/docs/_sass/custom/custom.scss
new file mode 100644
index 000000000..8a5ed3d8d
--- /dev/null
+++ b/docs/_sass/custom/custom.scss
@@ -0,0 +1,3 @@
+footer.site-footer {
+    opacity: 10%;
+}
\ No newline at end of file
diff --git a/docs/administration.md b/docs/administration.md
new file mode 100644
index 000000000..08ad7803e
--- /dev/null
+++ b/docs/administration.md
@@ -0,0 +1,10 @@
+---
+title: Administration
+has_children: yes
+nav_order: 4
+permalink: /administration
+---
+
+# Administration
+
+This section contains documentation on managing your existing Dendrite deployment.
diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md
new file mode 100644
index 000000000..f40b7f576
--- /dev/null
+++ b/docs/administration/1_createusers.md
@@ -0,0 +1,53 @@
+---
+title: Creating user accounts
+parent: Administration
+permalink: /administration/createusers
+nav_order: 1
+---
+
+# Creating user accounts
+
+User accounts can be created on a Dendrite instance in a number of ways.
+
+## From the command line
+
+The `create-account` tool is built in the `bin` folder when building Dendrite with
+the `build.sh` script.
+
+It uses the `dendrite.yaml` configuration file to connect to the Dendrite user database
+and create the account entries directly. It can therefore be used even if Dendrite is not
+running yet, as long as the database is up.
+
+An example of using `create-account` to create a **normal account**:
+
+```bash
+./bin/create-account -config /path/to/dendrite.yaml -username USERNAME
+```
+
+You will be prompted to enter a new password for the new account.
+
+To create a new **admin account**, add the `-admin` flag:
+
+```bash
+./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
+```
+
+## Using shared secret registration
+
+Dendrite supports the Synapse-compatible shared secret registration endpoint.
+
+To enable shared secret registration, you must first enable it in the `dendrite.yaml`
+configuration file by specifying a shared secret. In the `client_api` section of the config,
+enter a new secret into the `registration_shared_secret` field:
+
+```yaml
+client_api:
+  # ...
+  registration_shared_secret: ""
+```
+
+You can then use the `/_synapse/admin/v1/register` endpoint as per the
+[Synapse documentation](https://matrix-org.github.io/synapse/latest/admin_api/register_api.html).
+
+Shared secret registration is only enabled once a secret is configured. To disable shared
+secret registration again, remove the secret from the configuration file.
diff --git a/docs/administration/2_registration.md b/docs/administration/2_registration.md
new file mode 100644
index 000000000..66949f2ca
--- /dev/null
+++ b/docs/administration/2_registration.md
@@ -0,0 +1,53 @@
+---
+title: Enabling registration
+parent: Administration
+permalink: /administration/registration
+nav_order: 2
+---
+
+# Enabling registration
+
+Enabling registration allows users to register their own user accounts on your
+Dendrite server using their Matrix client. They will be able to choose their own
+username and password and log in.
+
+Registration is controlled by the `registration_disabled` field in the `client_api`
+section of the configuration. By default, `registration_disabled` is set to `true`,
+disabling registration. If you want to enable registration, you should change this
+setting to `false`.
+
+Currently Dendrite supports secondary verification using [reCAPTCHA](https://www.google.com/recaptcha/about/).
+Other methods will be supported in the future.
+
+## reCAPTCHA verification
+
+Dendrite supports reCAPTCHA as a secondary verification method. If you want to enable
+registration, it is **highly recommended** to configure reCAPTCHA. This will make it
+much more difficult for automated spam systems from registering accounts on your
+homeserver automatically.
+
+You will need an API key from the [reCAPTCHA Admin Panel](https://www.google.com/recaptcha/admin).
+Then configure the relevant details in the `client_api` section of the configuration:
+
+```yaml
+client_api:
+  # ...
+  registration_disabled: false
+  recaptcha_public_key: "PUBLIC_KEY_HERE"
+  recaptcha_private_key: "PRIVATE_KEY_HERE"
+  enable_registration_captcha: true
+  captcha_bypass_secret: ""
+  recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
+```
+
+## Open registration
+
+Dendrite does support open registration — that is, allowing users to create their own
+user accounts without any verification or secondary authentication. However, it
+is **not recommended** to enable open registration, as this leaves your homeserver
+vulnerable to abuse by spammers or attackers, who create large numbers of user
+accounts on Matrix homeservers in order to send spam or abuse into the network.
+
+It isn't possible to enable open registration in Dendrite in a single step. If you
+try to disable the `registration_disabled` option without any secondary verification
+methods enabled (such as reCAPTCHA), Dendrite will log an error and fail to start.
diff --git a/docs/administration/3_presence.md b/docs/administration/3_presence.md
new file mode 100644
index 000000000..858025370
--- /dev/null
+++ b/docs/administration/3_presence.md
@@ -0,0 +1,39 @@
+---
+title: Enabling presence
+parent: Administration
+permalink: /administration/presence
+nav_order: 3
+---
+
+# Enabling presence
+
+Dendrite supports presence, which allows you to send your online/offline status
+to other users, and to receive their statuses automatically. They will be displayed
+by supported clients.
+
+Note that enabling presence **can negatively impact** the performance of your Dendrite
+server — it will require more CPU time and will increase the "chattiness" of your server
+over federation. It is disabled by default for this reason.
+
+Dendrite has two options for controlling presence:
+
+* **Enable inbound presence**: Dendrite will handle presence updates for remote users
+  and distribute them to local users on your homeserver;
+* **Enable outbound presence**: Dendrite will generate presence notifications for your
+  local users and distribute them to remote users over the federation.
+
+This means that you can configure only one or other direction if you prefer, i.e. to
+receive presence from other servers without revealing the presence of your own users.
+
+## Configuring presence
+
+Presence is controlled by the `presence` block in the `global` section of the
+configuration file:
+
+```yaml
+global:
+  # ...
+  presence:
+    enable_inbound: false
+    enable_outbound: false
+```
diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
new file mode 100644
index 000000000..e33482ec9
--- /dev/null
+++ b/docs/administration/4_adminapi.md
@@ -0,0 +1,25 @@
+---
+title: Supported admin APIs
+parent: Administration
+permalink: /administration/adminapi
+---
+
+# Supported admin APIs
+
+Dendrite supports, at present, a very small number of endpoints that allow
+admin users to perform administrative functions. Please note that there is no
+API stability guarantee on these endpoints at present — they may change shape
+without warning.
+
+More endpoints will be added in the future.
+
+## `/_dendrite/admin/evacuateRoom/{roomID}`
+
+This endpoint will instruct Dendrite to part all local users from the given `roomID`
+in the URL. It may take some time to complete. A JSON body will be returned containing
+the user IDs of all affected users.
+
+## `/_synapse/admin/v1/register`
+
+Shared secret registration — please see the [user creation page](createusers) for
+guidance on configuring and using this endpoint.
diff --git a/docs/development.md b/docs/development.md
new file mode 100644
index 000000000..cf296fb53
--- /dev/null
+++ b/docs/development.md
@@ -0,0 +1,10 @@
+---
+title: Development
+has_children: true
+permalink: /development
+---
+
+# Development
+
+This section contains documentation that may be useful when helping to develop
+Dendrite.
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 000000000..d77af87a8
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,24 @@
+---
+layout: home
+nav_exclude: true
+---
+
+# Dendrite
+
+Dendrite is a second-generation Matrix homeserver written in Go! Following the microservice
+architecture model, Dendrite is designed to be efficient, reliable and scalable. Despite being beta,
+many Matrix features are already supported.
+
+This site aims to include relevant documentation to help you to get started with and
+run Dendrite. Check out the following sections:
+
+* **[Installation](INSTALL.md)** for building and deploying your own Dendrite homeserver
+* **[Administration](administration.md)** for managing an existing Dendrite deployment
+* **[Development](development.md)** for developing against Dendrite
+
+You can also join us in our Matrix rooms dedicated to Dendrite, but please check first that
+your question hasn't already been [answered in the FAQ](FAQ.md):
+
+* **[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org)** for general project discussion and support
+* **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** for chat on Dendrite development specifically
+* **[#dendrite-alerts:matrix.org](https://matrix.to/#/#dendrite-alerts:matrix.org)** for release notifications and other important announcements
diff --git a/docs/installation.md b/docs/installation.md
new file mode 100644
index 000000000..c38a6dbb2
--- /dev/null
+++ b/docs/installation.md
@@ -0,0 +1,10 @@
+---
+title: Installation
+has_children: true
+nav_order: 2
+permalink: /installation
+---
+
+# Installation
+
+This section contains documentation on installing a new Dendrite deployment.
diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md
new file mode 100644
index 000000000..89cc5b4a6
--- /dev/null
+++ b/docs/installation/1_planning.md
@@ -0,0 +1,110 @@
+---
+title: Planning your installation
+parent: Installation
+nav_order: 1
+permalink: /installation/planning
+---
+
+# Planning your installation
+
+## Modes
+
+Dendrite can be run in one of two configurations:
+
+* **Monolith mode**: All components run in the same process. In this mode,
+   it is possible to run an in-process NATS Server instead of running a standalone deployment.
+   This will usually be the preferred model for low-to-mid volume deployments, providing the best
+   balance between performance and resource usage.
+
+* **Polylith mode**: A cluster of individual components running in their own processes, dealing
+  with different aspects of the Matrix protocol. Components communicate with each other using
+  internal HTTP APIs and NATS Server. This will almost certainly be the preferred model for very
+  large deployments but scalability comes with a cost. API calls are expensive and therefore a
+  polylith deployment may end up using disproportionately more resources for a smaller number of
+  users compared to a monolith deployment.
+
+At present, we **recommend monolith mode deployments** in all cases.
+
+## Databases
+
+Dendrite can run with either a PostgreSQL or a SQLite backend. There are considerable tradeoffs
+to consider:
+
+* **PostgreSQL**: Needs to run separately to Dendrite, needs to be installed and configured separately
+  and and will use more resources over all, but will be **considerably faster** than SQLite. PostgreSQL
+  has much better write concurrency which will allow Dendrite to process more tasks in parallel. This
+  will be necessary for federated deployments to perform adequately.
+
+* **SQLite**: Built into Dendrite, therefore no separate database engine is necessary and is quite
+  a bit easier to set up, but will be much slower than PostgreSQL in most cases. SQLite only allows a
+  single writer on a database at a given time, which will significantly restrict Dendrite's ability
+  to process multiple tasks in parallel.
+
+At this time, we **recommend the PostgreSQL database engine** for all production deployments.
+
+## Requirements
+
+Dendrite will run on Linux, macOS and Windows Server. It should also run fine on variants
+of BSD such as FreeBSD and OpenBSD. We have not tested Dendrite on AIX, Solaris, Plan 9 or z/OS —
+your mileage may vary with these platforms.
+
+It is difficult to state explicitly the amount of CPU, RAM or disk space that a Dendrite
+installation will need, as this varies considerably based on a number of factors. In particular:
+
+* The number of users using the server;
+* The number of rooms that the server is joined to — federated rooms in particular will typically
+  use more resources than rooms with only local users;
+* The complexity of rooms that the server is joined to — rooms with more members coming and
+  going will typically be of a much higher complexity.
+
+Some tasks are more expensive than others, such as joining rooms over federation, running state
+resolution or sending messages into very large federated rooms with lots of remote users. Therefore
+you should plan accordingly and ensure that you have enough resources available to endure spikes
+in CPU or RAM usage, as these may be considerably higher than the idle resource usage.
+
+At an absolute minimum, Dendrite will expect 1GB RAM. For a comfortable day-to-day deployment
+which can participate in federated rooms for a number of local users, be prepared to assign 2-4
+CPU cores and 8GB RAM — more if your user count increases.
+
+If you are running PostgreSQL on the same machine, allow extra headroom for this too, as the
+database engine will also have CPU and RAM requirements of its own. Running too many heavy
+services on the same machine may result in resource starvation and processes may end up being
+killed by the operating system if they try to use too much memory.
+
+## Dependencies
+
+In order to install Dendrite, you will need to satisfy the following dependencies.
+
+### Go
+
+At this time, Dendrite supports being built with Go 1.16 or later. We do not support building
+Dendrite with older versions of Go than this. If you are installing Go using a package manager,
+you should check (by running `go version`) that you are using a suitable version before you start.
+
+### PostgreSQL
+
+If using the PostgreSQL database engine, you should install PostgreSQL 12 or later.
+
+### NATS Server
+
+Monolith deployments come with a built-in [NATS Server](https://github.com/nats-io/nats-server) and
+therefore do not need this to be manually installed. If you are planning a monolith installation, you
+do not need to do anything.
+
+Polylith deployments, however, currently need a standalone NATS Server installation with JetStream
+enabled.
+
+To do so, follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). JetStream must be enabled, either by passing the `-js` flag to `nats-server`,
+or by specifying the `store_dir` option in the the `jetstream` configuration.
+
+### Reverse proxy (polylith deployments)
+
+Polylith deployments require a reverse proxy, such as [NGINX](https://www.nginx.com) or
+[HAProxy](http://www.haproxy.org). Configuring those is not covered in this documentation,
+although a [sample configuration for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
+is provided.
+
+### Windows
+
+Finally, if you want to build Dendrite on Windows, you will need need `gcc` in the path. The best
+way to achieve this is by installing and building Dendrite under [MinGW-w64](https://www.mingw-w64.org/).
diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md
new file mode 100644
index 000000000..0d4300eca
--- /dev/null
+++ b/docs/installation/2_domainname.md
@@ -0,0 +1,93 @@
+---
+title: Setting up the domain
+parent: Installation
+nav_order: 2
+permalink: /installation/domainname
+---
+
+# Setting up the domain
+
+Every Matrix server deployment requires a server name which uniquely identifies it. For
+example, if you are using the server name `example.com`, then your users will have usernames
+that take the format `@user:example.com`.
+
+For federation to work, the server name must be resolvable by other homeservers on the internet
+— that is, the domain must be registered and properly configured with the relevant DNS records.
+
+Matrix servers discover each other when federating using the following methods:
+
+1. If a well-known delegation exists on `example.com`, use the path server from the
+   well-known file to connect to the remote homeserver;
+2. If a DNS SRV delegation exists on `example.com`, use the hostname and port from the DNS SRV
+   record to connect to the remote homeserver;
+3. If neither well-known or DNS SRV delegation are configured, attempt to connect to the remote
+   homeserver by connecting to `example.com` port TCP/8448 using HTTPS.
+
+## TLS certificates
+
+Matrix federation requires that valid TLS certificates are present on the domain. You must
+obtain certificates from a publicly accepted Certificate Authority (CA). [LetsEncrypt](https://letsencrypt.org)
+is an example of such a CA that can be used. Self-signed certificates are not suitable for
+federation and will typically not be accepted by other homeservers.
+
+A common practice to help ease the management of certificates is to install a reverse proxy in
+front of Dendrite which manages the TLS certificates and HTTPS proxying itself. Software such as
+[NGINX](https://www.nginx.com) and [HAProxy](http://www.haproxy.org) can be used for the task.
+Although the finer details of configuring these are not described here, you must reverse proxy
+all `/_matrix` paths to your Dendrite server.
+
+It is possible for the reverse proxy to listen on the standard HTTPS port TCP/443 so long as your
+domain delegation is configured to point to port TCP/443.
+
+## Delegation
+
+Delegation allows you to specify the server name and port that your Dendrite installation is
+reachable at, or to host the Dendrite server at a different server name to the domain that
+is being delegated.
+
+For example, if your Dendrite installation is actually reachable at `matrix.example.com` port 8448,
+you will be able to delegate from `example.com` to `matrix.example.com` so that your users will have
+`@user:example.com` user names instead of `@user:matrix.example.com` usernames.
+
+Delegation can be performed in one of two ways:
+
+* **Well-known delegation**: A well-known text file is served over HTTPS on the domain name
+  that you want to use, pointing to your server on `matrix.example.com` port 8448;
+* **DNS SRV delegation**: A DNS SRV record is created on the domain name that you want to
+  use, pointing to your server on `matrix.example.com` port TCP/8448.
+
+If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or DNS SRV
+delegation must refer to the hostname and port that the reverse proxy is listening on instead.
+
+Well-known delegation is typically easier to set up and usually preferred. However, you can use
+either or both methods to delegate. If you configure both methods of delegation, it is important
+that they both agree and refer to the same hostname and port.
+
+## Well-known delegation
+
+Using well-known delegation requires that you are running a web server at `example.com` which
+is listening on the standard HTTPS port TCP/443.
+
+Assuming that your Dendrite installation is listening for HTTPS connections at `matrix.example.com`
+on port 8448, the delegation file must be served at `https://example.com/.well-known/matrix/server`
+and contain the following JSON document:
+
+```json
+{
+    "m.server": "https://matrix.example.com:8448"
+}
+```
+
+## DNS SRV delegation
+
+Using DNS SRV delegation requires creating DNS SRV records on the `example.com` zone which
+refer to your Dendrite installation.
+
+Assuming that your Dendrite installation is listening for HTTPS connections at `matrix.example.com`
+port 8448, the DNS SRV record must have the following fields:
+
+* Name: `@` (or whichever term your DNS provider uses to signal the root)
+* Service: `_matrix`
+* Protocol: `_tcp`
+* Port: `8448`
+* Target: `matrix.example.com`
diff --git a/docs/installation/3_database.md b/docs/installation/3_database.md
new file mode 100644
index 000000000..f64fe9150
--- /dev/null
+++ b/docs/installation/3_database.md
@@ -0,0 +1,106 @@
+---
+title: Preparing database storage
+parent: Installation
+nav_order: 3
+permalink: /installation/database
+---
+
+# Preparing database storage
+
+Dendrite uses SQL databases to store data. Depending on the database engine being used, you
+may need to perform some manual steps outlined below.
+
+## SQLite
+
+SQLite deployments do not require manual database creation. Simply configure the database
+filenames in the Dendrite configuration file and start Dendrite. The databases will be created
+and populated automatically.
+
+Note that Dendrite **cannot share a single SQLite database across multiple components**. Each
+component must be configured with its own SQLite database filename.
+
+### Connection strings
+
+Connection strings for SQLite databases take the following forms:
+
+* Current working directory path: `file:dendrite_component.db`
+* Full specified path: `file:///path/to/dendrite_component.db`
+
+## PostgreSQL
+
+Dendrite can automatically populate the database with the relevant tables and indexes, but
+it is not capable of creating the databases themselves. You will need to create the databases
+manually.
+
+At this point, you can choose to either use a single database for all Dendrite components,
+or you can run each component with its own separate database:
+
+* **Single database**: You will need to create a single PostgreSQL database. Monolith deployments
+  can use a single global connection pool, which makes updating the configuration file much easier.
+  Only one database connection string to manage and likely simpler to back up the database. All
+  components will be sharing the same database resources (CPU, RAM, storage).
+
+* **Separate databases**: You will need to create a separate PostgreSQL database for each
+  component. You will need to configure each component that has storage in the Dendrite
+  configuration file with its own connection parameters. Allows running a different database engine
+  for each component on a different machine if needs be, each with their own CPU, RAM and storage —
+  almost certainly overkill unless you are running a very large Dendrite deployment.
+
+For either configuration, you will want to:
+
+1. Configure a role (with a username and password) which Dendrite can use to connect to the
+   database;
+2. Create the database(s) themselves, ensuring that the Dendrite role has privileges over them.
+   As Dendrite will create and manage the database tables, indexes and sequences by itself, the
+   Dendrite role must have suitable privileges over the database.
+
+### Connection strings
+
+The format of connection strings for PostgreSQL databases is described in the [PostgreSQL libpq manual](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). Note that Dendrite only
+supports the "Connection URIs" format and **will not** work with the "Keyword/Value Connection
+string" format.
+
+Example supported connection strings take the format:
+
+* `postgresql://user:pass@hostname/database?options=...`
+* `postgres://user:pass@hostname/database?options=...`
+
+If you need to disable SSL/TLS on the database connection, you may need to append `?sslmode=disable` to the end of the connection string.
+
+### Role creation
+
+Create a role which Dendrite can use to connect to the database, choosing a new password when
+prompted. On macOS, you may need to omit the `sudo -u postgres` from the below instructions.
+
+```bash
+sudo -u postgres createuser -P dendrite
+```
+
+### Single database creation
+
+Create the database itself, using the `dendrite` role from above:
+
+```bash
+sudo -u postgres createdb -O dendrite dendrite
+```
+
+### Multiple database creation
+
+The following eight components require a database. In this example they will be named:
+
+| Appservice API | `dendrite_appservice`    |
+| Federation API | `dendrite_federationapi` |
+| Media API      | `dendrite_mediaapi`      |
+| MSCs           | `dendrite_mscs`          |
+| Roomserver     | `dendrite_roomserver`    |
+| Sync API       | `dendrite_syncapi`       |
+| Key server     | `dendrite_keyserver`     |
+| User API       | `dendrite_userapi`       |
+
+... therefore you will need to create eight different databases:
+
+```bash
+for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver userapi; do
+    sudo -u postgres createdb -O dendrite dendrite_$i
+done
+```
diff --git a/docs/installation/4_signingkey.md b/docs/installation/4_signingkey.md
new file mode 100644
index 000000000..07dc485ff
--- /dev/null
+++ b/docs/installation/4_signingkey.md
@@ -0,0 +1,79 @@
+---
+title: Generating signing keys
+parent: Installation
+nav_order: 4
+permalink: /installation/signingkeys
+---
+
+# Generating signing keys
+
+All Matrix homeservers require a signing private key, which will be used to authenticate
+federation requests and events.
+
+The `generate-keys` utility can be used to generate a private key. Assuming that Dendrite was
+built using `build.sh`, you should find the `generate-keys` utility in the `bin` folder.
+
+To generate a Matrix signing private key:
+
+```bash
+./bin/generate-keys --private-key matrix_key.pem
+```
+
+The generated `matrix_key.pem` file is your new signing key.
+
+## Important warning
+
+You must treat this key as if it is highly sensitive and private, so **never share it with
+anyone**. No one should ever ask you for this key for any reason, even to debug a problematic
+Dendrite server.
+
+Make sure take a safe backup of this key. You will likely need it if you want to reinstall
+Dendrite, or any other Matrix homeserver, on the same domain name in the future. If you lose
+this key, you may have trouble joining federated rooms.
+
+## Old signing keys
+
+If you already have old signing keys from a previous Matrix installation on the same domain
+name, you can reuse those instead, as long as they have not been previously marked as expired —
+a key that has been marked as expired in the past is unusable.
+
+Old keys from a previous Dendrite installation can be reused as-is without any further
+configuration required. Simply use that key file in the Dendrite configuration.
+
+If you have server keys from an older Synapse instance, you can convert them to Dendrite's PEM
+format and configure them as `old_private_keys` in your config.
+
+## Key format
+
+Dendrite stores the server signing key in the PEM format with the following structure.
+
+```
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:<Key ID>
+
+<Base64 Encoded Key Data>
+-----END MATRIX PRIVATE KEY-----
+```
+
+## Converting Synapse keys
+
+If you have signing keys from a previous Synapse installation, you should ideally configure them
+as `old_private_keys` in your Dendrite config file. Synapse stores signing keys in the following
+format:
+
+```
+ed25519 <Key ID> <Base64 Encoded Key Data>
+```
+
+To convert this key to Dendrite's PEM format, use the following template. You must copy the Key ID
+exactly without modifying it. **It is important to include the trailing equals sign on the Base64
+Encoded Key Data** if it is not already present in the original key, as the key data needs to be
+padded to exactly 32 bytes:
+
+```
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:<Key ID>
+
+<Base64 Encoded Key Data>=
+-----END MATRIX PRIVATE KEY-----
+```
diff --git a/docs/installation/5_install_monolith.md b/docs/installation/5_install_monolith.md
new file mode 100644
index 000000000..7de066cf7
--- /dev/null
+++ b/docs/installation/5_install_monolith.md
@@ -0,0 +1,21 @@
+---
+title: Installing as a monolith
+parent: Installation
+has_toc: true
+nav_order: 5
+permalink: /installation/install/monolith
+---
+
+# Installing as a monolith
+
+You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`:
+
+```sh
+go install ./cmd/dendrite-monolith-server
+```
+
+Alternatively, you can specify a custom path for the binary to be written to using `go build`:
+
+```sh
+go build -o /usr/local/bin/ ./cmd/dendrite-monolith-server
+```
diff --git a/docs/installation/6_install_polylith.md b/docs/installation/6_install_polylith.md
new file mode 100644
index 000000000..375512f8f
--- /dev/null
+++ b/docs/installation/6_install_polylith.md
@@ -0,0 +1,33 @@
+---
+title: Installing as a polylith
+parent: Installation
+has_toc: true
+nav_order: 6
+permalink: /installation/install/polylith
+---
+
+# Installing as a polylith
+
+You can install the Dendrite polylith binary into `$GOPATH/bin` by using `go install`:
+
+```sh
+go install ./cmd/dendrite-polylith-multi
+```
+
+Alternatively, you can specify a custom path for the binary to be written to using `go build`:
+
+```sh
+go build -o /usr/local/bin/ ./cmd/dendrite-polylith-multi
+```
+
+The `dendrite-polylith-multi` binary is a "multi-personality" binary which can run as
+any of the components depending on the supplied command line parameters.
+
+## Reverse proxy
+
+Polylith deployments require a reverse proxy in order to ensure that requests are
+sent to the correct endpoint. You must ensure that a suitable reverse proxy is installed
+and configured.
+
+A [sample configuration file](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
+is provided for [NGINX](https://www.nginx.com).
diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md
new file mode 100644
index 000000000..868aba6ec
--- /dev/null
+++ b/docs/installation/7_configuration.md
@@ -0,0 +1,145 @@
+---
+title: Populate the configuration
+parent: Installation
+nav_order: 7
+permalink: /installation/configuration
+---
+
+# Populate the configuration
+
+The configuration file is used to configure Dendrite. A sample configuration file,
+called [`dendrite-config.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-config.yaml),
+is present in the top level of the Dendrite repository.
+
+You will need to duplicate this file, calling it `dendrite.yaml` for example, and then
+tailor it to your installation. At a minimum, you will need to populate the following
+sections:
+
+## Server name
+
+First of all, you will need to configure the server name of your Matrix homeserver.
+This must match the domain name that you have selected whilst [configuring the domain
+name delegation](domainname).
+
+In the `global` section, set the `server_name` to your delegated domain name:
+
+```yaml
+global:
+  # ...
+  server_name: example.com
+```
+
+## Server signing keys
+
+Next, you should tell Dendrite where to find your [server signing keys](signingkeys).
+
+In the `global` section, set the `private_key` to the path to your server signing key:
+
+```yaml
+global:
+  # ...
+  private_key: /path/to/matrix_key.pem
+```
+
+## JetStream configuration
+
+Monolith deployments can use the built-in NATS Server rather than running a standalone
+server. If you are building a polylith deployment, or you want to use a standalone NATS
+Server anyway, you can also configure that too.
+
+### Built-in NATS Server (monolith only)
+
+In the `global` section, under the `jetstream` key, ensure that no server addresses are
+configured and set a `storage_path` to a persistent folder on the filesystem:
+
+```yaml
+global:
+  # ...
+  jetstream:
+    in_memory: false
+    storage_path: /path/to/storage/folder
+    topic_prefix: Dendrite
+```
+
+### Standalone NATS Server (monolith and polylith)
+
+To use a standalone NATS Server instance, you will need to configure `addresses` field
+to point to the port that your NATS Server is listening on:
+
+```yaml
+global:
+  # ...
+  jetstream:
+    addresses:
+      - localhost:4222
+    topic_prefix: Dendrite
+```
+
+You do not need to configure the `storage_path` when using a standalone NATS Server instance.
+In the case that you are connecting to a multi-node NATS cluster, you can configure more than
+one address in the `addresses` field.
+
+## Database connections
+
+Configuring database connections varies based on the [database configuration](database)
+that you chose.
+
+### Global connection pool (monolith with a single PostgreSQL database only)
+
+If you are running a monolith deployment and want to use a single connection pool to a
+single PostgreSQL database, then you must uncomment and configure the `database` section
+within the `global` section:
+
+```yaml
+global:
+  # ...
+  database:
+    connection_string: postgres://user:pass@hostname/database?sslmode=disable
+    max_open_conns: 100
+    max_idle_conns: 5
+    conn_max_lifetime: -1
+```
+
+**You must then remove or comment out** the `database` sections from other areas of the
+configuration file, e.g. under the `app_service_api`, `federation_api`, `key_server`,
+`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks, otherwise these will
+override the `global` database configuration.
+
+### Per-component connections (all other configurations)
+
+If you are building a polylith deployment, are using SQLite databases or separate PostgreSQL
+databases per component, then you must instead configure the `database` sections under each
+of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`,
+`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks.
+
+For example, with PostgreSQL:
+
+```yaml
+room_server:
+  # ...
+  database:
+    connection_string: postgres://user:pass@hostname/dendrite_component?sslmode=disable
+    max_open_conns: 10
+    max_idle_conns: 2
+    conn_max_lifetime: -1
+```
+
+... or with SQLite:
+
+```yaml
+room_server:
+  # ...
+  database:
+    connection_string: file:roomserver.db
+    max_open_conns: 10
+    max_idle_conns: 2
+    conn_max_lifetime: -1
+```
+
+## Other sections
+
+There are other options which may be useful so review them all. In particular, if you are
+trying to federate from your Dendrite instance into public rooms then configuring the
+`key_perspectives` (like `matrix.org` in the sample) can help to improve reliability
+considerably by allowing your homeserver to fetch public keys for dead homeservers from
+another living server.
diff --git a/docs/installation/8_starting_monolith.md b/docs/installation/8_starting_monolith.md
new file mode 100644
index 000000000..e0e7309d2
--- /dev/null
+++ b/docs/installation/8_starting_monolith.md
@@ -0,0 +1,41 @@
+---
+title: Starting the monolith
+parent: Installation
+has_toc: true
+nav_order: 9
+permalink: /installation/start/monolith
+---
+
+# Starting the monolith
+
+Once you have completed all of the preparation and installation steps,
+you can start your Dendrite monolith deployment by starting the `dendrite-monolith-server`:
+
+```bash
+./dendrite-monolith-server -config /path/to/dendrite.yaml
+```
+
+If you want to change the addresses or ports that Dendrite listens on, you
+can use the `-http-bind-address` and `-https-bind-address` command line arguments:
+
+```bash
+./dendrite-monolith-server -config /path/to/dendrite.yaml \
+    -http-bind-address 1.2.3.4:12345 \
+    -https-bind-address 1.2.3.4:54321
+```
+
+## Running under systemd
+
+A common deployment pattern is to run the monolith under systemd. For this, you
+will need to create a service unit file. An example service unit file is available
+in the [GitHub repository](https://github.com/matrix-org/dendrite/blob/main/docs/systemd/monolith-example.service).
+
+Once you have installed the service unit, you can notify systemd, enable and start
+the service:
+
+```bash
+systemctl daemon-reload
+systemctl enable dendrite
+systemctl start dendrite
+journalctl -fu dendrite
+```
diff --git a/docs/installation/9_starting_polylith.md b/docs/installation/9_starting_polylith.md
new file mode 100644
index 000000000..228e52e85
--- /dev/null
+++ b/docs/installation/9_starting_polylith.md
@@ -0,0 +1,73 @@
+---
+title: Starting the polylith
+parent: Installation
+has_toc: true
+nav_order: 9
+permalink: /installation/start/polylith
+---
+
+# Starting the polylith
+
+Once you have completed all of the preparation and installation steps,
+you can start your Dendrite polylith deployment by starting the various components
+using the `dendrite-polylith-multi` personalities.
+
+## Start the reverse proxy
+
+Ensure that your reverse proxy is started and is proxying the correct
+endpoints to the correct components. Software such as [NGINX](https://www.nginx.com) or
+[HAProxy](http://www.haproxy.org) can be used for this purpose. A [sample configuration
+for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
+is provided.
+
+## Starting the components
+
+Each component must be started individually:
+
+### Client API
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml clientapi
+```
+
+### Sync API
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml syncapi
+```
+
+### Media API
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml mediaapi
+```
+
+### Federation API
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml federationapi
+```
+
+### Roomserver
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml roomserver
+```
+
+### Appservice API
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml appservice
+```
+
+### User API
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml userapi
+```
+
+### Key server
+
+```bash
+./dendrite-polylith-multi -config /path/to/dendrite.yaml keyserver
+```
diff --git a/docs/p2p.md b/docs/other/p2p.md
similarity index 71%
rename from docs/p2p.md
rename to docs/other/p2p.md
index 4e9a50524..9f104f025 100644
--- a/docs/p2p.md
+++ b/docs/other/p2p.md
@@ -1,27 +1,34 @@
-## Peer-to-peer Matrix
+---
+title: P2P Matrix
+nav_exclude: true
+---
+
+# P2P Matrix
 
 These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up.
 
-### Dendrite
+## Dendrite
 
-#### Build
+### Build
 
 - The `main` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
 - Build it and copy assets to riot-web.
+
 ```
-$ ./build-dendritejs.sh
-$ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm
+./build-dendritejs.sh
+cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm
 ```
 
-#### Test
+### Test
 
 To check that the Dendrite side is working well as Wasm, you can run the
 Wasm-specific tests:
+
 ```
-$ ./test-dendritejs.sh
+./test-dendritejs.sh
 ```
 
-### Rendezvous
+## Rendezvous
 
 This is how peers discover each other and communicate.
 
@@ -29,18 +36,18 @@ By default, Dendrite uses the Matrix-hosted websocket star relay server at TODO
 This is currently hard-coded in `./cmd/dendritejs/main.go` - you can also use a local one if you run your own relay:
 
 ```
-$ npm install --global libp2p-websocket-star-rendezvous
-$ rendezvous --port=9090 --host=127.0.0.1
+npm install --global libp2p-websocket-star-rendezvous
+rendezvous --port=9090 --host=127.0.0.1
 ```
 
 Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`.
 
-### Riot-web
+## Riot-web
 
 You need to check out this repo:
 
 ```
-$ git clone git@github.com:matrix-org/go-http-js-libp2p.git
+git clone git@github.com:matrix-org/go-http-js-libp2p.git
 ```
 
 Make sure to `yarn install` in the repo. Then:
@@ -53,26 +60,30 @@ if (!global.fs && global.require) {
     global.fs = require("fs");
 }
 ```
-- Add the diff at https://github.com/vector-im/riot-web/compare/matthew/p2p?expand=1 - ignore the `package.json` stuff.
+
+- Add the diff at <https://github.com/vector-im/riot-web/compare/matthew/p2p?expand=1> - ignore the `package.json` stuff.
 - Add the following symlinks: they HAVE to be symlinks as the diff in `webpack.config.js` references specific paths.
+
 ```
-$ cd node_modules
-$ ln -s ../../go-http-js-libp2p
+cd node_modules
+ln -s ../../go-http-js-libp2p
 ```
 
 NB: If you don't run the server with `yarn start` you need to make sure your server is sending the header `Service-Worker-Allowed: /`.
 
 TODO: Make a Docker image with all of this in it and a volume mount for `dendrite.wasm`.
 
-### Running
+## Running
 
 You need a Chrome and a Firefox running to test locally as service workers don't work in incognito tabs.
+
 - For Chrome, use `chrome://serviceworker-internals/` to unregister/see logs.
 - For Firefox, use `about:debugging#/runtime/this-firefox` to unregister. Use the console window to see logs.
 
 Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and register with `http://localhost:8080` as your HS URL.
 
 You can:
- - join rooms by room alias e.g `/join #foo:bar`.
- - invite specific users to a room.
- - explore the published room list. All members of the room can re-publish aliases (unlike Synapse).
+
+- join rooms by room alias e.g `/join #foo:bar`.
+- invite specific users to a room.
+- explore the published room list. All members of the room can re-publish aliases (unlike Synapse).
diff --git a/docs/other/peeking.md b/docs/other/peeking.md
new file mode 100644
index 000000000..c4ae89811
--- /dev/null
+++ b/docs/other/peeking.md
@@ -0,0 +1,33 @@
+---
+nav_exclude: true
+---
+
+## Peeking
+
+Local peeking is implemented as per [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753).
+
+Implementationwise, this means:
+
+* Users call `/peek` and `/unpeek` on the clientapi from a given device.
+* The clientapi delegates these via HTTP to the roomserver, which coordinates peeking in general for a given room
+* The roomserver writes an NewPeek event into the kafka log headed to the syncserver
+* The syncserver tracks the existence of the local peek in the syncapi_peeks table in its DB, and then starts waking up the peeking devices for the room in question, putting it in the `peek` section of the /sync response.
+
+Peeking over federation is implemented as per [MSC2444](https://github.com/matrix-org/matrix-doc/pull/2444).
+
+For requests to peek our rooms ("inbound peeks"):
+
+* Remote servers call `/peek` on federationapi
+  * The federationapi queries the federationsender to check if this is renewing an inbound peek or not.
+  * If not, it hits the PerformInboundPeek on the roomserver to ask it for the current state of the room.
+  * The roomserver atomically (in theory) adds a NewInboundPeek to its kafka stream to tell the federationserver to start peeking.
+  * The federationsender receives the event, tracks the inbound peek in the federationsender_inbound_peeks table, and starts sending events to the peeking server.
+  * The federationsender evicts stale inbound peeks which haven't been renewed.
+
+For peeking into other server's rooms ("outbound peeks"):
+
+* The `roomserver` will kick the `federationsender` much as it does for a federated `/join` in order to trigger a federated outbound `/peek`
+* The `federationsender` tracks the existence of the outbound peek in in its federationsender_outbound_peeks table.
+* The `federationsender` regularly renews the remote peek as long as there are still peeking devices syncing for it.
+* TBD: how do we tell if there are no devices currently syncing for a given peeked room?  The syncserver needs to tell the roomserver
+    somehow who then needs to warn the federationsender.
diff --git a/docs/peeking.md b/docs/peeking.md
deleted file mode 100644
index 60f359072..000000000
--- a/docs/peeking.md
+++ /dev/null
@@ -1,26 +0,0 @@
-## Peeking
-
-Local peeking is implemented as per [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753).
-
-Implementationwise, this means:
- * Users call `/peek` and `/unpeek` on the clientapi from a given device.
- * The clientapi delegates these via HTTP to the roomserver, which coordinates peeking in general for a given room
- * The roomserver writes an NewPeek event into the kafka log headed to the syncserver
- * The syncserver tracks the existence of the local peek in the syncapi_peeks table in its DB, and then starts waking up the peeking devices for the room in question, putting it in the `peek` section of the /sync response.
-
-Peeking over federation is implemented as per [MSC2444](https://github.com/matrix-org/matrix-doc/pull/2444).
-
-For requests to peek our rooms ("inbound peeks"):
- * Remote servers call `/peek` on federationapi
-   * The federationapi queries the federationsender to check if this is renewing an inbound peek or not.
-   * If not, it hits the PerformInboundPeek on the roomserver to ask it for the current state of the room.
-   * The roomserver atomically (in theory) adds a NewInboundPeek to its kafka stream to tell the federationserver to start peeking.
-   * The federationsender receives the event, tracks the inbound peek in the federationsender_inbound_peeks table, and starts sending events to the peeking server.
-   * The federationsender evicts stale inbound peeks which haven't been renewed.
-
-For peeking into other server's rooms ("outbound peeks"):
- * The `roomserver` will kick the `federationsender` much as it does for a federated `/join` in order to trigger a federated outbound `/peek`
- * The `federationsender` tracks the existence of the outbound peek in in its federationsender_outbound_peeks table.
- * The `federationsender` regularly renews the remote peek as long as there are still peeking devices syncing for it.
-  * TBD: how do we tell if there are no devices currently syncing for a given peeked room?  The syncserver needs to tell the roomserver
-    somehow who then needs to warn the federationsender.
\ No newline at end of file
diff --git a/docs/serverkeyformat.md b/docs/serverkeyformat.md
deleted file mode 100644
index feda93454..000000000
--- a/docs/serverkeyformat.md
+++ /dev/null
@@ -1,29 +0,0 @@
-# Server Key Format
-
-Dendrite stores the server signing key in the PEM format with the following structure.
-
-```
------BEGIN MATRIX PRIVATE KEY-----
-Key-ID: ed25519:<Key Handle>
-
-<Base64 Encoded Key Data>
------END MATRIX PRIVATE KEY-----
-```
-
-## Converting Synapse Keys
-
-If you have signing keys from a previous synapse server, you should ideally configure them as `old_private_keys` in your Dendrite config file. Synapse stores signing keys in the following format.
-
-```
-ed25519 <Key Handle> <Base64 Encoded Key Data>
-```
-
-To convert this key to Dendrite's PEM format, use the following template. **It is important to include the equals sign, as the key data needs to be padded to 32 bytes.**
-
-```
------BEGIN MATRIX PRIVATE KEY-----
-Key-ID: ed25519:<Key Handle>
-
-<Base64 Encoded Key Data>=
------END MATRIX PRIVATE KEY-----
-```
\ No newline at end of file
diff --git a/docs/sytest.md b/docs/sytest.md
index 0d42013ec..3cfb99e60 100644
--- a/docs/sytest.md
+++ b/docs/sytest.md
@@ -1,3 +1,9 @@
+---
+title: SyTest
+parent: Development
+permalink: /development/sytest
+---
+
 # SyTest
 
 Dendrite uses [SyTest](https://github.com/matrix-org/sytest) for its
@@ -43,6 +49,7 @@ source code. The test results TAP file and homeserver logging output will go to
 add any tests to `sytest-whitelist`.
 
 When debugging, the following Docker `run` options may also be useful:
+
 * `-v /path/to/sytest/:/sytest/`: Use your local SyTest repository at
   `/path/to/sytest` instead of pulling from GitHub. This is useful when you want
   to speed things up or make modifications to SyTest.
@@ -58,6 +65,7 @@ When debugging, the following Docker `run` options may also be useful:
 
 The docker command also supports a single positional argument for the test file to
 run, so you can run a single `.pl` file rather than the whole test suite. For example:
+
 ```
 docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest"
 -v "/Users/kegan/github/dendrite:/src" -v "/Users/kegan/logs:/logs"
@@ -118,7 +126,7 @@ POSTGRES=1 ./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrit
 where `tee` lets you see the results while they're being piped to the file, and
 `POSTGRES=1` enables testing with PostgeSQL. If the `POSTGRES` environment
 variable is not set or is set to 0, SyTest will fall back to SQLite 3. For more
-flags and options, see https://github.com/matrix-org/sytest#running.
+flags and options, see <https://github.com/matrix-org/sytest#running>.
 
 Once the tests are complete, run the helper script to see if you need to add
 any newly passing test names to `sytest-whitelist` in the project's root
diff --git a/docs/tracing/jaeger.png b/docs/tracing/jaeger.png
deleted file mode 100644
index 8b1e61feb66ea4c58fdad1b1a881ef12817bfae7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 264127
zcmeFYcT^MY);^3B5vBMbih>jiO*$w@M^q3(QBXQWKzi>G2t`y>1f=&SO{CY*5<pat
z(4>VPkP;w3fP|J1_$Hq7mi2vWz0dEQ_pfiQU)BmUOfvV(+<V{G-uv2D-aXdUU^>fj
zmWqmsN%PSIePBbNqM{i*a~e3(<Hq@bii+`t<Nf=OHSgc&eeC6K=jdWfMfK=ik`cYJ
z!Ou(CX8M{A;g1Vm%|3|ZP}Mpk`ZDb4y{Brdj|3(8X)h|=DxI5r$cB7h@xF0o?QFN{
z`RW$IJ%sV&Cu84VG9}4gJ7ob~3c`?<`*Wo2j}4@}ag0>*8^LKJETeS1U{l`M*fUnq
zIyyFS+|<;A^lWoyrVpuKf4IQT&XmhYI7yT>I`wHyyy;6Lg6E|2lQ^3YEfw$TdHb^$
zUk1O;plXkzlL|aV_3iOzSeibsftBR9(_WW(zezv6YZjeS;!(umz5&zxfKi*KqDuRL
zLDrrcbPI<V=J9gwIH<fD*SVGW=+vNO*xkpx%jd4?z6x*%Ilg%@o2L3kUCO;y8f(90
zqs)nuxq+*T7<?^#wR~mBwMZ?t8pY$fliJ%SW;^&q8s&AX{C7)n+h2~Xm?|;4@Aa)X
zzF7?QrW!vhzZN3!*B2}^mDWY!8wn$OpVL%3UhmRqznyt?G${DCkC!=R`TGNBLYhgu
z2lqFR`dkA(y4DAM)32{TG=BNU8y>75yiV00?=6?!R39AJ22O%TNy)y;pKlYpw0-&j
z%>M3z0dEm?*pqX7KPAsQ-*?IF)_JQx^?dB1wOemkwE30F_vlO34Uahf`n23NRJ%C8
z%@JgRu;vAqJrJ4y?%?q-{H~<F1&>Q{7@DU|@|*8<ApyrQje^UZN*|*hvvCylb2(qs
zOt^h$?QctWZ9Bbbrl$37w36<nrH7vhW9Kt&oxO0|dpnNK!}H7Gwu8w%^H1`+iZ>!v
zZs~HdSf2TGiv1P!rN>{B8TiAm@;YvO!M$u7xiKHcvdGK%Hmp=#TH;1w=!3I447q2H
z=;F=@p2oZ;#J*R(HWJD)@+GH&w=}e_DpXx8fYwv_Ep^})tR^(sXV6S4;LM$PT<f{#
zWr0;CKM*L-Tz=F1C(K(lKBq<AS$%tKeN!@X<yFS-@J1DGrkswdrUjR2f8I&)y;mBq
zWWGYbUQ|<<|M5Vx`Y6L#J;JY>r{d>bJobZUQ1|NV$hlJ=xuh1fr`Z~RXAiQ_v%JiP
z=O&lFGfPsyq8H^-JB;q2#-B_J3Vwgfs_l8?SJY73fZ~n%@(GO6;BsDm{;VnD2AYTo
zd1ilYJs(m3?UFUADq;M|`fb1SrSu}}XU4*^I+;th<~ShZZEsGue&6p(I2U*meYoHp
z{N+}ON2w>)hxGXh4c(^_x-U#WvTkPZr@V+dXMFYG;`!<T{f??U5r0{KS*oXKwq577
zi?t_a;SvMEzb>|&=2aa>q5QX^#5gur(U;m?k5{7l=Ou!<E}!!FqMe(_{>8U(?OvEW
z4Oe5nb{Jh%I9kMLj=42l<;-cT=WK83-tk`44)eX#xhq0%(9S5qo){)o#aMP4+s<D_
zw@%d-2G8ZbN~_Vq;Bm>E%BWqmF+{#ydXQ;0%#!xrqcdDqy!>zZtRowFH_vb~Jox(2
z;U>4{L#dB058eyLUFlXVU^EFYyDxKNw7atV)S{plhvR)W-u;idi+nq$zCY%03QN!5
zTk%MY_8Jtt{TElPU4Fu^>psfOwB@R$gBA{45H_VK%X?G5>>XH6ONf4WXJS=xnRCIq
z+L$Bx_0MV(V<E$sXIAybH;#(Mtu~C=_qYXqGiJVS`FiK-JArxXU@7Hu;189Z{&T(o
z=Rz3TBK;R~7<^t{>3BV-G56OG)*1$bGkmYSzRG+J`Rdu2A%DjG?StzHE2@O>7RjX4
z`<c4t+S97jIxg26{^B_w^yp?h!_U`js_*%>HR%dihfIgWhW;9Qs87GjWfJA0=_e4G
zFx;*0CE1kOnh=~C9CwsD4f<6${^Uron(dk`hpmvUaABNR_Rzi6!nG${pHGcEaANiq
ztAA+<QqqO%Knn{CZOfNR@2G=5-O(sjbB-MoDi1QnITCN%bEG8e8Vn^5HVmgb$U0Q6
z_%tgirDm&(=ZfbU4afe%4WS*S>mnWTb+0FD>xJvFb>dDcsJxZ&RSzHj6}Htx?E0`=
z<!~YBS3PQJZO7Ms)o_h_5MDHu-!;gY$6lIJ2{U#to^^x^5N|?*$P&=J<6yEVv5UYX
zA@PPO0(o9yS+YSYMp~Zn?P=!IiV;^Mq$0qKrcBLDyG%+<2~5Qth8)~dF80>iLHx04
zUFJR0$kv{t=P%s8NwQuMmHKk0*?}+?@Yb`cy1Q{C0JeQ`frVcrK_bCCVTpg2-(TC)
zz<yY=$Us{*om+N;!&>57=-cOyRzJe;yQSyZ_1kUR4fi#qW=UfuVG^+tDQT5=RP1JL
zJL=9fbXObOdX9e@j<>qvAXIH`8~ANU2>J25)l&5>W6?4<Vd>IG!iA%DrG}-J&h*%u
zSZnOen7_+s8J7%^jAt@^GBXXy4f(G2<74Bqb@}7f8{hn#*L5f1_5M|Q)h8W@+2eN+
z)ieWn-FZLe8*a=<c6-*?BrVtqe7Z0tgmR1+rn4WJK$@(ZuLjT3`EL8gHM)2>a%x>(
zQ+O(0=d^OiC|$U#ez5+>?ySUe)r!^k&b3G-9ybL=*MpsTfyhulycg1YYR%92)$!XW
zH{NU9jtN-ZTo?1G^l4kaw|{0ob>r9E_SfO>28&I<@XhR<&UGjj%RbATR%q+eZ2fGO
za)a^}Wh}I!nrYl~{rkEBf);%qJ%ONyqDg_Iiw7IKq$%{M3M#btBq5IZ#={%1nBeoz
zx!PIAuMTo|pY1(!Bvhr;ed{gH(~CYT<NU^#TSX5fyWS;qPJ9pK{(b2>%LLC(GPW!0
zr|a48YB$tC_su#9o$+7CUe7SWm<O0E8E`RM?^C&dv1z{j@ZOv|FxtNV!q%6-nfze&
zzN@3G52Li+nzH+<Ov;&Z&)ntUQgR~n{DRr6TNg;~R`Vvaw@B^jX5OD(Y(#gR@A?*9
zauqKa^5E#hK?1{vSML}UF3jG1b5l&GHEA)mK-)=oOjB2{LN_FXHNEC~A#{DKTDmJ+
z=Z;QNflNW%Q2cO1;qHfXoG^&FMzaw_lQ_D~yigpYqSWD|Hv8)S1z-LhflVaui-H&H
zy$9W~OGev4Qh{k$=@vTk4vr3Kt16GADg!oN(`%5-iy!N+V>SI$_j=ZKJwG;@PxH-d
zjcHmgK3I0?3C!TkXi<WRi}ShrE6b_eI0<_SP8W7fdm-t#Rv?oWRrX*^dBS~X?dQyn
z;GqoNr^}TNf)Zho``%g_OVms7ukBxZzWN))7q4f0k$on=JAE7D7*(Vsc2=AiD6ecG
z0S|mL`k_gBvv09?TJc<3l3ZzEL*M#gt-|ty-U-PRd0GEWp6_noh2Mh*KxwGE=-}hb
zLzYVqL_Q1s5{pYQCv}`e>YUqt9sjyKO7T*;1)=GwF{G%sysUM7!MfwOv`WlD_3thO
zp9#n`1tLDv_@Pm|d3|jscS*r;rCiW-`}xvJl5eSTsg@DkQpt07)oWl!$N95!qVuHi
zCt;4U$qI{xy#@L0X5pl3NiC?1a&X=8#_9PtQJfL7V{TIDrwyx}dht0G?lI%$Mz-K*
zh}VsU9?1OipJsZ;y~j<EjMau3*PI3r42FA){-wWRU;!U?!3=TKshbQp2}4n@!D^w7
zpMA?`S~HqjTv*2Qr||^T2XN(P^yE})EWC10wgo+gvtJi4zrR*A>WhlPPI2mUPBMRx
zADmu6H=gWUwYjS}_*U(FoX;zaEQ!SMY%1vc6clTYjriUstwb!p>x0EaEBQ6qYJ;%b
zpBIXCU*+8Oi}&8?E(tTB*CS;qd%S@b9hdD-G<#cO_D<$^yN(TUW6gMF45!$ErGpFS
zgx`Vn;P?C#HSw&*-j?7!t=^=DQca^uFsN-T#3F$FGrE;i-0jlrI{XesX5Md=Yer$o
zzi-o!yh^}q`r-EO@2lamT5>#2#%MYs>{%|dsQC4UJ_vn!Jig<(dtqOIUZGP#T#;+P
zZXs>m#NFs87!q0@x_NK}Yl}EhMkzJ4EWAE?h(>;|)V6dD-8gJ+5~#pzW*o`F-9hNI
z)~W4m^6_G1B1>#gaqvxO*})>AWZQcLGX*__LZuLy=uo!y=l5%Cr~;a}s35nfKAhV+
zlF)@>u<Q?@dK(&ZDptGjDsbzm&#$xA8v9+TpR>5^dGq>9P<_dVSWBoXDW1+yXEWj6
zbQ0re-9A6Ny9<}14;wktE)WO?<K7?;dRsB?XQ*-=Ylsyktk+vPj|!TxFFA#~22P>h
z61j=bV$gf47sCY(8cx`sr@ymTqd!;z%8vK8#+r6II#joS{WDawr#PtSfW1?|Tlo~{
zf9|WF5~QN}bDWxrD$<dP_MhL;1-6veN8n94=O0^|_%~Gaz<;d3J0O?(KfcXG$ffy@
zeVRewGpc(A_cb+vt$~e~t*x84gS*dR+`SvX0S1ppCf-z3m-r~}Q=0lWw}A0cj!%t!
zjCHi-ZQNbOtX{ZV+lmFadQirpQVfs>c3o|Ktat-lUEI9o1C*}+`G!2OPdO}po%hdI
ze4Lf88|ysgz3=X2%X>%cwwT0q<+HrJyoz2g?Bw+ysQ>eH;6J774n95}^5WwD{{CYA
z(qisj_TrLqa&qDlQsPolqQEyqy#w8RtO7*cyl?ztkpCFxfvvZVm!pS|qq`d~Wn3$3
zcV8c+>(?n4`p?%t?$b8F@xQL*=Kasd0v=GD@`<>ln1uL$jt!iuNI5G1*fGG?#rT1v
zD==rkHI$|A$Vw^xIpP2F(SKd?AE!R^w)ML2?h2ggqx@g9{?C*D`-lJU6aQS(<iD;d
zCoA*cFZtg-`RAF6;*>}KZ&UG)dH!=0m}up*isJuiH084m8PE>Ej#nKY=sg9tfR|BT
zr&fSJg8$e8`=>CK=k0uQsHjw_G#}i18gOcThN0ER*Ft5B=6$U2OIpQ}GlCA6O;Sq|
z)6Us#a>Ty+6#lU#?}=Ly=kJ$7*MG-z@`^qBTu_)N<W$l`YxnMwDepVMi|@~dzYkRw
z2*sbw;4_>2m;0xK>^Gq&mg3oI3f^P;a^B0C^CY=8SHIanOwMhNQ`EF)&hx5<QT-b`
zRiCJ*&JfB!SH8ROZ;k+5H{4J)cOd59yMce4Wq40D*ZtzhzL)>bGyL<yX9yGY|Monn
zhPB^1155;Ujmf_|m6Y2ES0?>0p5VMO-KlWIGgX;${|k%q4%+kl`=&faUCPTl2s_(t
zckkcwr+?Ujmbgyy@B5_c^NX~!M8i`PBL53>hN<R2{})eirtD>yD*6_02Hn50sQ&4Y
z=g+UyvYa=jr_PXdcVm`c$+c@E?>X<y2H||G)D6Qaj`HK;-IF7HhWCIHOpZ(#&H#xu
zy_o*Rr1vl1`F~pNIUik&;!d|rFDxD};Y#tNRPpdvMBTgduMSNeK%jqd<Zvd6JDifK
zd1l^5Es*6il&gW*fT0l+o?ToSIvFzM=!*?@6jMt)o%>Q+;Uw0wUbDP|6nBr*jD^Mq
z_S(E*#xecN-fRp7*1@Q(tOm8gUxG?_*OrZjKDU+){%+Ze&n{d8*6Pu@o8$h!9<tv-
znS1Nx7b6bQ*<=6elkM^WPljkHuAkujW1;@L3m^Pk{n1c=D+E)`OgTSbAN7lmKI6Wg
z6L+CU*Vux*fAXb#kpD9OFT3S)Iy%6i&!Y+$$`Vyqv)m)O{2cryY*UibqgPph#oB)L
z+|6(Phf68gLiZS0-a~=^@Sy**h5Ve%06<rpt5dX0Si7b`*2p@1e>QsdP^SGy!+XFu
z#zm^Dg}5X9fIBxnwM)eNmimo<?E(PAJDxp5&@mXUJ5v_ljy@ryVcyLfuI?>b-nTjb
zIGILxC<zwo%uaIiZ<$gQL)h~Fi<f9`zHolU%s*q~tSc>G{phNs@*98!_}R~|WK0jT
z{i|~>p#u!!=#+jb{241SE}ug!#cwjtogwrco~}Fi7tyIq0T|5}rdd9y4y@{_GiDR=
zfZd*Z0Y`r_>HJyB|H=&xvjGoPROI-VF&{9(vzIcIIqYo$5GT`JHR@kn{r`93{_n*7
zEl&LZ$m2MO0xE}^_ywMmMy{=96nt~&M(@OV3eYE<G)&lKEX$*s`ZGo0-}zLZ*HaP&
z&}6VvIG9)nb~;g{L{pxlIr?3*>^})@c+;iE-E#LC7a;WZT{JTPm549%G+eO{-LuaS
zHEXzy$?~_v$HfM}$b7=|e-IlT=(M6|VTj%G<qY%@|3zxr<)_a#si8u_T48+j*{n|K
z2Bb9eb+%&ij`)dJi%Sl9-9F=-c&o?HrPg23ZX?qt$NTP)p!q#^#MW!N1;wp!78f^<
z2+lgX8J>f7`b({t*d>NU=t%<%zOlnE@!MEPU)w<>yxOHR(CLI&c_M%J-bCmLnG$&W
zhJTAUl}*rx5b}BIJ7*TS_l>;{Mmgqsge$Mo3Uy`KH~jLOa+Te5z`7mq9B)PTJu|Wx
ztw2S)2QScJQwD65Q=sLq6l??+Cj{yw!NRYX0ob0A;cq`%;oG0GH|5^8%pNk&o<23R
z>DPC1j60D5AN^*Z!sqG*|Gv-{N=Aj2&rRF=59vYI3^G&>6I2}8w_aV)b%>E$PMxqA
zXR3SAwjb1n+AM{&E{SGv`HkltZu+-PWeY&pZFroIe~TYm(`Xt$qt@{r2%4>J>g3uS
z4B_9K_8wS{2Z=czujU+=$YC<Q)Ad4$!!Ns~s}e@qBp201?K3@pFp7+xh<*({>I%JW
zf)GJfv~0@nc`eB%WXuwZHUjj5CWq7gTNk)&^g7S{Ey}kO{}MEbD*H8Ip96ofVf8hb
zhd3B%8n&n;g%}h6$SIyeoEV``N}oK=gKItCrCYu==#NoAHyxI?5lanMU!D_l*n!Dd
zgsYHhRh*fX2!no~@-JDb?8L>o#Hbt}_+`e19xe2B0`pIIZg;~yG-zudNRXrgHh=EW
z843@kT*nRB3EehKcC6M2M|}4&+=N4NuK0^7Y}6T;87plDU|kO~B`kqVVZ&)=x5D4G
zB9x3}G2H$gCC<J2!fVPE<|^YpJH6g8<>t-)gR87e07#eEd6r(jxOHJ>?t<oDCyvEg
zW7A7w{#S0rJ>K-r#%C@sCg^d?_d=ixEXQ*!>Gmq5iH$WlakSFS^583b1?pK=)Gp+Q
zboMXBz3N!5R~p1i#37KlYvkcbvGVAC1L{l}XZTTSV2@Do?ZZ4bV^E{`sqv$6IpOl@
zZ%dr$gCB1gIJrH){nZDu+==ht5BPS2TT&W6k)#T~<Tj0#^-|u;-fy+9vcy4gK^a{t
z$J-HY_?xmOS}gbzyvj-KA`9W1y7_mNWveSjah+v<7gF16=yp!#PUd-ztt5`$31p97
zg%W;6w2`t4q+haxY(;FC<(wR0=((hKnG~><<vvVbdC0$b?v&(#6e!7dvEq_Yt*-T<
zOezp~PfyK($}3dS`f9b?2vzjZ)V*AH$NZ$rYPs%SqmH!1wZcd4+CdxjM&x)5=Odt6
zZRS*fZ)ZW)O8WZTL-29@NB#bYTFqEr&JqSfU;kLhaK`M2;z0)sw`2H_!a5#!ZvF)a
z+Ohoh4tk#Ht3m74H)dAm)=Degf;e;z+{?|96=Wwhryik&4u8q1D5s3E2MgoM_ZDu3
zBSf$%ZAVM|vQnBsI|(_hxhW$#hiN&E{R+6&J$SEubq0MMO?ih3zUqT>yLIy4`TTPe
z>XHd9vZ`a#JU)c9VRXC>H5Ui6?a%kluV80hx8A)^O%0W<leDvXLQAx>5*C3oQbR$U
z3<OcvI@bF~s8F&I1)Bp(HNgS%um4~w0)g*Wkd<S=Hc(tZ2ZESfaUD@bGb^lD<wmA8
zcHM;svSF>$tcHc78@2(&E^6shIsB<76NH2BKmWvY*sU?Cc2pU>B;2Rt=9RFl>)&j^
zIft~bsIynbr}Y5|?bBoP{a9*fZS}iY0AH3pV}qujTUE5~^ekK8AVxv<MFi>~ps#`W
z1CA6}2O@;;=}!2{Lvbg|-7iK=$9u2ZA?F{(2Ir8bG3i*~tc|tBOd+Mu+0ea-K00>u
z!99my3KGd3ZFNp%|B@Pv1%Q<wUP30dosiO53+$VB(xwId8K$87GgI8GFdz<vL;GF#
z$8BOe(9VZ1hE%~7{xxk-;4Wsr2y_!8*VZ%q5%?^W*Rvpu5({#?WzO>E-(|$of?Kb}
zgRM%LPjdr^Ez5Iq&=o1R0bwMRKh1ijiXNQl=9Iq9J1BdD)7tRT{akm$hHt--*9KvY
zdb1Z-%qAV~L>{aewM`yjLXYhgkl>^5U>Cn>&#uu-vU#5~U=UaLps8}*-^lm;joSk!
zM+4_xLK&{j{JsPej0l+K1<XD?H7NAugaU3Iw6OqyEq@me*`CAg-0X;x9#E{E0P57K
z6a=AAy#6(5iM)F(I0)M_yQ`#`?K@@?@qCCc#e7JSkjJvY#6Lrt-FMSf-2QgY{fP>&
z3ev%cNnl6TK>$G5!-<hiw7bl5iyvirQ5J24H+w*62s(chgfyESQpB1JtT8uy33dOb
ziq1LunM2rf3mSQ%aR<2fDZ|k=ce{ksL}HPNWkzJxw4#X3HsR6V6@GRmbFv`|vXjho
z&B%ZsSL|==L532V#aWIot#CdTIV`B)oD`HDwq*TICCyhQMgwiiCqXy8bM|yetJjvW
z;1ftf*tcR8@+$wsFQ9t1j^&tDN>H1!Mns7t8vjJ8Hu6-Ph0{m_tbnc8?#d92>U+5I
zZi(KVlY`ik@tBNn1OTC@GW}-*UB^GUzZCj-Yj_-3dKn;bc%A0u<+@>AEN3fJw9|dE
zt|SXh!Q_VhQX1yFxuV@Vm7J7?kwN%a|MbF&nU;ghYz|BnO$p}aA%{zULUThHs_l4p
zD*MCET`iC}2b#R;pYbYD|5e7eTR<mg!|-L9W5I9)f#A`12gR7Y3ZztSBt`sBMf^ll
z2a}tG%cZ{kwi9yskG^w8V5lz%tc*43ue%(K3qm{d3{O3P*fp-|xKAQrnV~f-->z?P
z?>B0YQf*_rkCEE~O0$JspWRAP3Z0D3lr??2tM@7xc-j`L2(d%r=Ws;X{iETY8uC`8
zipx;OW6J^J^tzp_nM9jQXPNhGEObIPxi-}HSZ)*O-G&-V)b{g8_8_hdi`Zw@xRRz7
z@|nrr9N`G5Z3;tTf!Nuf!G-MaDSSTK@FWAJoQ^x^pJ2=S8-(1SD0lNM&AXtoUe%eN
z0$;b?Ac1yo`D}I2%fg!aT+=Q5CJM>f-GSJkS$sAh%`>?%3Oc(*M6_%+JL4%Ot<L6H
zf=zdoH-aFO^xVBa5_ESP7J`MPU*(j7w;f~K+>`;|7B#MH&9?0&F1N}L_XloNDNyaw
z5xv|p0&fZ?<9@8CtX^?04?>t$eZsE9%EDK5HV|>odiJf?*=zel;-D3E?5%Tjea@En
z^c~F0F7q2R+cEwwG)r<hKYjK0TC7l-6QSHdCfG215E*(9>37mZNnRJ_mw+XW$pGR^
zMjuAy<gon5z@1@b()#deCLl71%U+SiuR70Qa~Ts*do%uMzqbBEQ?*UNfFz<icx$o|
z6$izpw3FeRzLkE5-fkaxUN?(kdAexBq<du%+tJJUJmi%@Z0F{|F}EHs$$PJ)&RjSJ
zTH}7p9dMvYvH%YH`$CRGP<0v1yPtHt?0#J2KC)MT^t>5m8B{-jR0gwTawkY@@8L2#
zEq#7H+>nAUOXrl(J4gQX*-0a9ppQ59-~f_D^-<!(ULeFD=~-79O@a^Y&wPa5R^h+|
z%fkncm%ztMFL8xOcJVeKM4MN10`B$Erew5)M^QcCQ1zwBxI%ixxl<eXfaM@}<5HMG
zHO~^>m<Q$6+q>_umtfV^hfoDKI9~M*MtVwim0NXX(rOG(J#RRnAYol;g3SPc`=~wh
zIk|yAU6LuH6qI`#C3skksElrbV;#S_jNlGDlO?+v)_ELl%|`H|1Kzc-rm6B8x`U82
z$uZ>*$0&rv|CE&UMzjNXLA+=2$l^v^diYSDDjLFgj!?*d?aq>H+hN}n@%6}gf9xqz
zQ3|TTWn!qH|9bt~k}h4P;V$L9>bza;gpt!93?d|40yUt}5mm6$<sI;vFbJ-_Hrjkg
zsdHIDldQg+(VXxiC0_PXADLcxE4-nh!tc0SLs_a$#(w%}*vnb^O?ewCMZBtl8ah#J
zNnSIO71G{3XI*e+?QkG81W$3K```5+Cw;9)B!4ZBV3v1qh$DCdadDYJFF*~~s&X{X
zB3qp4)iW2fBlJlbh=;IrowE+1xc!hy*}M2GTyU}6PWKJiE#5NR$L#BE!Ebr?sv_?Q
z)rh*5R2U&WsG$$78+b5$gDGEIGZ_oIYb%$eGpv_sF>mVSQ&C*}r)m|Z=b7R2C6HF~
zKDv#`babH@));K41$uG_q%_mVX}@Pgf|T!f#?Ui(7u=N;1t-H=W;P}<hXM~j+h!Sb
zr(PdtoAEPCR(S~bybK!!@*|tf3r^Su+Ft9-uAv^{5-yWAanLHTO>s<}!n}rTVnQlf
z@9ZrAXhdtGwe8|!ZS*oX&9UK0!!yS;H(ibkW!MInwsoGU-P6W6ftl?K??22`J%6rt
z29pyS(o#u?Zb65TL^;lE#26fB+X3pD0X8H1P~8^thCBMw*_6OQ^OHZZ^M~&v@%g4#
zl9!xU4@E@L^bDtc%&0lUp{t;&kLLSu+TbDQNlIS4X|{o0g|EE55p$U~r-cT-568K3
z1F_y8?A8Wjm3f4e!!ihDO02xQRS92vR_SC(_{6^CI3<<CXcQWmxbJ0|HE8X|@dW0U
zD~^^l8FMpp;%ez61^I*r%kpPtCwriIPPTc(<-A@BFeDVfPslj5Q-*C@AW*o-5h6bY
zhn{TWj@^QW;~{oUr-kO*75NK>*}F9*R{cXT{*zln!j{|b_qyIe<O40=lz=NMBUl*~
z_r`59%m>yzU5$&TSTnWU?Ux#j+uEkF-4L=DSDJU>(OrF0M5s6OGt&@){O*WBr2Mi*
ziby_C8p#)=%nAeWqQR1NZ?A;FpOuU~$xPwj0?HY9DO40=xVjrK9{rNlPYexcOKMe$
zmm<3p^ppcT0$oTy^iF;<z4DqC3ty*Q@nVuLE5!6=V#ZvX(85hcLTIM_7BqQs&Ul}A
z^WG1YqfQl<`%Z@<JOOjzW-~XhW7aA3bNYH0($%;9E=un(PcCP|Ty_bch{t9|?ALp$
z#F}6T2S(JXGmgvhH%VF({7Yo{!ITBNn6xGN8peb!!XRl;Kqatr!J2x|c4|q3Fug9+
z0Vk+78iNE8mdQ(QIH_3AwuLT}ANGn1S0W_v04I*h#w8w(#WHj)LxeYEL-Dfo9UGpo
zLL2AuHMze7k0@1(h>z*-Xpg)u;OmNk1i}?=h;H_wf&R>C1$>QX<iqGf!SS^_r#sz-
zGEFNeMFHIRLz=Gqh?f1C8PhaPCz>B6_dcU+<pxsD5(>wJu{r69h~y*PHH|lPB>r;r
z<z$N<of19y)q;lmnO^r-MotxRS53XqxzWjlREajr_?1=MRxCh44&Uu*>j#H}NtOm{
z^ZEs*?<=t=EJo;~4lZ;iLjP=1<VA~=ZE0Ph57={rQMy6f%uSo3I2wCMnqaxu3(-nn
z9(*3>Sb~;PnJTzkvSk2sZ7?Nl6r;S~)PSk^esh?{Qq9(vs-CgUc>D`v!nXd(1e3++
zf*%dk_p#`(=4J7`)*-DqS0L`QjqKBH^t7_t_3rGx{)m)-#`@;EmjvESO|~8;yiZI%
zZ>r|uC{(65zyTE%;3<T)vY94(sG_St@P<rPF#0xD6%5;NLd@eQtInIfj~I^lVJ%8h
zecwDR{A|X{nk0U*BfbkVQyP$oCN)m^r2|khf<o-oH|(b<4nS@SmbhLGXEp!%JzFMC
zl}oBS(W{!!Yp?8Wa)CSwCXdQo9b!)`6-ZsRSM~*29p!WHekK;qVh*#rzgy3&4XUdN
z%`6yjtsr%79uxFV>bT+ifwGFmIRj%D1pEe=^m{nxn6OvG-Z{HZ44`H>#8AWQ=#^IP
z-4E3k>|B&YU04d{oo|(wNZ9u(E}A~&P`y{~4lQMRF|shmehej<E5T_8FL((l5yo&y
z7zXa;Wr4y`)V;&)n|V22@MmvIc<?k2Ap=-Tr0BkUjb$4|c_|RMnHqj=LkPEUrw>mH
zb{vXEFPu{(O$@ksxoCkd%XPts-cmhm7y57Im22ZDWvyX8AFrX?L7a1*`NGYxB=29e
zgygj>g}mWu#+xKeV>m(+VYG&O6CC4R41^PId+&vK@X0P17t{izvx|)eCzg^vbJ?=2
zb^|(zL#$!Fv2DA!+npiEpIZwd+?2YY;U4mXLKzvTM)WuhD<TeeV^=5aqM$|MjdL&k
zR6ZB0M6052PAMR3GJ)6}qMfs-K@veXP7PJxar(QycW;8K8!-z4uM-bFM7q-7X4qaj
z4R?g@hHh}8jt%IEUNp-iwxWjKNf0bhtRQR&9X3wa4N|I~a@nXsSpvK7pI%t{d9(i1
z!U(TO>Z;VSvB+SGY0k)Z(Pum=>-?yMG54F9D1YxG$em#u@Psln9MN1+#9bLOn-a=V
zER&x-COEK=CjVoxkSM>tnxVeKR^qQ?(Px`tF(%vH40WBMB{qjQ2R9DwI-H#_c%DRK
zzB($z*|e!6Y$p5KYW3NtI<|wtj6ni-eAQrHB)7aEM7MhNKEO{&IZS=G-5k%M9;##O
zu%++q5Qg7h)Ljr-E;W-KOySz9uY_*!dL;1n0qD3X!<40PWPc2F<v<GWv{m4=i3L6u
z-KNWNEfZb59Us|5m;3Sgx5E)n630SP1J4i&GbY#!i}@ZUU*FQF&49}0HLqtl$~OJf
znM{b<i_JMs_}z76WO^2a2fqwDWVAnGmBF6VWjaUj?F?SkEZmgi!)Ms#R*Hl+i}&wz
zT~|^lJg6-@vCjJ;6|!H=Ojv@`4p_Gi>nhYo*m0Ynb{?iu|MuDsYfMt0`&J>olJFul
z#@F#M-n`5M#^vxXl%IiMOL_sX*}#Ki>R0)03ZxbmMlCR=?ANX{h$fT8!+&Jn9u+-i
zXiiFoO6T=6W{{<=US-l2`l3`Gjxm%bCwtoDEcFDXxvf6)kel+G3`al$BVBPdV%(@4
zfbt<UII5|w-YePMm3~MDm__xVenSYmg;Pw^8SU8He1e%(8vVmw(>?wV#YR9Z;HJU;
z84E3UjVrQU%dCUC?TbL&<{VU~sTZ(%MM&vciXr4!C=`_!-I{Fkte+i3rGgGM$u=)5
zL}8_J%IwwD)4(B&-Ax<WL7OK0Q>PiFv+CprjZSwHDGaV_an@tcek2H2#Q@SCqH#%!
z>^iy7edm5ZX4s4QBa|OagLIw1dC%xq<nM50IG54RNs%*ofMl5EGU3q_`~cUoC5|##
zeIkT<y4qCcW-?3ooK4=blWj4XLK;k<YX)Z53}GCMv;JI(+qEA;_x(dr**42>M0a(}
zYI58sYHTH<H7jnA#>7>smgJ@UVLeUXC}onLrSDsTykbhZ8!@m+#)Y2H<;*O^svrWl
zqri&B>U#Vax_5(PtmMFAqL{A}GQ~L|#78-C)2#v|tx15z|G8|%5@eh<GADU$onW1k
zOf^me@Z5WW4YmxQmXB^L<=LI|N}hz#Fgs))d=p?!-k{JQK}UU<>uolR_-D%*wz`#}
zj&goQ_OLntX#gi`xt2GZP{}lGYl63NQ>$8n#W3bviWn~}ZV$#S!V)v3i$%w7m?^Yh
zKUr+4J8Hk$cDHE1KUkn}i6l5xTN#ZH?A>=FTV!~Em0?Ynx{3V8CLxS{O@VG}PfY=I
zbUp;8`a6Tda*(vrR5G_7K4hXOFx!ylT+hBwOns9fa`(~d=xvn3p`{)Yr@usW9om3T
zQ;G)9c%F?~#DWWQa=7{q<ZqEFLe;I{#JO$HdE-+N_u|FMfQLZ_dp<Jp#E6m9;)cA%
zhikrHXrN2GBV)z=!V8#_?cyJm72rpGV+lD&i&6nW5o;sEi4s$VDVCElImA~Z4~vWP
zDw@`23xeY(!#thNpFY>xD-cCTTvN4ho6d$*jQlDX&QzFL)!kr90PS7djq}xD1n_GF
zne+_gYPV*j@14wn9AQjKZ>m|F0RZ=~*wab6=STx8Vl~<v@viWP(sHUzdY96sXVji{
z)5z`RU9#_m70|Z1P(0`t>RRO0n&%md5jmd**`P&Byas3Iyt<Cq+>^%qQTxGE-nk+V
z!{k(xTKbElhV<ul0XJ$B?ZIC1^&hUBQ3an>0iDOJ*ABQvjrHs-O42W0ZA9L~ujI$N
zdF>PH+^#vFb!1W8YNu0juA_<Bid99Qc_b~-T}Dr2<rmJ0^(eesJ$2*zK4Kr<uu|<Y
z2LH_0{&-9hSL~qx4)#T|+p95Mkr53?Y3J7g#UbMPg;c?E@!pjEBCZus2jr|D)-bYc
z2LeCd_2|s@r-82i;(v(q#kxgiw(}{bCAWJrl<|3(zr1LBB!4iD0l(qg^=t}9TqX5X
zS7#fkA#Kf;UkLq}F|4$qrZ4PnYzgIKY}}s>62Gh@@Kxw(e@L>SxQ5aUP_5AXP7Bd>
zFbK#PgSk%1zB!WKPz8@&Bl$QU9uLdJ09fP$^KFjN8!W0^QsD23W=qZqTuy^V)KyXl
z=7^S=o93DAYBTvJ{U#UGZ=oG24IO$+eUr21s+3aK!G|gM^8u!7T4zXs13{K}3c*t)
z;_Gk&Q^s(Xusdg-lfpk9cj$NDO0GZY+z;9tUVkNzF%b&cO9^(*V6?`w00+Tn*~3CV
z<BjRa$o|0h7`$SiBVu)2&9BdNL?J-?D6NAIVJKXkJ9w+3k5Zjn=0|Ds`HaA~26jOe
zHd@HzY<$y@ffsc|nkAGwb$@5ZtpC}Lje*<MxG^|$kN@m^B+qQ{91V|PtpiVslG&IN
zyRH}4ghkuY=lI9AFutnYm+Y;%P`SKsj0w(&X1;F;tA%`zM*ZCkSa62{)b;L4c6(_@
z=aZpg^9vAkX(Y1gIfD_#TpSZKvtT_lY(qBr>CUv2n$4r$0;l2MO2MVnnd4JtY4QGt
z;lcwKa)Wvdvg3=-P2NNcr#&XmeDp9?w%Og+Up}11DEmtc5L3aHWuY1?L2Zn%aT~5_
zP~(wN$db|N2#mV;(&fi5Ln35Di7!S5Z2X~XXv3b{SPvh|(pUxGTyYgUOu!JC8ClJX
zvUHq5hpbVVl3Sj>DUWqD!E8q7N%a;?1;yhIo}^G9LOS*dr532W{Xf$P=Tl|FN9YUs
zm3{k~N+8tcg@y|h?47gg;#&a-Q=snKoolLwxT%}N<QL*mq1)`Co)S6oQ}LWDMaOv+
z<bf@Q7fH?SM_GXdW5TXOlVRF6iSq9lWi?B&tIhSYCG+o~lN6ppM)>J~Fwn3dJ;M5#
zOE{9%oq|Bc=S!9zMcd8j8>Mi*Deg=gS_5c^IR2k1(WV(smX-Q#<T0A$Sp;cwYPp5=
zv_)A`dfq_jF@8find58p$e2#9dt$U7r9JegIJ=j#S69q*_TBIuN61-C5OB2t{*5RV
zvH;KIj#VIWIA`_+#8F64)GF&ntYQsZfuyH#MRD<d((iU!jiI$q7236IyGvo|3fuej
z`7!ceG}-eVg$O}M3p{A$N*24oYj84?@B<eO<Helj^Wv|YG+V?**M9G@NhYg}NLjv=
zEljdJjYM;&2t01UX|WT>=!u?cs+#*LVI9_1Rcua0yi%fIttF78l2SpHjHuG>n@h7O
zR`u=<1yQT1ox@%<VBdS;v^-i=A~=?F_$9dGX(Di#T^|g3^ocEW`J8_xd~c1jY`=b;
zhOf7Ays`fN*;u6HXXf>#RPT(bX}gYK))7G^-XG%NiiEl)S|aO+uoAVX{p_wHR>B4z
zS}4(2`B(;zoVmr?pb1LlMJCwzsL?q#(WpakE6x)qc)3R)7GXZZkgUsoy(2wQicGA{
zv=xK9`C>M3y8=Kbfb5zs)(<8Qr8KZ7bQWbx=u$gVtO4UFVCGxAdh0cL#t`fg%h}U&
zXa%9sLLS;ejk4tHB+KZl2-I2G#lx&_x^MG>^0N26*0seA9OVr8A0!z9(8{QJ27scV
zvdpfle&beVbXCM3+ml1ozIxA_atCyj`5l|8#4uJfEh3}C<)N)VXI|Fu6ppeVH>KU*
z55Cdax|39S^_5}dZg6s7ih}xR_~%XE$JAvEHGB2)E=eD)A*Zg^GquE&p~NvdGqb0W
z1n&)_e%T@6AawP$&gIrBf^M>eL4Q*HD#K#4N>}&yerIpjc%Btqpl~g>%fy$k_JSVE
zg)a?x;p48_4A+RiXk<Mor5Q}~<BAbpU<ta)Hz>QhuLZ2~1&H#1FVg+WQo3j*DtuhZ
z9C9QEf3keFbxOhSid%ITA;7G^eE0_VPz-r@m+RM1scQ)5YQqzi*j-Q2YyAYVaTKen
znCxcveh7olSU7?hIRxQ2)Zw3!7e&#o?pD@m(m)ejaGGrkwamMo{>CdFLp&QA6spj+
za_8^b68M{sBYXAfLr+NP(5eHCtDX<=@hPnZWCdSbA$wZ~6-v1Tdk#;mGW|RfT(BI7
zU+&vj@y)Vy8V51yKo{2h+ShVMa+8Sf2+xgbL=whMzqph?!uvF;rmVQ$S9VaghZf5z
zfC}%{T$9zeNsj3mVzordY?fXCZP>19Yel~B&FU#QEPmoFtuDF7$v^SN$1V;kLh`>#
zGOf8$!gNGw*86=)P9KgQ8v|CssSm+}!_vlI)mA$y*^<eWj%$!T-epT38a!rV@0wI0
zs_;-PbQK}B0Tiju2`mAyM$x=z{YIl7!m&cL(|~r4WY4uy8iZ&(_#KsTRfJ^vUKP|X
zb}eg&qy?1#sX=0u_EtEd1t?mgy0s}j+)imQ#d>{6!Q_1$UT)P{$WJHS*_10ri;6Gj
zbsbw5Kil21(R+o#h)RDrwmxF)UIUl^bTyNGk|U5vDE9%90Mp~?HBX?RZ?=bRei*tB
z5<0RFX&mPg4@Cj2!eq*C+-B9`ONISu@2M25B_q~yts-a!D<kw(RCG#gN@C-*Y-7iq
zF3S>mNpWnxIdkwu2BBVlF;#8FQ?0!T^zf^|sv{pZC83a25V638ySp-9H{0gb%WW8p
zh6VdpMl#UYHA6-7Uen@-cDQcqvS<ne>Bd%l=Jc*;3u-I{i1$6N1Jf68`e*(;G+8Cr
zDZtpsnCa?Q(^&3oDHq6<wU&%Wo2?16P%X3dBPmSyYTli$(W4Y=GNCmk+2i|#?5$T)
zX>@RjT{<bL=TOV^r8cC+rqt!oL;>I2%Bj9!r4Z<`()ydryIa|@{2wjnhHRdgLPQ6Y
zWi*ld{;SuHH1#0*3GF&yB-0ZwXks$7dHTv{#1^h)TjNb-sQG2T`*E_Z^Gq8U(N6J8
zfYpBWRQZ;=vYPfGl*^(aTqc#^{yx^<tImg><pjg5cB>{iRsZo<$vuHMK!(!zFo?7s
zgT?)#^|FXZ(`^Dxhz5TyfWY5?Xsj!QjCOUo9n>Ai)gZ<OzF|3OmT<2uy%~FB4WTMO
zY*UAl&k}SNf(n@lOjxp*?6>Y+)Bb8?--jNl<?7=+$&L}$_VP%R^*{s?aqwMSrb$c_
zS+6M<`4fF;g#(lg{lj@m#xcaevRLPdBEE04y1}nPf%pZBgiAr-7R~6w1fY?>YG=8S
ztuaFK@Nep=5#KdFIj-XfSMC}sY<RoR@IQ;cpU=_`AuU#Ei#UIPTtQBAAf;fi(o>-7
zhG6yc$Ll}4f6C;n6c8CNS1~SAu10hOCo9}ZaP%$jr7(kj@q=0Z$NhcDpmJY-3-ywF
zD;djoE2A&0L_w2FMy!j(Zezht6QJBmtgIVZ9h$!aG}S9L$v>--E!jc#R)A$I+OEo;
zBnC#C+Dp4MsRt_E`#z#mc;S-^$tUp%%%?7aSdwbrb}LJ9wA1Ak277BTh2;q|^F+z;
zg2wy2$a;X=w1y8!k5|pNRHob}zyS>gzeP+@Vd-%2<H&l77$Tx&8}%?mg9l1@X^pj7
zKvkk%`|NgzC<M!`jCRHn&2!zKmeg~(jR8bmw^+xpRnB+XD*b5-(?c~aN&Ds^9W#-m
zr2T});@2kS6^(&#W$x(fQPnyI&9uvPKOyhC%eYRCWIZJ9SMB1s8u8X#D`vfo%$TLc
zidX5<O|)dz0}zOp?^JzV^FHD#i1glX>QvW(y<9j#{*|nYq1<Ls`vFB++>*sDW3E*0
zxmUSKkZM~uTc~rx#et#~fRtJ-rs=-A<C3@G@m?c>PD?VFYtjPk+$!UoVzWYN={5+g
z3}MYp;^o=mqv8o7x7|Rw`2@y9&kKZ~blMO}oh^v5naH4#m{`lg^73D4A*O1tBe@|C
z{vRKnRM0>x1N)rQUwtN;&LSULrK^H-26rq=J2{%DHaBAzbn9ssq%CEi8|Vq2YGi#e
zfL+qaCVp*eQAyqgSjdWpJ78F0aVcvuC$OL@W1AN+xCj>IM$P9EqHx`1`HH$FOO2SC
zkTi`A<K?Ci8wfqp_fta+wVas1$Mmk91+z4p7_gjSQZjavRUUbqJX?wMeN+*ug@dkc
zi>jz6C!;yM$lwqM(MpUYCb?;^+y8#kFHQ2XKA?dOt{e$RSfQDKjtpmf;-Diub64^?
zFssvwLR~M1yhKXPDg=gWT^t!~mp9FQ&H6E()eELd8$^IE#BNI4HcsYqW@g+3Lh4>N
zK|vX7^k=;Q19F*M>KYSt_#Yc$fG%8*y5xqYj+Ns8U;uU>(*zU%_2c(+_JxWwhZM=H
zdg2i<fYrrOR7Th{?2M1ealP7*IgoJu2Ve5Nmtlp=g&eM1geS3(rwQ$rrfW|?qH7Qa
zLLp-FbAF5y7NA$x>4njO`fE;3ZT-Aa$0lp4-uDki+6%LQ3MvB9hls<@S#qsp+&oD<
z{9tN74AWj{O*S1FDJ^KYwHvfrCm<?I4gor)Q+{{<I$0VSjj6p_tGM3GKv+_Z=aN_2
zj;18QFC`Z)wItK%LyHfrzH8eC>iZ-ox&eei(05_WUy8(Ev0@R?N5*Guu`j;>DPnwg
zKYeJ4VWkmxfYUc_wCW|SaCx5mni|ixVs=Y0k;ccd5L4ydFIUXuu(T^*VDzSx2fJm!
zHOX;l1=*<i1-&c-D-o!jJG@N;je5NyH4GsGP45<%5?I7I^;#%k%<muY-&1yl%DH6(
ztN;r8WC-yrqagJzT%bs_{%Aw>#-H+1&B>%6kzG!_R6HxY#=Bq*=iJl5e2Nlt0toKP
zU3mS9b3N%3jU~wru2<}zFjVOK12mAtmz#tufATds1?ZOZzWBui{19DaHYYT^rY$k*
z^C38Y0vwUaiCDK4H#^TLgmKx;diKL4fl!n3a<7h-<Q19n#1wWWBysQu5`4VFZ<Sa#
zU3dLjlqY4Tc1yuCt$@5`oy#J(GR5{c`>^+mX^`3aCdf9`Dwd&%@2$S+L@W|L(?!f|
zI|`8Od3{m`=#J<vv)r8vTjs(AXtazUDvQecq%F#OCGKM7{bt^fp=%|@GEH`)hI+hR
zEk|J{kax&KXQ=cC5lx6UNbj@bK*(Ng2Bq=p8vDBtwGqrr>5lJ_i8)tc_=Tta8*g?K
z%}>-H|FF0<C>xDSdV=6I1W+<VZ+g=YCR_?%v_YJ+F@P-vbEn8xj1G$rprrm|8Jyn5
z>2-$CWDqn+OD@P7Xfnjd#X_r_^f#-aDxP}xq=m;;d6gG~0UDE+7)|MkT1B=URHHn=
zj{=O%22xXorltS|N88?ywWbb;DHy8#rw<lggC>r_8bS--$m4Z0>STC_FBhJ^)~Pde
z37aQ=s`k}IQ2C%fUg(Y|!0QyGe7g~I_fyu)YH`kN!Elw;hNycWD1n=5c<m3lqwSnH
z`GHY3I+=5d$7k?fpEH;=xlHK}EBqW8w?iu)E+$Mt9Xar6D95r4RWu-uRC>f-j4E9(
zU~mmy;6ytZAXi`CT3vi4OIaLeF+;UlrbAyS3@RzOI#i#QI57~?P=aekJgq$M>rp8-
zRvKk&a5b4_RbTX4G7G>0Y&_rXZ~-J!?<*)8C)jWwF^Q(#8fn1(B4yD}DcE9Kc+dzb
za&kyvdDeR|5~4GOZm=0YxUfPQri#)~)!DrKF?lv}sBo<+c{2#~87t=Bgv>4e1Xz!o
zOJ6(7avEqSZ|Ev)0LrngyIQ%p_retpv8S#`c7-=n-4Rek1C0Nqqu8ZKWhfP7>elq<
za%O_4U8d!2t#D7q8pT3ePFo9nYC;@s$Qn0!$kszwx#$-_8IiuSBdT(-GZy|c^&7wM
zDm}Bc*g+p+a#U8-nwbfsBosxH@nZVP)E|j3un4>q4k(IS{sr;TA>l=9B?sTG96z9g
z+&7f8x7?Qjlds10qtRFppeJ-~ut&ts0s>(cyX>eOpq1O0E%ps0tTdn07O7NZwoRaZ
z2JW^UzZ$Ddk<JNy)(en5nZa7_J9*W<rqq^RzxL~jmCCtPI#wxdqz(L&lfPCtpE=pS
z0gbn03fM8vWPiPs;anj!)Z?$7jY%&)dLD;;Y0g8afAxDUgDuhf_=(sN);UiTvV!yy
zz0@z{Px|R`IIXyD#ey8Ri^q098574p5_c)=TBAt7bRZIDUkU{ppS&@1lAq<#%RR2H
z#t}o1$)X#vpfsn6*fmeqEzqF5G1B+`OZ4h@g|>T((`9al)wlgEg<VTj(FFh#GXQwM
zmxbZJn%Nk%y34$FF~A7&sJALVgd4kH8uGk6HZ7{JwiT8P^qFA$1?X+b&!M3lKpe}y
zh>R|OZxz_v^rR`fX9nvPog%Z=;Nv11&;559oFy#+s7`OowoLT>Fr6NuK>%qG=Leu2
zq(6Y@;tpb*#VTuttx*zWl9v<vu(Y{FXw}HU6=kKKLNXvx_4Wp0MGMptg%l~4;k|IB
z@N{0*1ntR9m8X!qWu4w7C3<7LT!jUNH!?>Y?MMZNiWB*-guEB#7&8p1q4i)(9vE}o
z=B`8*FW+ssvcif&A9JAG2G|u_%lt7rMzab}OLU#1!PFJgqoJmUiV3A!P$#DY{rvj4
zDQ!BUes|K~eKj$LZ$4%-oR;r7z5Pjb0HDDsB$n)D#k{&^`}aFP=z+f4y<3fdKmHu`
znK7nzwQVir##`xb@rf$<zmF%HT8IGCzQ(<|HhY~|^lRS~Dv@y;$Q6J7(%_lt=f9V0
zHnV03tu@~=Rn`{D*1>{)o+XIRrT6d@u2fykEtdL~{chy-!~#Jg`P1OKiX&+Q2KENT
zu66cnI-C&UoLyEyWSH+k>84zTLLBX{wE;6~$HJ9Xp!xF+(6BcBe3?BkB&|-(?u)ax
zB?7i=!xv7WqSPTJ-6?*P)<9ZvG0*JtRE*Dtijb;kS4C##%sjKdpu6Z{Dl|7Zyf8A&
z3Z8JhA9~^+M4iMi3y2pTD8Hzq8}65XVKJotG&3Rgpx8F>hmxa<#`<CLY|){bSXb|6
zE<kso=)Yrq&s1_(OkwA$JnVF@)*d_V)(v%9ZnDOW*$*Kk7jp>524{6Bq9^9BbN5e!
z$B@H}1U_ScD>O;nF)<!{dBsr-_$5~nY{Cm+9D`}<=DOr^%ez;HPb(L0{+I&L+^@={
z&DF-(Ed#8s*#7LAF1y}UUWb@{w*#5sB6(3moA98frU%q7eV;20gTDZxTE4m+91626
z2Y&S2wj`Oy*{eSxIU^&H$CLG}_<ZXSrwO{w3-;-P?%g~nmV`bDnX)lCzP$7`6sT<p
zk5ip15PLmFh#Ejkd4Cc%yY)u)b||qpRDRocQ4pJQWh~|}d?nF9ch&4cWqFutMUr=P
zI;Y6u^kD}ekN)KSbN9jOUw3HsUO(_2ert|?0g;3C*F)XZ;*)tqy_w~mkUNWA4Z$h}
zOA2|2Q(ZtVso7>EHBiq?;9G<o%tzWkNvD)3KQZ3MNYBsHA8w%iD}gHdbOca=X&W~L
z=d{hyi92{jeDa($Jx%1FnN0}k>MA)VJ0_}vA?l*CXIJD=7%e0NL9W5?Az9~dWSq4k
zALZlzDC(~Bt2T`5<Qvc@ajF+*0n*bxihgY`@VJBqdZ>_=ZY9ecN)fx9nNKo}awB(G
zFSJo)H2Q7vhA0t$##tIU{Q?4zB`+Gko34%s%U0v0KPx<W``sXiIO69;N%#{o22u=B
zqKT-TR0y7SXh`c%QO4x}#UH(3pyUvAvh{BEej6$bh(g_x7dL>gN+DCQE<H8c!~lE-
zx|K=0NIGl!vdR<|P(2Lf(tOq|RvB0rcnV;DJlka9?#B24WUMxwLTW-)UZ=}FE?kO8
zz1}>e^~>>4A@;Dn*rRlP5GeUJ%dgGH65^ljYd<MU*wRzG<|q^+`Wj-N9cYKA=yW%*
zLhYcf5`;nU?B}Rbyg!h$?-fXvoxhWp7YWD}ufR3`nm`}*q3Z^)5fcUBh>E40lfxX@
zfxv$6$wLC!fc@1H`RbxqRcQuAs*#1M<z6Z6Gy{JAL7wv-wjjwjK8^gnHu#iz@-R5b
zQfJL^5hOKVR(Ep|iIJdCDVc+d$lIsd*~%<^N1t6O;1UqHw$wO`2-r~ZWeEyw6DJr&
zeqXa4xV^S5jn>W=XEUx0egFD`E_Xd3+X`+<n$>7o5RfpA(U3c}!p&pga{?qG2rkk2
zpI1p<@u^F2(hP34WqRiSVedVoqS}@<P=%HtNK^qyijqO1<Sd9}0VPY6BpD>rAVGp8
zl`KdU5Xm_=2uRL3H#ujT)TB4tbKU#CbMHO-?DO-DHwJr;9?*8LHCN4=RbPGem21eJ
zbFGs69RNiTcGfs_ZkAOY{ispLME0v1ZJB+Dwn{or^Rpf&@j@z5o-t2kZoS~}je=%V
zvGs6WLJazep6dluA&I_s!ASE<Hi8!yV9+0;<PzY_b&IdWlNln(fI3tx3vZ9r0Al!^
zniqN-LqC@*uS!`<_Kgssbrv$#_GYwXmf+-V2C*5;l{DsUTGt?kCt*7SUPQ7=u|ifL
znc52FMA5Xu$P4FxYXLl}WY{i4`rqVd>vFXR>7x4UgUc;l=fM$Ss9eMq73hWT(VDv6
z@p6kd;`-!>2*@JJL|GPUk^89F7sURJZlpb)=jEWS(l=iI47A{Qo|0|qp1$dcC=v;N
z`72v&Mm2k)s=%!Shusp5ZZP@dBEP6<;D_+KXU7T8i&?ypTKvR(pn8WfI{?F`TX9|Z
zC|zbVG&|@viVSGiVT&P0OCK2|3ds%Vd0E<TKOX{{cdDgcjP(LsY<qS$$`$f5FQvWO
zPh`0O1-u8d%q%%OBp5Krz4-WO#Aq?kh79Jk;dwbU(1DP{|5BNv5@xaJmPPe&M0pX(
za@vGyKY`V1f%KDC<MS(bB_t*FTO`*bOS1iVwaPX%YyVIC$<~51Z_z)WCF^NH^SFN<
z;%<L=h*LqP56jALTu{3Se3$)ZsgW(%AS*0m-Kn=mUIpNS7!@V9#&Z;%G?}Do;TejG
zUF!RFD|L`@i1%%Ni;^vC?Qpv4my)VzTU2qxCi0FSE^p4zleT*YjyVwrY^4KPf^7=3
z11hRVfbUZbP1Fhjeoc3CADJ|04{C+^)HTo1psog=kf9+-oyu9PN^}g5+@}sNj_SOg
z#l#)`mpicFs_NiwgSV-E&7dzm&PrmY+>qz&I$+-dN4fpmQH%Uqt&xql*uhcJ_400W
zOtsI_3o5^Us$LzKoT|k;HlB2@JDu`1H~s8rV1b8$?Ht*wc)eZ4Ruj*SOyr#_<98P7
zgero9twYX6I*N$u_T?+lf(P^p=c6qmPU7qt?lU>=+j8NuV=~1Q>BeLZGY_vBJEmCv
z<E%d!dN*_8uJMiS-6TJIOtOlE22{8i8zE&^Wd*<;Y(LuLXIk%7I!vo;q9t0Z)Kek-
zMDlWSq)h9v=uTRC`nh<<#<2<9gTk_h*8s|+_*w+y@#Xui@YR?1Yq5rN@sbV>5@Hs+
z_hQO4lGz#&3ine&&eX>%gfqdmm%rb3u(h1e)5_l1U<%6F^q496xG}>oGcVm8PcDZu
z0KLEH=7~tF=cj1HGa}nIxT|ojhA}lGh|m{H$JtsA<kKiR_B_i?tNl9VV%qq<TX!F_
z8pJ2)4|)%Dii%g=^v(>m(8Lt3_U^w)8S_@H%(=|HoVGq@w#e&N8t#B&`!Y9uBzDC#
z$g<62iW7&wZjTJ?d2B-3FFsKPl(a#@1h*<O{5Uz&A<6dM&y(1e4m9dO#!QL07!Vfy
z?oQKg%6+w>D!Ep^+V7@)aS-Sqp=T=>&qBz4R!hPuXVW=VayzdA%p3I!9kpb(`}DRd
zK69(yOeDeBkE5dJdr)5aM<K5$cVxUZ7UEWVse9+qi|$5QKGtPb6h#_y_PtwMB3KsQ
zRgOH5aHq%a2jNs?+xejVFTO=+iR0kG_&F3r*}fy3U5Mito`3j?I8zZ!&CLF}kI*WQ
zup*`91cqgYh+l}3W*Bly>*N$Mpk$M{7n!-Rn!LyrNj~~@anu$hK_lU}(Z94MF6fja
zRG>d^sdJ;aQt`w}LU!3fbg?mFdX?E~-Mq!2)vQ;S&UE{%4#J}sYY|_+%o}T!S!_kC
zT|8_YgyT&0>eg;AP_VVYw~iErFDl^7n*OD@h@qC){&@rE^Y^M$7CmQQwkPfLxYm{n
z$qg^6>rShe>e2B#7cf{2O6<|ISkygtSb3pZZDM+1G6#+(tFz7p`XeH7Z`_skD3caR
zjmfH=Q|0pWPj>4{Ce7%0o0;mi7Y{(zv(_V>F*|zaTIgTMZkQ<Fy{r><oLa4?%gv`t
zuAa;aVz{)ag~`|h^maE4KvGM9hVfIMs4eE}QvM)Wrilr_sYq+|;LrKlqNh*WuV_v~
zx`-ag%bi1CX8}aP2KYrS@()2FD!#qlmaU?iEvUTI7rBk1IpUzXIKGsnQ#K7*J5rgC
zo1R<Kc`V@g)aE^HXDomN;a?BPvp8}swp46lssa{1^z&=J4k?m%%tnW59?9$}r45e_
z?k6?x557#t`2MDoE7ZLEl-q=<_oyhQHfPi7Yw{A_5#tjD{keO9rgnEDB6aX4^GH@_
z6$7J`<%uobWVB(;%?BgY8S-^t&ul+1+yB8J>`2@B(Xh&|ufId4s)$9HZ&54~|J77g
zhi=`-Q4RS@hd8TuEvkq@DJ1wBjgg;#zA2`mv<Of0@@7g@J-h@^BNkbHs-^QbvOM!)
z+*SN&BcX~dM*H=7IWq6yU>)Q(&2JL(rIOi%H;244VUJ|kOctAF0p0&ebzS6ak!aL8
z;A4!n$gTVD?zt&a^fOC{^q9vTfed|6B%_sUJ0^P7Ni@PSHkbUB+&1ZIVR4D|4Et<Q
zKf0;E%!~!-(zQnSC*u{*ta3!xcEpDoS5RoTn=YXkZaxRwIP+1v{=&oFC7kjS#-MlQ
zW<thsNeQH^A6-vx9pfgKJf;3Rx#{Exiu6H{dvnM{vk3n|2&x^XB!A>oeR>-M*&i<%
zwo3*p+0Yvs9=kZ;Q5+TgQPt0>cs<F%{&Tcdyxl%oTOsG^(>SlgiJNyuyz4l>7=))`
zAUV5;e7Z`CYKtci1fpCQ`~*(Tr}ii#poJxfRt*^X{`lAJm{m6&ZEux@hrPTfPOYX4
zyh*K6ugv*d+BigCmM$NEL_44mUGMSmxzDeZquoHkUk-X%bYR7MaUF#$`8LxvuIw(*
zdx<#R0+7hW*TtdxUt&*&=Z(>(sNO%pwt7{Qv#x3SrBvSyn{;6`o?Yf5y9c{p^))}+
zt(D-J@p3;3d*6@9ibRKENMygSW%sfJG{T|0fA9Qb;<_v;Xk}s7_Y`AdT{aja=`ZSu
zv4}kTV!JkwcvvE2HNik~F@T3KZa(qssFco&--}}s^#i;Ay{21OJ3K<?A9%IaksHTI
zWaV(EyH6FZZL~qLe89^IoJ<}{Zx5>J<7VE6eAdR7ttS_#7A2mu&lW)&J?V=zYstQN
zuWitjb+f69H`jWUco8q~wc$1N6M}WuUQP2mcb%wvzGBT4Sz0b-`PCb-=j5Ijv;OwX
zMSY7jnm6XO_LH6TiFNDCPJ1CTK=^|3+T+DXZ#N+)E1`C+qK*VmjkiUDudR}*lNYNw
ztP*;RJZEykU+@QXmzsu=O~J#foW-?)vl&ji;1D^LD|0-Iv45P6%hh(6xF9I9uSjl8
zwm+e4ov5mHI6-k_kN5GdBUGoG9poy;LjMlvy^dDd_hv1+zu!QnJS_pX=rxqBJ0vtm
z7;Xla4ZDA7Bb>|D8#BUIi|8*)<S1RpddH~wVUR&&H%Q5H9h(c(O{Sjfo(`A89$}Tz
zGOHAEk%{tu1f|ZG#)F4;fa1lOqWqemy#N}X#mmU+)^l1}=)n0jt4ql&f3W9Z(M!ck
z)B5a7x3Y@hCxM+;4k`|6XYC$FpI0M!K>tX_$L_|elK{?5SPMYQj_3ii0|;%Fv*^Yr
z)`P&LXg`y_7WsIn{QEDpmtb(zqFKxQ7?v7dtQE>2V506u#Kr*j&|)ctnZ5MIPc5_u
ztBy2Mu2a!-BZ!5O5tYGc!%|d})8+&qcj?N{aSt#EUprKwzxVOMHzlA#^J4;KR1M)C
z3-`g(W-DVczN&(VHf*pibaT~9&VMSuZO4iRO=E#x|7c?utc{CtK0X2)Ug)qcEYpZ(
z{ZS`SF~R%AV@dUmCMFK&BmfL*k|a6>WRi>?V>g0ahRxoPGl#Y!c2Q1F`Bf#l_ys83
z9N4<%PY|KWjap6u6o&uf)3<55F<T`?x{ISDP&P06ShcEW-EjV^r=@zQr9xk{;sHeW
zvx(6<_i^ylyWGucZy4^x=3=3s=Y?X5WAkQW<f2Bx+%m<be`x}wS=gY?ZXC3hC}+f3
z#YMEkwwv8VyGrb(0|flIS5MPek_^;z?~(|bgYL>g#ez8$5<9`Xp`zNk%jjH~<@?w;
zA)%V~?z1Bn?dfdlQ@0#cF`|0qjJQPh8M752n}x(XcAc#ksOjcwqY<n+T>#c8c@0Z1
zZ?eWdeWgBV(wrx$En~omV|Q=xa8wEbPiaYjIpy0=ttnp@GV4g3OY>{UJtNtM_l?&2
zAB@*L>GsASjC@SX(OXg;pMYN+=58fCpJooP<WR`_xkl~P?oJiM_Tij&Bqx}1pncH6
z!T&69%(+X?zpaq1jxn-KgOt^ILpJ|tv6&u4`#*}+o9<cW6mA+~y;_YdUG&U<_1WUq
zO2^5nI>6a};HP1aI=biNj#LWeH@cKIRwT8I)GR%!6=sh<S>WI~F{s&kWSfo-og#45
zOoSYDJVKGZ2eh}GJv5`0qjyovP%m_5(XAM}4kJ&j9-C{6Ue8#-iM`5ScKb#6Ux4lG
z*=UX~%&s37l%ro~oyym$npgg?=VSzl%)!E{EnVrA#_``x<Jz@f#zR@uPrQz_Z|n?^
zpi;R<*d$I9fc|Z)1)zdRgHx)3k1sd{4io}T{nytCcYjr%sH@5RI6dvrvd9?60<Dv6
z`awdVca*BJq)k<9^{Md}DiZ@Tl9~$8npNS*V;m=>sHdNS2Gy*kAXQ=bA~-bi!>1-@
zS{j_ncuY`M%+#B?ONForXCGl1ZhG4Y!RbulS!^W@+qcXE9RUZ0;m{4A>doa^$O+0-
zlt{Tda;TH#3tbG`oU838V+wyepE0Sp8SM#(Ro03kCktiTod~qpVUm@Ob;iSvANOWq
zRmNJY$}G;tY{JQrM@G?p4ruhQ32FrNVY}^RE6feDPdQ~4Ah>e%IV`H@vAXy~xD-yj
zmu?z8(!>nYmDT66WnTwLa9A9dzf7|LqFs9H9!dkD+)LQQ$85EV*8m_YZXNQZIa@L;
zjr1&0i3gmapAz#kRWV!l;#MXC)@Pn)e1z6oV?S83zBcA8*I~!FbwU)Xq;<UoG=-Qu
z5-{D%y4gnmTs5F)Z!u|6Y1^++5z>x8QaR|z5WaFVlS}64Ceib-BUfQ=H2u=wXnLXj
zJ!;js0!Bp~bH^*|X#I+@;)8%&sw2XJ4~OLR54&#GF8eXIr1jgYMC-eR#iwWfG#F^-
z5IB{^HjOra&>CY!5p2MELB@OfbjqHkyUh;fPz5(aaY0SLv~xS8YI-HnzWQ9SS`d9M
zRO`3se$QacpqorIDc1wTemP_#*pZEEz{P8n(+%nfU_rHJL18pn$e=S}?1wkqEr~#T
zpu9HahPDqyyhCD?Zsz*-H;DiKd4$6>==g$ahbHzM0sOw|&?EmMWVxD-*Q0e;$k|f{
z^f^xtC#pvWd7y~ljh?{=4_^3p8HJ7XxHNN(ldBEFOZiRc_+o?4tao6rh-cP^9}ZS5
zfS`%jFooMn%BD7yqyswdV&&nDI6DNg4W+dc&v+9Ovlh>ogojOHtClBV{9yNDQhV4=
z#A}{=HYlNJ+sQFG$lA$8h7ob|sq;eU?zk7{wjZ(|$QCS@o;sgOAM~I&3_@$l;ji^%
zdK@bSpMwU<j#6beQ?Gh-s$S1SqUrB-zWdOmdg;pBmeP3_np-FgQoSn!>ek88QX*V8
zG@Iu^VtO(kYr$+~46)aZp(fd%*Mf&y6eT5qYN%C(W0sh4*lgwPhpeJSPVVI_+lw5h
zuhHIBcITHvVO;hiuBd7sst0ym8KcLcHj)~S2Z+HxDjYxL;#N%lD&NTlvZjn7pU@dk
zYu26Vew>$wQG=dW_o#Azkzr8*`nw|SkL2j0H)E4uYKaI|x=NLe*o%atkhpQ94xr4@
zugG?}Ajm1|xm;bc-LgU9Py{=HGL57{a}Qg0jw;INC;EvdJ3T+F$0V$q>P|(^8~W59
zoS^z@b~>h}jdeRt!r11Q>>`cj3xtwFb>fHM;{497&az?4+GUpJ<?|P(+oA;;-JnvD
zTOPLjSMJLKtun}F&3yp0<$0R+kt>XJB)WyD#Oql{CkPx47Gz!s9XzSza9qn)2`E_4
ztHyhx^itiPzfr)d{cV<9+hwB{pxg3kz&O1JOTH!=YbeP1Yg=21#Yt01gD$avZIN*4
z*Mmr(pe#9{v63W$cyfeY2IMi$m9+zuN1iS%9cCHJnd)ZUqEvI>2i(wM+Jgveiwqv_
zNAs29_G@xQxLLeffGp8W&3jprFdmXuU6*;E-2wDil=e@&B+A8BTj<_g2Hg%mf@vd-
zPDz;zT&!7x>F-qmw_^LIK5cRY+PdA+1aI!1*3@ZJfzL3frO1cXP0JQaXUpKv?O#Sa
zPP0#9jb#`ZCulI!AcZPov1rZd*D2K<_<8$lfO-nA!oW@KXj^^Q&Kesa^+oI!Uq<b&
z(QGO=C%uZjsZFQ<ab7&`(r^G|Rs3O5^}2D2(X|iFbS$;zVI-ME*5pGB^9xORA-pVC
z__vC3mUa$cFKSxiyvFJbpH(SQnVxOLYx*l6eYDH4(m;y<+K>vV<%o*PSOssWo3?`7
z(=td<4Ato+{zshl1XJ1)?FO04%c<EOPF<K1k-l!5!_Nd9I<b%Y^ALPBd)>y7VZjKj
zR}-{Nk-GQxU&cIh?pBX3W-W#<2PAoHr<YGG${T%h%xdX!i|Ua_vvbUEsif+(Uy_cq
zXUS=yJoM;aZ#5X$ge6it5Mjs$vP?*5zx1|T_Mo%J?OMJ!ayGxwal*klyHygR2cI;`
zder6_ubZ%+wpsxvBja6;mwOq&cI0lHWvrLKy<8E!qtm@gY@Zkvy$w$=TF{8!MjBXs
z+7w=OKKs@YdudJppR9RR=tx!;mIm7z75E<cS9*8)!q>lc=f7zhbdH$)Ib0bM-)+LV
zRY8^@V}br8SDdsAFDtu^^J|*_;JZxpDCPAc&h1ayvH^G<Syr)MJ&i^MqR6F}&kZO)
zN7)r$40MY|nY-)od&l7+awlIaIk(HO#w=J|zZ&sutIR_TbV-Nba4YF0mQino&tr8O
zCEFw%#2*<1Ttk{T*7g%W?>{qN)!f2!u4v48_<fN^Okduc$WVrj57m+}950qv=zk44
zJW-{UG)B~V;_0@k=-x;^-8-FSSr*;f|F6kkzv~n1!DyyQP^XA1`r|0cp)%h3E&B}B
z6qMS>=8X=A{d3SEnviVZTy>Gi3e?>gEyGb2xlPP0`8A*fWb3dLXNtGA0T><d+Rf|S
z!iHbdGxHy~JVNWZ1$652Cf7oE@x5XFf=X__Re&ss$&UKq30-AOHW!Jv^*Sg4yMv|T
z9S9c7J)mC*`B0w8^g+-C^maJErrP@uz+Jb4Q`f6_3Ur_>Jx^l1TsH(iDLDR+5BUp#
z<R5MA-j#$dt8#kd6rRVkzKaUq*uDgOLHXuoAxtV4{e6HfmdV%znd<?qpwFnX0tN1N
z8<G%++4Yy59k+;1qv)4%iks4+NM#2^pEPei5G;zJ7`_7W2MUEZBH}_vP`#jeA3!pg
z&vr#l`3w-rt7jg00iF;X^ic2Dp0fyFMGFQu$+|T^wR+tH^ch(KjiVa<)O^o_?0A%J
z7LM*d=3lhZ{(g}D*k>C(7nb{3O3?3>K!A(e^23jHz-`fVXd;y6rWNIMX^e0L%gFn)
z%LNFfJPM$&e_GtdGl5V%5QLv7?l;kk6asY`k^WN{33dq6GrZRH>>=eAuE{kll$wWv
z$Tt7&Nzlnm=_ar7C4DtXj<?qtRDV2r9c?a2b%8z^6$lHK?6NgYP*Reu_;->{I!%Ac
z&vdERA%~U#V0`&4?M_(zC>o$Z0X-x;hqK)lUQmbL7#-k1`}gjWTwDKW%Gm}bwz7nx
zK|BlmC8dL^i7f%?N(zAx-X(2N>}a20yP7QsnmxKey>!aSE^e+L1zB}fE+<ZW9eEg9
z&yAAiNvSBz{5UP48kkUKK9CU_YOsl&V}uevtg*}e)z_5sy&gp!TtcaR0tp`3k;7~z
zn_m*3e<g1N8O{_#gO=BwZ%%}{qBKI5Q1VCi^SdZ66yu}%b%(Cp7vLH;Fd<9Nzy;&+
zewG3%lJ$46A|p<K1c0>#yL0hhis=0Fsp_5g7Svvv;E*LT)MxU1k^&!5j|&`nSN)TM
zzm^F5r+xDcTp$y!5_V@WVDZ*iGIq!Sam-TG*46fM_?x<&Z^hKblJlA8Y|tEl$|^4=
zc^eH%RT2Yyg^B)s4ai?~iTn@Izvo~J-SW?bEJ=Yo)h&x$MBV8)5bn&)t*iXI`p(JB
zcY8-}?eco_1I5dD-_TctfLg(h4!(k`fRN?iw9P*s1{1KM>y4^t`zQe;HtlwM)SX&^
z?J+zIpZp8b#&6I2hk^Bi^<IB}9A8g?dW=7w6a|7WioFWHf@~D)_TQG{i~<(&ZkEy8
zm5>impW<r|$Fs+P&eJcMy$(ZdMbvZ9kn+yXtNLEK(|{9?az$MR0hDF$9e9}k?n5FV
zr2{J<-@Fbn(U@tyA8z?e8Thv%ESauK@DtSiTF}}8wOQ^MJx^B$<Z31_eS&R+7OVdW
zShoMM9RKn+VnF6r-M~dwUl)QBivt3BWSetq>A*fd|FVxY?8xa8&K8tv7WR?(pAYO`
z>74xbui!lehrl8%>Ta-H^)|+i)J2v}IZc=Vu9gKL`<CRak_Msn&v1-bg9pjWH_8qG
zb6xpj1GUn!c>MQnqh0`*p*ALYACskyHxP)Z97EO5-u~7izIMv9wWZ&MO)xrxhP?;a
zi^g4xF?!Ro9)Ndfg%TpWk!N`Dcg>7l8{AuQsW8h8@4{=cN>HP4vtN5sqZ|#SOl}Sh
z*z`FQ2gQbechO?ifu*`m*+<8V(H4NtjwKWty5-U$i08JEJ$)F7S~8xuVr~n9C<#Qr
zlZTZHpKcU1t9|?@U-Xxzu#f5y@|x%*`n#L|&u3Fl0sKeT?33cwzk8Q|ULCs^SjYYH
zE*6e|??n6GzNs?>_*)8V?qK;pukHU|j{ko-{$UOO-?EMg9q7T3Xk6XTX;L{4pmJWy
z?6$5AcQeW<DE-^v&+i>qFK`<i<Flb4Z*C-SNhSPqcmB)M<9}>1eR<%!IDV)(*aFSS
zK_0fhef<6Y;+z~*8x&HJ+tC{qbR6;@Z)@*3pzj@@)m3O~072e%I}ZLIm;CQKjC~g1
zEgQ9!l5PBX!=(PUMEV~q9Sgjjkx%Z@3>q{$%S!iu^Jf3;0R6w0Uqcftb=1KZvo$m*
z@h(~1e|%$HAOzr5jW?QY0C^Q+hvfH_{_PK3(gcB}e|e%99M^{8A92cma(CVwAnt31
zXm|a%hE!9XsQEWTjsLk)1w~+~nb!sT2CpIM=D0Zj;~O)P1vi$#)6(#ca7BA%_CL4k
z&SGHIxHiHd5}oF|{a37n|NF*`z^NOGUG~`*2a%gm=K7y+S7Q_~J?xyCg@aINGJW+Q
zZ`aO9aAWdTVefG6_m=lI>is9r{nz7(HARK#({3P-sIg=DE2#{yQD!pHP?~h{+aAk;
zzvO0h0M}6GkEI)qs|PHClJ4QwH3E$!G?en|=gqQ-yh;Euz!y$r%R)UVD?lo{<j-4U
zP~3<oShAF(u2NruO%VIY+nfR&AV*f&Y904~1<8NyDq_XGQL>$t8$}%^wrwm#zfG>b
z1eHD0nhQWcGfE1Kd}`$hGw0(GZyZf$fGn7xbjFY8A}<VuX`e>i6Vv|*iFU^Y{Xm5Z
zoF{(3czL<6{q>`%m&Mirn!CPKCuKm9j_#UJ<>T^S$pY$O6o^1BfYM^_%a6jKtreQ{
zq4aJ+_cc$@E6<CP^^BO_@9&q>bNdi);y`)q^?CJ+2++km+v}Jhv1hFTD2!4>m4WaJ
zMdJHn`tQ*O_%J{trk$_6y9%gV-hL$fZ(Q6P2n?u{@xo8Koj+TZPnC^<|3OYN8ecR0
zm#zHv)CLAvAuJ#C;V3zFKy(r{&lpopTCdOL6_;Rb6V?Gaf|ZLsssqU_*|?^`jm+!`
zSg@gMbo}6&a*<1Sp#GpftWEamuiaOH7TW6OtmwtO==fMVCR*)YZgkDbZ{Mp1l5ZQw
z+AaWmTq9aL1(fJsMb278dO1wy^LqWP8+OT8-r5p`Sry1=Zcw}UIctQ!eMNu<F?ph2
z*&^l*iyQOPW(yR957X&_(Gq`kF*&7TtBiN0QU=zb`ne-*XMf}(W#C+%7;j>REXkn6
z+bxOk;-9cV^Uz7)doeopG7exWh*Elt$Z)W>d(0K@F=CBwj%Ud>hh_8nK@cRy&BWig
z^T&qxpD#?F+!e-O*qU%#j*n9?6#xAOW{`@P)8x>}>JLF=0TlFwjrUdD*GfDcoqWWz
zD5?#ITp<}F;h;o2Yyu4&??u#B88(y)<x}#85vJh&VR--dy@Q-tWfTg^Sx&c>U#tD$
zF|t{ob)TZN-iH!PPYCa_(|RNB#FPP5C<X4H_CmIld<k}1PX<1B@6aCLf$f#S#=cc3
zmu!Qa(0{FZb0z*xI`+Te4m0T3t-ZSkO)Mq;{9Y#Ty}%nrO?6_)zW@*adqS{c4h0we
z4PiwKf#8QZyeNdj*xN4rB^p$9`pQ5;f*73pk;1<bQLs~b+u^*);8Iup!xS{EyzQsy
zd&|f7mZbIFz&SNgtn*rSF_y8nJTekA8f%23%~PSgj|<zcz|5%c_jCE<g8%u2_YG#$
zP5Hkon)|OmdGnJnEJM+t4zPQy7yiJKv}?Qc!s0;SVQQ@iMv8>hSD;LFY|KfUHAoDu
z_CWV{`1LQ$Ca1E#;+lX3tp4^NRt-eM7i&OZr0Ma5hMgGPOm;b0JBMnr%|T~Z5U@%@
z(j!uaUS7co>ug%Wm5KLFIRAX@zy0K&`^5VZ32C{QX&1>T+8-CgBA1+S1iB)Ed!9l}
zPz>z@p2z$qVB05F@&ii|{I4Z5ov<o>fCkn6=<;u1Jq1f>^S4)WPSCWk{PtofsA6Bl
z8)m#q7E~_}Fr?gR6NVOf5Yck+(t{l?tv_j$EC%1Otj>*=OtxYE6r_h@^-t=!E=F~H
z?grx#;2VI4vFTM_@sss4@DEm+104NxXr^Br)f_HPmf1+nP~!Z*RJ?&o`@YXe=57m0
z%zOzYwCW>c1wiJm`&7GuOF*J)%x7eIGaSXa?323i+W3Mw_&}<%P7Yh{!(AD_moo%m
zZ{If1%s$k#e|B3#Cskj=Ruckwi1z6kw(cG+1Hb82Y?`~NZv%BT^fh<h7T5;Lo}r^L
zA(zd!1j{orzGu2;n8Ff4hbupvmuqThXh_LXU0uDd&W*ntTy|F%n0ZvWI-ctR5acP)
z$d(juiCF2gJ{VH(QAmrLB4%(9@dBM5Tb*1zmSd}6G{Y*-w99<-;?zSv08LDuZr8y-
zb@^cvPl7j07U|$fCS!Szx&pp6w0*N;KI~<=mDFKAH_ctNfBn-KBc_S@WmZmB`r)7N
z^i`-wLWvMJoP@t48spzWpy%0M7F@I|>(maLG7!mR5lDNPHxXheb4yE>MqCf)QyvMJ
z32_zxMEXanx&v?tYAwOI5bFp99y_;Z*%0&SHXwcLRYh)HkwV)Dv==k2b4!O`Yff+a
zpypdRWhT14mg509T<iqU=GKnJ@ApBqEL#N!6NIm)l2<QAYfpmqzKHmrK!MrSdCGBQ
zF{7`23p<iO>2f#Rs{;Hz&p0zM1$@T^Xh15*QQ1Vbc(0fwK@Y8qM-Ce!#nx)(D3)}h
zgWh5V{_7vHe5^>v-qjcUk@^4atztIV8g)vZskMtalTyhskOwR4v{Oqpxm}UT`EkLX
z;A3&|(Tpd=u#vAlbf5OBLl?hNTvksFfWr11ox<k@&;<}?U3}+o0^cWj8x&rGuO|ZW
z4$5*61dL?h1!Ez>a*;cOvW@eLr66MZDqy)>y!hBi``zbdMI$~Ul5bJ=E$3<)XMFnz
zOtYvquB^K|bRpZjtlEE61W{Rzf<TRBm}>vqCI7FtDW-{LO!=mLDIf9Qt>WOW{<#P?
za!r|<=^fS^2{9j-;y1^#(9$>}gk!UkHtOzpyo;=6#Bh+)(4&_}O~gtSeQDD8PPw$Q
zY~p%o@+85kVV5sojg7tmV-XzzKv$)0o*s3l1CcJ|Elz;=V&BfV6~fhR#$SPw&$6DZ
zuw(5Acr8vSPHpTCY@E_3M{y&!eA+%cr$}}}fp*0r_}eE)?QH}IYP=kn^C}GGG=n7%
z<Vx#^5T!GNK1BbxVcNS)S$}!M(vWt6l{Y2CRuBF(ZyNRmNDiD{9R5N5-SxX@l*FVD
z#Nj}P-NqS6B~&x2W>f&7@18LZHrVl;HJBCVXw)2Hmjq<;uG_h6RW4705nt<xo=Ec}
zP(-OxPJ6;XUU)F9-2(ttop*aH4?g}{I0>Z*U~0GmsD-P)jwINp1cN86$4ktu8~m`X
z!FwqXRThkO5Gd=@D%}tEXWIfsY`z_Abvmhl9Wa(pgovVzR@8gm8$jUbdxPhxX=L#d
zay{P}lfqU^@qM_uOz+5TWm@b9E7_1VPO`gbS*KMHDEjjtrtg2TB)AOTMR)?}Zm~>d
zf4&pf9c_3!BH3m%={g#FJp@Cl_o$iNrUKO?pqU&-x`mRvQ%K<J>*9rb6b_9Qf!d<p
zmOr_*Eh@g{?#e0d{=C?i{Z@9^M7EdYXZ|~2T?72TOy)Y$gCl8Z8Nsj?k=*mTys9U1
zUbHX$SJ%QupxJYv;-uBF=Y(Tjv-?E-x%|+>{Mj-X+y_?NEzki*-pR1W;tOuDm!BU_
zl(U%rK-^BTK{*xrBoBLp#supb;At*;g24}Ea!U{XVKqdE7e2~yxCGvV(3d{2tKTJW
zy?{uH{i`30NR$V>DlC6X`_H%e3twG7lIi8$P`Y-NR3FXv?#k&-BmUG!X{yykkG(dX
z{`=BlZ-87!NxXSRT<WTI?SY!}?oSfwkrSM7fi;HmCL+1xV5Spms|*M<YUVi|SP=9l
z$(H1Nwmz`f*l+<F3F=hXY8#Sq<rL_PZL7U>9D>cGhH^}*DhV$}>v^8QHUt-s>i{*T
z0v%t7T?<2k<o<>`C}0;6G?&9YtK)@S?YVPV7hLix=C1S3hj5M4KyN?jwGoE{+f_OO
zb85;W@w}HQ96I}o60QxzCx;uk&3I$V2K3TFcxS7FV<Au}2FIj~kmT^%+6(0R))(h#
z^YLBO{HGRly-qV6bsf8((Jum137;)$em{o-G5;x0Voog28V(Zs1pMizWVRrA=6}7k
z;IB+H&xXY_Q>U272??(L{_z$eF7VV7y#_@|Fi|R(nSh?U0MQtEaI^@b*7}#nM~$@L
z>+SJa;<9eM!TSca|0rPVfVit-G^M6(wTK!>)ogPr@fp;MMm$ZD;%#7++O|KgO$}zL
zPN9CMaRi#-hbg_zcCG!%Ow3ra=Yml?^M$yoD{5!n`DLHjNdJh~aN>MD9~fxdZ$OnV
zwt%y?)3&K~gm=3(dBD$>c&9*!F#E7-g{FMczT?0H&e(D(B(9h9O!9$|6S|})qadR9
z@f0X^e9+2W#6~;6ji7!1vD(G$CIX0Wi`u{0=@<o5BNkC(HzeZ0>42p=64ZVj*}nD6
z2UgS+2^)c^964em-4t|Bg6n&U_0@{-MUIoR?t)EV!4o)EvXe$F<jgK3T^`*I-#Z7j
z>e*?}Q>&w;cpDRC7$e-WxGNfaghWH%+f34SqO8@_rL(4jk<+^sI9tpkKz!nL;sAN+
zk)_0~c-%$cXWQxU-d;17tPur)MAbVI-HK;%<nPYmYzr+j0v;#b!OVN};WB)&A--2F
z<Nov!|8#I7B2e4LGr9{-GVdghXycn=Dej!hUz>1l=4Hg}Ua^0=P42Y$^!JuZrU4#i
zJu`VbN?o^YdY@*6?7N7L)Ny1e)}T-QT`c5I%lUrNWx}r3tB+EXD8<5x3&NAR<x?>I
zgHc;)!nR+AzO<kgpbL}Wf=4wJo9cjl6z!gZMR1DIbu*vmbp;P7H5C5BqmW0{2gYQ^
zgilD|Eq?!5*$oO(akwq<zTMox<KBS4o;tfDxj|Dyk>C|SVmeAcRHz|byQC6!X?zWO
z6HFin{DZLjRR*1b8jj#RpnrU)rL2#08F<<)76u1lLURyRb~BgmgAK#)dJ#Y-(@R3M
z+GPWtU*n5PP;Og;Jrvi2uFKOBdoy9pAbwoA-*Xejkw%&-2zSXyRnmP2o`9P*I4G^!
zu1nucr4$8&(^%KfuGPFs>3P&6w0|6o<8`^n<`ow-X6XwYZ`B8Oe6_$mC^@fn!e#NN
z?V4vuFp6WDfM<!!w;Oe0-n&ZWBQ{Di*TFT48q5<mHvj6spYH#!*S&100HBI-1Vk9B
zuN}^Ab#8&vthr)pr7x{mrpWSToe0Xen^qgL*pv>bt&dojQvcqyR|&uQy*)TZ-di~Z
zao8%sR?TiRdD!;xFz3e{NodsbbWXBn$lAk|#-mr6Uz}bK_5#z}Rs(LN*#<@+hUuL+
zhT=QS(K~?9rxLZTG(Uk8lWX6h9N{tNz-2er<WG$OAJH(nAu2Nh&+SS`yapY7ry$w=
z0KR$lqL)GlsN0iGfb-UILvYY>k-<0Qila2;XTr$uW?yByZT2-AopIn9plr+E@m*e|
z%m%9Hg^E+r@2!xB8Hy-&>Ea+&l$%koOf_rSXsW3?X6OviD_fKAG!>*#fqX=Z(J;i;
zetu%wzFS8uPW>)TobFZO)T>4`lp-iz3p}+s#|2PF@+XbLg`*#Bq1I_CknsMb1P*KD
z)P~)aX64cq1>eob%Ttw0pF4hx;|5HCRURO$M5ljMX0YV{lSFTF*0LJJeAOr^>Q6Bj
z9*1M5lsAZrC2V#0kaJkVaF6}UTWT6VKy<97`4z9D0(fUEO7{IwO!f9{^01#^q#5;8
zB%^B4{QWMVmj38yDBtr@^~YukzjHErT~N7@uQzwq1wL@B@^#&&12|5c;;pN)&O1}X
z3Eq7TeDU&VqZoW7LNf8Ik!I71M-HgBOMNmi7a&BO1B+{Ge|3OyAA@=0Y?fkC+(*%?
zFOVf+8$>~da+RxhPr1bHr-@)^K-t8Jt!76eI`Jx5H%=Z-w>E~Xz@zqEa6X#Oc2sWh
z@m#nR`bUT^vyTX{6_Z9XzBYzHg<?_A7^~)hNUx&)J+{<SFj+^>e9Yfo4}|jy9hJ>-
zivuh}7b3?qzM*eWK|%AEFwP^f@%)eXrA;;oH|#EKj%JKb$-gV<X6#9Patc3;6kWTW
z7y0Ed!Dx26w~N1NmykC4FkWd~%}X?40BGREOsUARz97BU8RW7+Zso%kHam-M&<f6m
zVs{K^$@jHD@uypv1@3_KQw;q8wqVcpZ5><j^0n{TRYU4}wRWB=-Gi==Zu-Dzj&!<r
z!Ex|NP_g>Zd_cdourJRZ9!_R;&ycDt?S=_KfJ56jZ8ziP=itSU<sti-?IKSr*w#pt
zhZfo*C_#szX6(s1MIL@w+Da|Mq;y@L=oSvHhcf<T_J%-a?Gh=U9*MRkNBLIOx=CMS
zgZ=Gu1;MUTr;(S={9AJD^Q4v11=QVMTfR=+&jJvk9`hGZ-Jc;uWXQ!R`{-wM_K(`^
z`Z6pFeqf9_u5M?-;e;+f9+adIAnfoa!93j?rm+R`vo>pn=LaJ;YvBRrhILtb;~wxe
znw9w)gHOcWajDHGO(*fr*8N*{q6|hl<U(9+|L2*WO#LgE)}lKjQX#&PxwlYzecQ#Q
zuAMZ4Z2J~+HqiRCSP2^$;wK{YJK)0j2popnSa^fdd*_Q|)1SC-o->IZ@xAj&Lvu_h
znFxmz^cCj0MrO;Ns#ijq>&2C<`DS$tYCqjUuc>QLp(7|r{VkKR@2?m7@ED?~4PW^I
zO+@=F2h}Rt$r~jJ=mowL-~ha{x{2dCzBKQBc?QB#vuvi55*`P)TZq|4F7y7c7|b4}
zAdW0(-X0~PVI%S;d|Xl4Mi<ZhX;wWTki0Rogq*tmNe24iH7|+0%Q%zIStUv@=mXj{
zu*JW~$2#|lWAZn-Y_ilSR$}zg>)5x^j%A>f+CGF-mN1tOAH|4`@e~~Tz-F12=1W%7
zlf%DxNY2Y?=1W2YPw3=hKZJbp{;&w-(|Qt;RgKq8T!DL01(}aHR{4CG;FI8Yjr(9r
zQN3Jd`<nK2c^$ac$~rzIWy>Y#Z*931^qr8n<zCGX?9X`Hz+27(u9;*&{oNh{7gV=O
zM?%Yf|H(R)>jL2sO8vjB>TrM)rwPotOPFdhG^X1`4RMs0T(ofpnYn;<YRWyA87T<m
z-3IabNUHIz>g_~3dF*5?DXTf5c8qMrkbT$mC^cE@@$4$o07h}@4%Uq_J$8ZMEQc|r
z7H0u!X-dcE6?_w>EZdwuB7Q_~tc?zaiyFgk-+g{Oi|?gjMknp3L~c`KC&%MQ2M4l)
zoCX-DPX<4k>IA@nx(;Q7Hz)doH+=bky5<ZB9IsMCYM`T4fJZV7{_}h%u$r|WXGB8{
zCs_X}ki1g^R;TEzP|3aP>rEATJz&~bb4J!7i8ysL6jRw*a01Mw;!<{*evBg_H69YX
zTkinE+h#S#(vX;M0ayt}zqJ5#wgXGwlUi|X9v|<{%fa`}Ch+)Xt{efC>$0g^Bh#p1
zg_BOLoIyi`_+dhCY%eojq9&QLy6zP}r92^bcv&BM`tbmIb};KqWj2P{FrX19BMQ92
z=y}IhrO+E{5M731mhZ%F@UFUk+n>lvJ+M0WEi`H|@v-kHo-$?Ie!SrhlD@IhF5&8x
zjUNflxD0^#r}tX@%n~Y^CrUNPuSCce6Li?zgg>cgxM?;0<DR8S>Xj{R*Jl1k;Sksz
zC>0$gR@e4`-})JR^3;rN`~YXn-yQvQ^SQ_BcKL?0Uu2euL><^H*_>C!G}E^09U1_q
zU(-z0FA8{ErQnsShdv_hhd|`*mcVUMLSnt%u$GYhf|p(_Wt1JZ&kfNq{zZ)U85QFx
zb_|3jheJ89KHPR=r@LirSqY~T*Q*P%@cJXPctHf>c{T_%FedgXAMFB8xbIOH3g-#<
zszeeEG*wxT(V%J`2Csaj>d~<7wi`68aW)9~y*-xxfEjV}Njgf+S)97%y;8!-CV|UU
zxCDGh_2jU;Zc1ZNv6tj}%P7I6PU>acty`R$NSdT!6UBTV7%;QGmv%pIDKP9{cjH`%
zpkHY;CGvXK=`+M_EQ0HK*Ti_k_pW5^&UWa%2My^8d12HfKfaHX%`=10L@T<}FX1>0
zq|jlLJ6d{WH*!joZTNyu_^i6JzA!w0aR<FrlFnlJZiuW`2Wxqb<;X@%+*PyAJ=|sE
z=RLIrk@H@>U`Z>10t>xq(|x8X0BytyQ4DXFj_8IW73px*?$*p@M8!DE>xl7SnGqO)
zQM95s_Fb24UD;2~79&TQMV(`U<n7w)X8p;vK1CBE?%fd+5^te1-h@D5SH6>rQ_p4x
z^%58B4hx^kVIx=Gwm@o)C7rQu8XD4c^Gh<P-0C_7Sb)tN`9`7f^Cf|d^PRMycN1SB
zP|E;YKc4U3X8dY`uIvNF`Ugh>C7c95zzNk5H?c>P6u2?-nI4Zei3tM5_1Az5Swo-+
zUP2YI>En7$900$<u;5cykswN?2Aj96Y&}Ei^7ymbo85UJUe_+E@x{ar<hZ7Nn8B7_
z0}8yAAjv9!+MmIlAUb)v&!Q$BNHgj**A%1~IaITG?zp+u+TYJBCQo5;#Hz3n;pQiZ
zev>C~HZwaoIh^yJkdIzY7ouasOIlLNy$SvPL^OxV3keF%CO0Ux;bC0S=65}7jBqgS
z*XATx3wAV*H0n*bTC6y3!_5A%5So&~8V=2%V+l+zHAYC0m4ss>WA(45Xt$GMrF$^m
zd-GZ_&tc2k_~-kqv-d2SIr49R+_mmRvaT2r1!;UW)lFn)@3M55yfoUpdGPK^Oi%1?
zzrR(G&S#>IA@SapU|NT!-Ed#in;4^UltfP`&>Kd3`MjbvP2z0rh6C6!?|aeqO_E(%
zq`3Q<MVW=r{}e+t(Ws+_bf<j8`OjGS&usTs+RGrDoa3w5=h65EmW-b{43Za4`To85
zBQsA)TpCz`)sSK#tDquFfn5{$p7dVH+#t&mjo-<W7{nhHd4Io+*u{%-&N~(`tT8qA
zv7lo)g40IeaS(|NnK*QE1l)hyjCfU}y(C;fwmluJ7-=yvxJpcoxyI9zRA}a!<&N9~
zqa+>r6#O4h!thXG-JxRA-tAilCx#;3(6bxgJ(oqXd|+JNeD~vb#+dL$?fjk`QjFv#
z<13Siza~)p`~{pyVTWI3i^0))6ET~%2l9~epCk-)^@unCMGlyqi`?`P@d(4B^X*fd
z#~jI<GBAXqi5o=5$?Re$`_xhU=RJ=2V9MZsP*-FH7^u+3mz^z|lg#4OuKNUH3lV_Z
zADtwV&7?xz9qan>B<BiBIVz2C#1dMb#8HUm>cqTKjp&yv#&R#*y9d*Fot18w&G}BG
zp-?dtOM}mRpMaz|G8sRGT)CnBPS}KGrp#veSw)I+#`j!Q9Q{a=ubtTBMrm^R7@3>q
z(GCbzR`@mwTd2aYcvH^jNb1bUw_;XIg&z1lX@L~_h`I-)o71n%izj>&Ljf*U{eASl
zbXc0v03uj91cxJ9Z1pxw@F*e3<hW0Q=Abk_?rL7($zFHc^NS06KMo<>uYNVY3@d3b
zgRm*j=EI<R0K%ofkKkC3Ne&O-YO+SvrE^HrqwS~-o(x9bV-Nt7zKR*%W^r#uJ5WEV
z`Ruk5ICVd6*U$U5pPn&mepqX;H|hACqLhm3vf>vAbdb5BtjK94C4lbq_U7K)_*<{U
z1Eh##b$~{>w$Wh<kiH{l9Yd@m8)eWq^q6RL<bK%Sw)L9xjNn9eNLp@jGqFpIj6&&@
zdZqQQT+PC+R5k9<_5^KSRiQlN3~VyS>w%-u&OL$HNV7)6hJqT2duB-ha#AJ2VB=SG
zcstAg{@X~3T~`2`oeJS$Ea_WuGcJsM!o58f`O1m0XE?|TJVFhB4_7q}o7#iVNM?#2
zkY-NwUpBc0U`4=wT%Gka+rifx-kDni3A}a9Op(<}HVs|m|B26wv0<ZN`4Se??%4!6
zQp!M%zoj5RZ*Xg!Sbjsh&#nU0?%lReb=zm>Z#8mdLCV2_BLo4nrxp`4g1-6n2z`06
zABFGy?}i+;`<h`*xLkG#BCMzN>QDd}h(Wrie=JK~$;P?gtwhJ(%$pPVjqT>X@^US~
zLp+~keCy+*cNoN6dLh!}1jtXw^|9u%#behpgcZQE$6gIy6fwcH<N1XF;z>w)^>!>S
zMIKhe&KS;mql;{>>4UkX$M;s!HsiR>Rg5^DJw{bw%aNbmG7?N6XV~YlmS02ycmW`#
z&w33?v=8+lI#TIvp+eHGWe3NG8-)ek(wNRo-UiWJ#N(ILTn#JD!;Zv3op5~oAj!-i
zsqeDb$g|B3PkK&r!^&8~m*H3(wFR6$u*k>NdU+`~5eI2o{KJvS;jJN^SZ|M+xv4P5
zrCNmm`a`koXeN0yYmfu--BT$Rt#`X0eAsP4R%+As5Ww&<8usFe1f2O$hRS#r{)YNN
zN1i}X93UO#$@=DSPK3C(V@@L6*{-!fZ17Z;y{nT_l51m_fX_C!SrVG%{)+cheu&UD
z5oFI>Kp}dI?sTEdkqAMHGuC*k#5XX~$|=KEy%vVg@u32_mK{*N3Dy^2JDl1!b$NA6
z+e%lGCksW$*IimoZzp|crCpg%tX<8B=b<B9HK~)lG&T&=y@Ov>vh8}C7_pcho@-R#
zkIlbv4a?k&i_j>Hlq0ne+mTE$Ozdmr^Ck(J!6^_ghHjn|&G_I5(5<jP{-Bv5%01`<
z({LpeB64MSN}A=fsa%ZC(7dnN?7DI^F>_rH9!o*<$n3U}+`{6|+uEHy67W9~n}5!c
zlI>BriU6xvxsOOfpp{`>@9YHK`(g%m`Gt2qc*I4V)6#BqLi_idzJ{UG8|@i++%kPq
zR&4fw2$BBHyz)25R)3cO#8FW`FaWEU?t-2&cD9>nG&{1`Pcgh%jlb5lyBmgu8pU~C
zoRIslBU4Z_?lJdBcl(}v3q&FF3fn)6-(9=tt1Al%cg<H6@zob1zDam+AlHdm49YOd
ziXL)-gCiD=_`dqL885@E4{fZL5DY8(X$*4_KbxkG8}v^%io+ZAXiaJf5WBcxGyIG9
zNtpxcLhR7;#GBDonn|zDqi?h%ZmXI%E{3oc;P2VccWz1cwrF54b})NsWbE*I^Tj@;
zFiPDOx-16*A<kQ%RNK*?SJE$QU3Y%i0gE{wv8s)l(qj@n(ixW;ztZe!`RVkz8!MY4
zz{WKM#gm#mF4f0C<)<(+HB}_xGN@)KsIOwm>+-xZ{!oB3X;X++ZAGmYv*ppSMx)aJ
zLgqQi%}JPD;{ERhVP$n;+n}kwe2OVMXYRJ1$FBUzIf(MBz^LF+VE`Q&^lu!|#aj~H
zlDbKVAQbl1L?iIx$iCM1gy7nu6!B+HV_7-Ow_|#I(i=6Qw{A+C2{2E#u88&O6K2hW
zhMV`aHd<;=dPhnOb{d;J0iuC*D^nqUxD~rlW93z+ER80I(L0+2l8YRM^Y8BMALv)o
z@{E*yo`l|s3OYYC!m7LfOzP|HJ0C6eRn>Z>Pd|)cYgy!$a+)3iCJx6sf&B$x5g(fm
z;m3_YzMER0F1CzCh+79Vd-+`M%2$GlAs{kQl9-Gf*Miq3%1_C0E{<@%NHrv8=Fwvq
z1<F3=rl!&*jTH;@o8e@mxr1+otK+8zPCDQZbBfrgN?4oTRM@1-p7^tO)!BR<*YhGD
zH%z_73drr6(igie)IAjeKG+lfC50<4sBP^@tM$B-07+Q-2`l$aeUi%@c1xw4U+3WN
z*VH+Cd@*@Su5R^Ip1rpU+K<UQOIt134$*ifx4=|Q+xeDod=h%8ZO;aPu-NAT$Lcw(
z0Ox-=ekbonSm+PdZp?1!E%MErJRcF7F1t%=e*0I$m`46ok~*8XRJh4^O$_wIo@SN!
zB6?InX}?_RC(C&S{!4&pg_%`y%va5vrRkErNoT7sy&B9;^VzBn0K!#3$*&#Y|K}AK
zpfn>BAjV=G7=5e_aO9#fr-~&ruRIuX?bSlt^Coc&4yb{8h0?H{=vVxoGgu?&%O?Pd
zyG>_x@`VVoI7EEVcpeOc6gMd=P2$8!(ils=UU$0tFpAgmaSHiS9Dr@J%_BhQT*&Yi
zbU-)-6YQg~2BPy);v(f|VfQTK!8_Ld@5^uS#pAIbE5Az)7ZI|$-a}i0UqfM#94sPR
zO(NP!M_SgFz~x%(%SEh18W*+SMY7(fhjXep$rbyN)1)xa6G6Ko*#6J{<u{-r35w6C
zf~Hoq^)+a?(B9L-?uc`)X-mBL5SeieiuuqQ91qa5qW!+jMa0PK_Sil79e~1CgO2k8
z^UlG3_U6uGYNJk4MUU*<vd_80nZG>`sE0tn+4+KYB@O<wDP4ofjojR}tfhx7LC1R$
zh4z(EQTq%wHTYhMG5Qhn{s>m%Y-*?z7AHanyLk-XjbHSo<Cz)#W;)$*+M-sXOVF?r
zMu~Ae(QsFY<o(j4o2z^QjG`0_qCLciEE29r#w8;!Uz(M)oMQ!c&5QjW(Vm`mD^UNp
zJ^pgE=265`tSy@QeJ|_!-JV`zI{DTjZNng1PA?w4Z%1G%df#k0HJ9iVU>-*9wuEC{
znkSy*=2g;b#E11zh&+PLa``9mBrWm2+ZO1V658$oA+P6NXWa&<XLd%<dbJ+7prADm
zm6Lg$`WT8jwq{Hh6Z+$($N0jLCvEdM(kJXqLkYacmp{ji^32c1cF-SuO0l6JF}ix}
z_xTdP#TH0)y=vPTXspPb)ecHY!#bbrvzRIVS!vK&!P{8>hGeYy+U3cC1ljHn#g`<I
z{JUnD{ya@+R0W-9IrNGNq?Ce;sx~`ZUMy$rhMpS7cjjY4X4&d`C2)-4o79fpmvl#J
z9`!pM&ELGZ2@@V4bDv^Juic(PKYFRT48ZJ{r9_BXF&7ZE@?e{aiK83c&|fygu#qEv
za&K)d{Xk_ffSvWtC~kPVo}tKLJW=Eu=9RQY%4zORKBJZZbWcl+Yj+w!Evv9CWoS>l
zQ|whuWFU5?Wi|gO4@f;xEtpPRcs4oQ3zHuzSAVk55#5y-({<mxC6~yW<`Cq<ZZb(K
z(PtE!2QpONr^Leqn(r_v*FwnT+Od*z<g<{ES8iTMT9Tj2TWLE^71s%gQ-3MXEgNTI
zy7SmJ^8Ld`914L~DVhsU1W^9-&255`-77>~5~KsJp=1sCCdOEl^Q=}ZuXP2%zY|z%
zB{9(QoYZHu=p^__6vV<UV9sH1Z`GC?RW2d_V&kP<VGAWv;4m3{DbJ$1<Y7Wx==Ow9
zGE=~$<T1zcag7yA((}^}x3WwH6layybv{Coe&odlMM|1E+vOc{4@A12j|hZ{!;WgM
zhqZjNpG{?6u-L**FA%%LJhQ<a9i1~U$xwp6|GvG?V>8_GKDzzhTX$;Aof`wnJ%<Sw
zVGEz+8d8nx_^BjigQaMg?Y!oMmf3pN=vF!eTL``ShSNh%8$r2)+$zT9gJKt=fv2zy
z18GMsfT|RN64;pi)2y;!zY-aQFkeRROv}tW(fDTo4>B3#KSvBS&b_Kr_0Vkd-R5Ya
zh)`%Iu(mwvuss1~v8{)7XM47GhZA)70&~CKm~<R@X_S4Z!_Y@$oW|tV`>DsP5rs-g
z<;KnE!o@N_>C#Ds-QL{uPFZ|`h;aH4mt#3BP^JyWgcpma+V+0{E~F%GR$q!`jmk=y
z^ifn?2K1i$#&Lz{7<zT?vSy69a|jaaU2ycn!!ZB=IS_JKxzx;h?j8O(#N|6F1Az5X
zHx{W;i+(b^#jrk9GuMNed8LE7JSiU*{J*kKz%RvkA(l1y+Vnr2j8_7Z@msW-xA#J1
zmX`ozmGQu`QG<2<jM1oOY*?;0CiUy$ANZ0Z9bl+<@O@_w?hY`!BKk;A=<SD4=e=Xz
z&$G*cN#RLhfiiSUX1to&i>fV3lJ0c3vC~Fpd7EFIkrEFy)79tCbDS8bQipuQhhx%>
zN)gPDq$LesDvHK$9zPKFyv8Al=M^vDsT73dd)VZ;GkZ*MdY~o7`P4@wTQ1MRgJI*&
zkE5|8wx)8PknzAvHex{|NGW@rm0&nujE6z`W_G)h1y?mxEsrVbd)SR6IS)?`y2F;3
zcNXN-2HmC`gQ`kaS;Wz&4IIP>Lae(=M&8@iCzp0zGfEUl=8xrMo-ru-`GE_<fE2d+
z-jx{~QwhH4xa9EpxnqLCl(8Qz?8e6!DfqiL@Dk$bB=<*7;FZI>tdfH*h0E6-V;%Xz
z$CW(yBAO3omM!3ftj!SHGLbXfb$m|_xO<-X*-+QaJF2UV`CT3ceXhsQcZzFW;gDAT
z=E@~<C<OA(<QR|WRRiy(N8@smLzvOeWsX}-O=a}9#Ax-p<-}#8m&B}O7iNK6kFA1}
zx-apJV()$X*;9h2R<`*)G?|TdDeZn00I?4JSi2(}w`LCHdNC(;1CeWVOY?lHhVOuZ
zxw-~OB-MPA<o?8QkMJa(@PP3)WtbvKDnEP=<giMz5HTvr$r%$;E*jBXQ14dAb^?`A
z`O>M99WSp1{#tFM&{aHCeQFV#!LrU6@~lij7Pn8acciB(GpA7TTF2_(y_NfGMl9o6
z)0IE?n_19P2n#Ve)HKE6AeV>@qG~uh4y|~U@baF#bf$0E6L}3Wa<6XwWp1l}#om@u
z<6>=YtK&LR6`BW-S_A30Q!OSUdP+=z=+2c7oRE81jX7MD8IM^Wi@>7zlUzyjf|J>X
zGmcxL=t)hSf<GAjKdgOqT$J0|wuFS_fYKq22$IsxP|~4*L8p?^!hrP9AR$PHfQljw
zf-}-xg3?mb-7wU*#`ErdJh9LBJKy{LH`^`p%z9Sb>$<Nn_K~lPBjZM$aPLUDM-PZ;
zg=lh2NkZ;%Le;|MM@2}cys>T*j=4C&@jXa2T{d@l++$falH8k{G`u$Mi&62(b`NhD
zt5~+y7o{>OcOhS9c^^JxkoP`{rp7YB+p*Y(q9o-cS0SV=*d=()K(;OMLa5l}kO_x;
z1GJ!9{nopHoO~T#{8AveNEm*#V1_Bhb*Pwen6Xh9hfBVTe8lKJjx_0HsM1CFYKI6m
z;>`&?7ArKMN07lhHJZbUU(V!?tcw4}bP$QART7U)^8D6B8PN{=<&Q#EGO1@BGHuj^
zay4zox}c@8n1jY*G+HeB1-~;YJ+u|PIf_$ZvM03@<wtPpYq2$i!$&R3Uq~TCG$nSA
z_O#qxQSnLQH%e%k6tB-(OW6ENR4^8Vxha4K`}I(e6KG1>3EPDtxld!_#DuUzg-Jo-
zg`qgLR9C>R9j<uA(iC^55U@f1ta|?O!!uPhmR4vrGkGQE36+J9Rr=safl9FSSF`Sw
zUM~)dQqg*W8&thIR9%g#q23%8<&AUKJJnH%@!Og|W&}dpJq7=%OBD{lHD8W1$IjRb
z9E6!Vo!V3fAtM>|A%7m@yt2Br&rSoW%TP<W)V+ft0h7e`W(Y-Phn7~>SKr6*b2?P`
zvE3>v<8c3o)pUsEJp8~<@}4!S?+Ew$k4BZ)c57{(tNShp-IBAbyA6({mtkfZa|tU1
zZ?@)%&K_;QY4Rr%oH0z%D)E1iW_Qk)u~G8YSA5dHhTfc?T#Xr8j_<V9A#6ewXb?p&
zAv__(wWQD=$_wcmTEJ42aOyifVuFZbH*rMDL>*kplFLL+dCf(uLv0XuADSpLcw6R4
z?O%o+jRTKZ|IVFhMof#`OS2~qfr!@5SVR=HAp5R@jr#MlF=>@1Z{f#7MM_V76K(-#
zyfu+|3;FEV$@jVHWE$2A(i|V-ZF@|(D2Mq&_Mc$NPAT^_+u1t>DxJ=ym`)KgD)jO;
zZ(?_SL3T%(a;Z-n4{e=QnW`m5?u0Bvu5INs`uB5d^$)9lJ{}f3tk<TUeHkJ!%@Sf#
z;Dqf~_M8wq?aSsBrK9;YNJVhA#U6(!@2Tq2z}x&&QHK*_-`Tn_C#kVC5Ul(p#3R5D
zgVB}w8X0TuuN^1^y#uY@)=nQ4lUsfKeRRV;SX$zoOA+RNb=M{RMydR@K<UOWC=u~F
z8l{J`p@HxlLXOu<(_~X(XwA7$ZJ;AP$f~OmPT=LmV)?<Hy%9FDnW(}W^{N$1nfo#2
zaX$0Fi1ewJBj-6$gUN7scC3ns+m)T-s7swY%ld!Yfo_xr0pxj-eZfmJET)%IurZi|
zJSL8TiB9teAd_@6m^x%2dBr7%ydi4%<f9aF;UOjqo<H`3zI^i^VS?Gh+q~Mtjl9kV
zRaqi*x<7ncd24@=Fe2_xEqH6-MwJ}C|D&}rE?4Pc#ePb1V^mv*C&Nm3CY7K>)|opU
zZN~Q$PD~8CMT2X}5l=g?T0k^t9GyyzF}^B;;FvCsV>jt>bMEmrX*`+mNR{tb{zYR_
z8j@!@RBZ0C#h+rlX1dSVtexll#(GWLu@uz1yU^GSe@YjYU9m>xoH~A+*Ag3-BWG^u
z`VllmLxSqQdirF^2`SU0g@Y*ZB|_t1ELFczh{Wjge57wUTe9ug2e9!B2xWmQ7?Nt#
zli>x#Y~JWjI#nH#2~&xWf#&Qs%0m&yY8R!Q1MWpgB9Eh!pMJg5c;s&@1fl-IaWjSQ
zbG6nN+b-RPkxg;y(w^bcc9-pEC@op=rLQJdwTO(dq-k`R^qP_$GERB9m}1>Ie|q7=
zolAdqn-;!17&=v3ax7?BilCJ<N%60na=spsd7&HNW8@WEhEknxyB<fxhq9|Fc>g|6
zdP(o;#bNX!*M$|uYLQ?j<LT%Pquyy-<2p5(zwU*(sr}k?PIfQD>Xjoj#+DZ~_Q;yu
ztraiBB6+G(=K4fhr%y`_)AQ`pN!#RHuUgTz_9}4L|4`M+5U-49n>TT&{6>S^SN!{~
zh7*9s^a0Z1ILNN4STl2ie>|{a{t(Yp-lQ+eb7gPXtB+A>xlG9YjGOCensKv(Vsqp@
z0|Y$&++9M(Wc)*K7NV%HbF`%0w@L<VXwXd=G-n1IrX_P$(d5bS05Imui+q)KfEwTN
zwDsZ&qw7U#x|?5fg|=-t$q)uEU7S@I9G^eIsbYU2G766b$us4T(?FlgPTn#T6y(pp
zK!D=b!XziP?;>G`TtTHg_X7m=D%FGV{!Vr^Mj1cfgseq~GTlJG_Vha?F<284o{dM<
zAh71eO*F7cb@D}v3Kr`$Au?kTcxZT4%6E<X85_bd|Hfl=R!58%r2R8t^xqk0jZWK2
z`L3AW=^xhkJfr+Xy4Cj(%QDzK#EA*<$}bh-E+9v~S|s9I*8gETGNNVWWhpHIS#u4M
z4hrQkaT8CU&tXicdf3><%6zb1VA=p)L$S><T<EX4=rFwMZZ-bpJ|4>X4hcQ84^hM^
z05o4e)E|0D$y4ybTslUW?Osviu_&85v;G^#0d979uD$3-N|$VQ2d*6r&R`XMl)3!%
z9ir&8!W$K5?tpKPRDe0OUXs?_<Zm*oo+P+DPAsQ?r(NeBXi?s9V4Sg`#z=G+f0JF3
z<*sEkj*n7Fh&Zx^9I9Bf_DHR9mi5Lm=qvRr+pw_}=gw*GDH5BD7}K>ton~&wb1V_G
z_<Ky8+Zg}s)b3ZtO;WMQ8R&@SAsHNX5~l@bpfAqseVL<bPd4gkk;l3VzUVrnIIjR}
z$4|RW@+g=hQlF||`P-&`v(iCZ5Os!e@3v+hMj^BsS{+&0Q#$w4jGgV2pznA{s%%A^
z+5QCW4X^vKS*^QfmPEgbb%(;4-X)`@SThg-FmS{=7IV&bJ5PnVOuP$83yCU{XzkU5
zY^PO}J7Z_$TU8^k5ewloN1RVSurum%P3hVV(%U;*bobT!Uq&P=PoeE)uav{jtqXF7
ziiL8rdGE^X@p_ZqG6$Z{DHdBC6(gT-sO#;C(1qJ}7E(8ArWNPhkz;JsphC4~n`b&F
zNZ*<j62wt7HIL!0nz&%4@#KUW11M*VSUOI*1RI8KU<q4XJLK_!$ot>8I;evKT@>D>
zL;|M&XshWpB>#pE;fs833RrAKj@JuHL+B;K=+TDMVvHXX9MpV7?;lNuEbsru^Zln1
zL!1rGRvkP$0<<6Rt;`Y6%$v+h7TrAi8_7l!>yNlMo!1``TLedOm`9Bt&xQGkdH30G
z9<wg(b}ZHX0i}G&MuWH_e#CS)&p4*!<xFaRXmuU(<!0fdxasAw+4J7l%~UI0#CdM0
z58aviM#&sJuUs4$7o_!dCQX~N@a$u;Hxkysj@g}p<U4cRy(wEi`ew?=SS<80ecx?9
zzE#hka$vRl{;}E-@GJZ@xk6KTD?zp-1eZ;?SC#t106o>0@3h<ep`F1fuLIj5rmqmh
zB_vS#^uhwe_jHb>Kd=PS?r}PRY-9AV=LZ&Zv<^f4M10;CfDiYIjm7BzLbG0dSuY%a
z161h<LnqJ;?qX*p*<GyeTRxhLb-IE>uC~dmz-l)~%VaTjt268vQ*2zCZE3fQe8FzK
zV5KMcalXP_;HAN+-PjYE*)~9z5qhAWjhmK@-%$T;>eK3naXdFy&}R_w?cF{>cgDE8
zfU9Y1b>w;YHv<c+EJQtM38dOy6F$N{iNAiv?5r`P(E0f)leuU^Vpi4ER996%CFvdp
z{yqmJ6J<xX*J9_%vhuDxB{okZ{2JI(UpMPbNZP_n6GfYH4=uy5oaQy`DF#-B^A?*M
z-RTc=^}6%i#(TTNJ)GRj7Mw@67H&%N3eZ5)>ZsSj@pP_7m>9|{iFp#l)1{hfhRFh$
zoa0sGfb+1LSys@iDw9i{Zf}htAuSELoZWxO<30gDvNr*k2GE;^jRSVz=_AXjS_XK<
zi$zjKM_?s+){cfSXUr+D*n#nM6W+zaX;0U*;wpQ>xum9eFNh`gRqg@k=+^<}nbfQF
zlLE%&x829;HXQ6#iuxf`T-I-F)4j(HrQaM>7cEh`ya*Ytl02rnBWJSWd)U?H`GI65
zj#y}Y7If*y4jx+6B`$@=2#}a4Mu*}1FX)fovNt%nt6U*gA(hj@+!<nbI^~@^Esl>O
zZ4-Q_k@*@LGXs-K#vOj%V?0+vav643Px3l<@OXYs0+X46CS0?%vH?cxq61mo-#5r7
zOa|N&tRB3`6b-m|DXy=YMHast<KbJwx(7n9-pxmwPSA1LQ)hv7&ldXg94b=`cOj9h
zxmN!6gA7itm#m0IHVsqJKFM3MJstmH6MtBOXe<T~JO4+x%RQV6avh;V+bOEz@&<^%
z*c8y&t`|QddFjfiT=Ap#8Pk%*rC7Ndz8CcSUB!<84DP5`s!|Q4Kf)`|sj(eAc-7i(
zM~U{WbR>^+BtVX^_Nvkm5Z)}`xmV_2$%8``qhQQOYX-q5%vQQIf6=86SWf%Jcd^?8
zJmERL$1em;;;K8j-U;P4l4itm!l)ouylkih(;Muqy}{`aT=C}>C41wSpv2_xb_kq=
zk>H2gZVaW1F~kId&2n^Q`<#lOalCSOe-%;jZJ~c-<jf)%zb)n2p|L~9vn*N!^$_E}
zqmmReO~-D}$BMhYclr%3x0rhyrWrMdN$|G4-RT%tj(10x63J1@#W#8oTu`T-N>NM}
z0~8LC+Xp%+3r{Es@=e>GI~{@flX2Eu=$~qp3>F1{yttnQKPXtB;*rOyuJmc`1SW&3
z3GBDa%ATZoh}9QtOCXLH(r6oyTZKd{OY`_BV0S|nW=q_J<Y99=xEwau8trP)7UPgu
zGD+$C9W969rbbL5kc%J^{lp$w-0bOq_KfTVy~5RkAGRePQ?NsO8HCCUeyZI%Wto6D
zl5y|F3|hbCqx+DkOS_lT{g0X$Uj=qezf+X@T8+QTJ?@2f0pTHibZOSJchtE)+=abY
zPg-nvmpzLyqKR|d<C{`HZgENx$v4XqdEr@Y#<UqzPOe*3TIu?wInF9;&>;g)en#%U
zAq}EhVL4W#j0*%;3)E9*s{j-qYp^#hLtN{2wObdo8~kQur+RxOHYL%j@Jd#K%P_cn
z@%AA(dUNYYLjrgsGlGipiq;6|iBm?rJB1xhb>U^(*Cl==3K7Y8p0Z!JaU&>vt#Jn&
z4nwtFJ+VccEJ{)B!i#cLw-G(*w%!6=%=A3viC>u6H|l32Zr&HS`bn--Sy8_$t+&{S
zLu((`BV;xX^dKK|*A%&1Zc8reHb&%{>mj+{RCSf8eSImGwJbX<*RM?V82j?NJzzV!
zqP_YaydZj=O%j@TlU>OEMp5+_s5($1b&Q!a?3gR%-&u8T`|ep66;4!hk@~u~{Ou~|
zw_aDqQS(TiVo?VfZ`mj*(K55Ufw@&7TV(kgwcYmeC#g1Xp1cf;-ooJP#%0o_3Xc_X
zKCr1<K7?_)tD7A+O+iN<0UbDf7Q$3aeRy@1cY+aFOZoM?;FrLDxE6CA)9lU106AFk
z3m~oT?&gPYH*vZq-n_a81G!Hw(1_KfYu6qxuFFvH$EIeWVZB!CyMRN$+sNqzZ=`DO
z6tu1&-aI1|9z4X$;tv;ShvL9d@^dMoRUW_kRYJ|)Z!oCGMu1e1DXsepZyMw4SfSDm
z396+AZw?{Rw&r*Fb7G+<c}_pI{Ds#1QFf=BaA`Xyg>yd|80DY(l7IRT!1aqf;5LOO
z4>Wkh+==2jA@{x(-e29$9LWmPqaqQxxMtGd_A@VP&Ma!Zu$7j2u=)@zAoph!wi17A
z76Cb|re_EMY0+VA#-?&o6R_+tc|8LPfpYg!e>jyF?_=xbrgm5rA|^}ylG+7@n&KO^
zr14neFy!4#6IHI*9AnL?6whgj4AKFHGuvK*<U!;DOj?KFO!`dwWnJ2tt2#i<484uz
z5*v5?B2YLQ{cXoFS(p3FJ$g=oM#xiS(oHW<d`ko$#_B6Wj7d7Oo0nlAdBRh_)fg@L
zY`bNwds(r6Q5l}M-q6M-#QX#$N4Yh4c$O}HZO<hm1s(${@YOJ~S;U`$jL(p)C@(dU
zK;-Us`^rtdDa?FT*=Ht><mMk>UKVrOKAD{WnYcfVa&b`i?)8U#>|!3cCgtZ#!t*v0
zxjE#??k*C*eBO-?om9})xWw*j%a1ufEwcGQy??Qs*<s-6gU_WsAkl6ecq#HDJ`qS?
z@oMsl<Gt04q>{qIl=hwEt|93{ueyTuBe{#fOLyT43Kr)6r>Ay;wk#{A>}C1tM{Qq2
z%|0rY1WYkVmzA!%6m_9ldP&(NUi><YF(vTa(6rI#OWJrS(?|M4VPmv5PjmHuQbT5R
z2Zna4-aUPEr}%j`hM7WPD^VMX&C22&0ej7xm$h<7$5)SgiC%<`aI^hV(UxJjtGz^V
z15hP6VEUF&b4>v@$?$7Ga)$k~u3Jxx9w~u)cWkBQ5t2YR5NqfH+8u7h5s2^`mYUbZ
z?~{iZu1Lz!Jr)tSHh^YR(}<|%%BChk_6E$Pmh%(r2U`ny10A^ScQ12zewgbgRXtkN
zyVKuBbn)uR2Yiv(b4{2143zhm1<9yUbP{JWo=~aVd71tmySSa-9ztMIFbK<?bZ;P)
zkdf=mqI%Q)>Yk1;!(MpveWiYjwq}mm%VVm|KkC2iMXv^}<{Kz2Ii$4&!SXla?+y<D
zb<{Vn{;k+`P!K#tTjwE*)(`ddhryX2G2yB=e2YXo-NZZjv1bz=><mG|<Tu9<aJ@)x
zE+Dc>>h?QL-VzyPHz9iyHPfg4zO#KRL|C9S4(BqmC5k$>tGnU=cu<z8S~_`$1|l4)
zMhp1c&FtQu{zRm`TFPzrgI}>J=Q2V8-HJFZ-^VFmUCMJ=sxD!mY?#0x{d#0AD0|#x
z8`)7v?}LMD9w<_kXJS^4L1sm2LydFSqV+k^$rGEq1SoU$CIP<jcp}14aUAzrlN2Tj
z>(F=oUfYe*RO$Je4sl}YHpp6GFC#0ZGXXrs2fE)=6zj+3X37AU;E(0{&wIek-$27e
zFySS4yXWp1nuX{ZU4!iLGkJ8%NrBRpW6rYded0{<Y1-5&`vV?1lZY3|fA%qkQo&ta
z_w03<)gp`&mah#DE0=04Oc{vXnR-+9`Iv9o|Ef|+_&onm*AL&_9`0Af^Rj$s(bdaa
zWVr-s>Mz$L)mcrRaY+5RWODhmoPXsSlFC;Pju{#v-)BGb{Jg2Uk#d)*5t<lCa=EP=
z<rj{^QSDzOXn&@&e*~LO`1a1DX{_53Ci1qk@ImNN<;i)2;*LVFvNL(PCcyXT-N~rB
zn5&Kh5kSvzb2)370M)<K+)MPqfSfWkQ2GOJE>=MWYsELHUK5a=0s9B7Fw?Rqeux;t
z&5AId3Za{Ve@GR}KZ)zGi+1Io)dP$g_Xkdi0UdJ#E~ZOPTZh7>3<$d=JrI&)B6{k`
z&rHbts+Bjz<K^lEG{Pt__IySphLP3k;Bt0Vrq=0)Eh&v-dz78e89hB#g6gpjxfv#{
zZy4-MzNAZ+{i4jl05luvfMkf#pw_)A>`uS(jfe=8^eq>ZKYg=r;w`p&*%n_x!>M60
zacPK>)7j!@>(`qWap@|u);DtQL0mAruU^~B@l!UuI`#|y#?))oHLnG9C$#zMMliW1
zXT^9TJooXhdOb)BtW?mhlz@`w9LMj#hivJwWaxSAQixi)*|@{QLGkdK4>Ny0<1oBj
zj5(*@R5rC6#kSPsl0i*key_?xi-DX&Gs?paHo&&rnchdXCs$09XA#-I2x6W3tf`h+
z4a7$#cu$QgQp}pf$1{G8I;LsIrF~Mq$Fr2@N>489LFT}JEnYoP`lVE{cAK?=3#0i~
zN*Kq+dZ~DI**r`@^h=Z9TbYS>t9_l#YvRJBW?i3-2`|ImoLp|ph_Fl^lR3L(U4YMX
zb<Y8)tzu5hN}^C)tLo-fa`i3zUs?z$62-2LY!?r`&yHPk65h>OCoyvwNDZBaYa?kb
z7Q}F<#*Ds!yX>qoJ7@OP67BvBiKwm(l(C8us3ioZ%uFFSsDsrM<1Ihpm0mBu#RP`|
zf^S}(oc^oFBDu7-Ie*+~|D~PhCW}VHaO>KI&A8L?KYj#T3srSRX#>ASt7S}$5mS)A
zXu-JG2W%Jy-PzSYmLe4%&;Y&z*RMR8%@*&?j{LV2?l@HN&t5hWeIeB3En_~q4;CWl
z+AT<PYq$L}5Buq~3J3PRl%q+bq}dxofE0-4MF0vqB2RBVME{WQN7{+b=duti6hMK?
zo0hAKoV2lF29MnDQdqW$?yrzDZ2@xU15PT1Q2|51%L5uiwbFJQE755KjgNG@o-06G
zoyAK-7*&-JJ`qVZTLP42w-uMrTF&y0T8nW{mL-dam-y-0;5xl`?-{X0601Jw%U?=&
zI+7baNV>Mi^ufa1*B<olY(Lpp6R=!R+_%k9PoXeMdKfGtNyAP^-v|T;Y0R_Hv?9IH
z^b%E8Cn?z_<IgNO{Dx$2l6;C=%<`AMN!wKbSVVYL?n64tR`TlDFvHB}3DP)zPU^G7
z4}61R3iT=(6!_m~oecIpMVX7Nn7uIzoj>FR%+R5LYfjug*4rO8*N&4Mu%6sl!U_gU
zN;%_si?#%_$4xYIkR%y^;DC|Fy!6oTcz!Ln#TmQWNS0kIbB&a7uai`W>4P9(qmr~7
zf;O{7or7=aKo-r-rwBUVt%d;=J8+=|><edrIyv<kpF7<}<6B8&^iOwZL+Q)GA!x<P
zwKNynW{)2%PE5q%j;-|ML#=feA+8xzrHY4TX|8~V+}Pu(Q~|V7bp9l;$Pe&s*MLS;
zfT?#)g!f&ynbfQ6Wqj8LqNuMe9lZkmwKtxMOM*|o`3m$a6=qUDCDm&A@TOi00!b*U
z7XFg_Noi+?vpnV`gg01YlI0!69@Ab7{%!H_l_aimY}oriL{1IK=yCfcdft6V9Jq%}
zzHYIU9AY8sqV0Z<LglTcpR6;pj`8cZvLzcbr+rQ_dd&HbsKAg$S^?->NEuqjB6Eis
zyiMP<`v5#ZmD8U6XKZ!xfy_FipqQsQT{+Rl$N{=-<>s9I0MN7MLF3(~&d&GiErBJI
zA#?e<p{j0TTdbYr(3b>$6^|Y;__ZGBv34>XPk`fK6riPAy_B><doRPL%&>1xHJ;xJ
zasa|N1L4SotXl^eK6K@)A?1?Cm<rI0ncD$z&e6BO+lD{wmL+ObH}*PKqIU$$#1NeA
z)AhYaf5(si3A^ef<y_6yV|S8qcyt*i>Su(V3b<vZGn6wgGd829G*SiF9&pQ<s3;@<
zBr*OCfChXo%V*24UzRCoDZW+s7}_3ro^VV?ebaZ4v~7DdTC{g(b(q6k@yGVlZUnq^
z?c6)8$7cn?q69s|?_3bkUxu)wHhUqO*C)~$qQoN<7YJhCXXPRQ-*DjjbS&U{5zn&v
zO4^d!mtFlzYRj^x2m>}3wDJ9<^W2p~x<(#HW~5P#GK=4heH^NRG4*lXyPtKHse?h;
zv9cr-`q%-K%)kaep$^PK5h}fxt=T|2by5a@0>|FQx@mFbV)TUpDQU(qiH8Et5;CHg
z^`>BfZngz?KtIi$0zSfSmq#nLjoa19+mb&<oIXJIaH%kxfyVy!?qcN=;sd`+bX_d$
zOBRnr_N~3i)0a9pvbvjB@kT_EeDBmy7)O|m?jU8kZzFt(=OZAHimb9&%8Z>_oB*kQ
z^^Gl(`HQbF5kuBpKT51$V8vtzJgwhj>VJ|VTzJUFi46N2$(;`6Ou7G!Nj@thFe3uB
z!U~Ffpf%3GgGUi%)PreM-Ljmx@0O2e^H}LBK2$qr;7u8h&_ja-k;a=0{phnuAq5iI
z9G1riDd?nHIX28s0OBk06h(#kWQgpjiyQOOs^_HM_uN3~uLZf>%r$dn&!2$<1FIAf
zmAwgyO&2^-g_E6^$h9T6B!28lH*NXdJi@XMfc|ojL}oip)py&ZRo5E=X%j#FX|Uj%
zoL!<$l&GZ!-n~s$kFlooPCH9#T=;HD24vm|I8n5}fcT{wsmjk~T9FXXd=fw|_oJ}<
z0NYnb2jajR*xKLN?$f>2q;Su+Ld2V--s~BU8{D1>bX@OWhJn*B;u}#H)kl^xE-aO9
z&WEgIk5Z+rt_?GiED_!h$z_&cAG5NaI0j_=6h&8yxDk;LC)#_>s1yJvH7f*H0@4j?
zlpQIEH3{sx=y;p}dN#RB9eh33F{(kqI&5P|o{5r|oz$fzN;nyVV2boD%ZFwvs!sBS
zQYmt$qgGjVcm3~P1C7(sJt>sq3=JGX_s+{PmX}Jdh1(i}CO0JuTJ>L?!qc_%WGlB_
zq!)H^-XM-MXBHQcqD&L)OY2af^L^MK*n$}i@saE9d<|)UPI2-EBDciLg&yuc<TVrP
zf<CCI-@EY#A^l(6cdXFvGB*;cvnUE*i-X>lWIq)jd)?PNH;-E!eQmfY79G?B5jYeu
z$e(BCb<FRJZ@|SN=Ru-0f`!iX#UOL*iO%V*t+WOlHgS`93tC6JBDEhGb8RBK-GVmf
z#ER?-?<ERz&7~z`&~>C7K!Ya$yUo&8;Nt$&a3X#mWHTiLcv$d|=Q9E*WCO5@dS9SD
zm53Sdmjk84_)HeOFPZvW{w{&)`<^}60|$RA>35xl-O^_jM;J5Y9fD%_f|Gcq6&{%G
ze*<OO*{dQ#WXTY8744bE-kDo`q&J8AjYN*|JV#{08LLi1DP%755J*Kt)ByL@697g+
z!PZB;jx{r(M$$7fts3g}@IIf;Med2-3)pNphp-SKF1?#J%Lv5J1o|Zsz_p+?H0pO+
zFXka3jmbX~7BWE0w;@l8>HF@e_b7_d=!Hk>RQe!r%COhpl`}>>zh-jllxEcw3W3H3
zl2FI9zljxpFk1EQd!V!|iH(j4T05a=q+>$e_GD)>>~-@Muy^P-V#na>7>WAGsi!ny
zQOC2rtpdkYN+lC}qpWR!?Yp9Z<ry3r*Gxt?IYvU1(cxMNP?P}6=vFmLX>otIJ$Z|J
zA<h&bpa#{~m->MFJtSs!=KO35o!lVQsp;i#s0TVfI1P7f-B!>p-OesMB7s5rxQUvd
zWSSzdVX9G;h7FepnWe<fN<Cua{eb1{Q_yhozh^A&{=mLeQEgPL+KcbqnC@ZwTY%=x
zvJiUMTIIeg<G~_%^F%ceaZir18wGSzS&C5&%wvKKx3ZqCyjfzO7__g-vkep`=rZ<)
zm(*4_bF<r$41z1__Qxn&E+L%D<~rw%Zl?=!>2ftGAwVvGmx&?8{EFx``0Uu(806FG
z9qdc}QC@OkJcZw)ES&JvN)aiz)i6D{T7t}kJv9~_z|lE|*TleIQIj=E%M4q^!`JR!
zL3v@2t{%3;T*F{)n+2eN&8q4X!W1U_(i^;)Dc8;86)8HP$%M0`U2NmM*dI&VBNd^s
z8ICq5!@sdn|H*8^O&(ota-5i^eF9cG%ly&~!cTv;Cx@?`a{A>*;-j_{1K2jhQfs2y
zSrP7aKl;>uW9ny<qpMC`(52hm!XSWcH<r!9wWivCR3}3)L%o+hrSO~?`ls5E*=a=~
z`=T^2kMa0!2rw(Mn;Dx1Tox!buiFX9mCiN2%i~CgEtjRp7epzop#Ey@0jR5X0)rHf
zXGIVZ(1#^oi+kw6?2v|2KuFB2xSy-GM_VYX+js!7k!avPxUuldd5Wrvm>00bN(yw5
zG1B<<G%hXxuO#tEb0ucqM7+0+w}(jNrhrSp+SYL+`j%D=dLTA{7J7J#`6$)!HhWCD
zYt|eKSB6viAsplV`(|8P8P+MoFzY7Yh7rcV*0NB}xNI4!b%A5^quj#N8rT4RuHfoT
zH$A6OO&L_!^DGr_1k(gM^O+{nw;_+`Ck+Wl<sx0Kx*^xtb?HZVLi~a8Zp;zg!W#`T
zoxL?W1DEinIOS)!(Y+SRJ(r(NcZUo_<QUy`^oI<T`5EW6!mr5>kl0015q|*2Uj_m<
z;v*Wj-UbUrE9NqF(wD!!%AXzrIX+Tae96g=0MxLy9)Wp27!V>`2W|M(>Q1Rf?mp`-
zz?CzRS!D=yvo}8|&pPAQ8#!5Zu43m(sgyYXG;+wdlkxfn%aYG-j|hFdp1G3EV<0??
ze*f4=X~{b{jNc*@>msKhD^S{6Ek}@nCgZlavm)acz4*j-j;I|LQ0>%tBC~upYPna8
zYj7DR8mmOrn`&c6e0F|z%)I5EO~=pirh?Dk2Gp10jmQXN!02Bfy?fNE@L;6$Z$;)A
zkfHVjNBVgm#Z1qEL?ycjA=-j%|H2KI{P=9%TT82LFgTvkM4y5MI|ovVcz8FhL6ZFr
zJiOh=X@4wsH5T{6P>eLbz<N9<%KFVxODP@<?%OAQVn1H7-SOoF#Mw`#XcU$*?^pq^
z21mlE;CG-J4J>=)IeA+>8Na?0de2(ml8>rs>VPyq)%Y7Qh{`9=zzE#FmLSKCJ*c?Z
zr$I}38epL|5{7P~O#mB%Dd8}#%doX5i=IeR+spD35rzXI<Wt985q=kd=ykujmQdh6
z(rIPr4G%dpUw;^jb|w=m;fbPT*f%@V8zrarIsW48;!F8OJb#uLW%r$5p0O=x@cP_=
zbv@F&o4n2Pl%{dJMZ3Sv=kX8vc!s6{m!!M*O4kI!rX9bF2z;gdJ0`Of2ym2cRb*uG
zE8+PJ3F%rmJs(Q<Xs@4RXX;S`F@)If)h{VynoXwnTA{RJHU+k*0c0-;vTM{ZN?`&X
z)@w>@sFCaaMj%4FILu#{D&z`M0V-_0&b$XGLME;zXDbGP?Ynz2b>TNBb}MRjSY@b%
z@`)hsBY=&nb8|V)Qd4fH9zP+=iB*Al2O>m%cr~qqdm^>V1_pM3ta??qK$m5_gwggK
z<Hg8KTlkD#=u)gH&(VxJD4=)w%VCZHn2(kvI|ICpZa?71WNno<fn7O0F`w=S><1_J
zhfT673DB5GJkI+|<UtP-f5~LyLoF13#@IT6I=ff6sgQ6cdlcLb=@%(22-R7GW$2<x
zwYRLtuO0{6be9kZuqJ$J<7Qi8aVXqh*u<4F5`8s;^$B!p#{0TArznp~XMy3?<5bo!
zl&db8{M#}EsIBo_7}VJtm<NjN@Q}rq#T%Nu!b^0(!j<oDgAgn0WZ^%FfZE%N6FfW3
z%P1-V^oNi6%~t?L$&ISy(+ZboO<^ucDN78qkc%zno<{LZ3CkTN9gda6K@ZMMwQzb;
z>etxXeyQl@iE>49e@c7+?FXkDRcQ`YMikMVTC^v<m!x9q@jjRf<AJ4kd0Hv8%<nt6
zAF?d5NYQhNA1OvvtiG={R{IJnS7G)t76^la!qpHgZ-|93wzwL<0^LuE4sb+z0%rd0
z9?RL6Br`a~r|aBWdo@5~UEO`eBxfR#`|Z7!;$}p+ySb0mJwhM6-C>r;yNXjlh*8B-
zW@szP-O8L7x=d=|@_wx8)q5KmD6|}xIk|A`Ni8AqA6@|4;Pk8?nrEOEw8k*Io|Dml
zEqp_8#^5oGH`&v*PvcSDo}ky4-g5oAlV~U3JUe!ZouxJyC+}NM%#R*H4$y-uHp``R
zNbn*AnI6l0K!R3U?YI+i(qr<7``J1q<b`cwPRkg)e(BjB(YT_nn&O=*7{E?*n_D(!
z@@+`N8ZteS$2gA1E@{+Bny;R)Ek%a6rWgi!d@SOR>s}i~@Q=icD9|NvjYLva&0QvJ
z`~+C+7vPb>Y3rw#XfhsqK}|m8a$#zYD~RGkL{srI8>G-j=7)K~-n>rkEc4#!;CO(&
zseY;9x^X6d6UAbQQ+gL-E*7cz+iCcZOiF0>m0!BGNt+DN*BERfky&$=RUZY(k||<u
z=WHP4duIzG$c|tidy{>Xerm<Tzu!JP(JY5=KnMSPidDMy=;$i{4?z~U>B;x%*efc}
zU8%Dwm=g%Y<`+`~@Eg2u6^#L7AO*Zo6X8C8Lv`5$U}9C;!FYm>u-tADM^pNxwT>*N
z2N!4&*MB;kG%$XCJaq-ETBH6jB-_A3_f{eAGvnEDYjKPX+9Q9^sjMNHR(m`!Z=ZE8
z_FQWcFj^?~tE={WKU9r9HmdERe8U)s05MT*^j^~eJkoFdYPU{i5FvfZqiDYeC1;w`
z9#R}sC@Gm7t5V(bY!c+SBnua#`H162JbZ?#knafxX*a+??>kjFNpFknhIkx}pc*6_
zt5~jSkY(NWW!#>zw>c_w0+8JL3p-dKFaU8j1(@chGx`3vyZy?K@}V$yid~(JM8lk-
zY?4Y*!6~L^5#ezoVbmf-xrxD{?bq2*v+SA*uWaycn)jYf?70M1rSW;YvD`8}LEr8I
zpx?r&GvN9~Mqgb88%F5{j5|Rg-&TBuKOlx)Pi8d}mV9k*KAdlj>L@=iNY7h6F`g_r
zANv_x5YLrPP&|)szi*k3K7)PrW5}v=9LJ{&r?abj0|Ax=Q7I}(FYi$rd*y(-vgLI4
zHf%8<JdBMQ)*8hrG;>|&&ef~rX}SosqbA#vm0`-nV|>zE`Hk;nz%7uN<2>>8*wot=
z^R)}mGkOXSw`g5s?QR`-&p%$oYrC~O*O2=tn5(%U7@s6MMtC@cngkbGOc?D7UnSaB
z$z)<+LmAjkgf;g6JS%2lzS`RBuo8w_;_ShMJe`QNV<crQ0Cyj+V1(a9SWO`?3F(Z%
z-S5{mca{MTaU;?|84>gqXuLoq+oovlT1cx@<gQ5HQj=hiMwFa-zq0Yegf~@(fSeNB
zo802j{SkU0QN{b+0)RPI_T+V;w!<n}B1N4|Qog4F1Ut>0(2+B*$b}2Sv61IyyRW{R
z;B`>#ZAGW1+HXbqa68=TwzW~<H;(6Y_A{P9+FNrxh%ubXQu&g++d=47afSf;j@*YQ
z%SXnQ*(Aa@%nH}je0+|&vF6b5r|Oa#Uu3e=9*gdw@>m#vG<sYzPFnZPF<sOB8XH8|
zYpA83RUlQnJjX(|XmWY*Pww#0J`VV6q*d-0@35|M<^{iel&5xgws9+Htd(}R1vyX^
zR@w4KwcY<I(|Xh5oXEb($I@pp+%-G8sf^fSEp|UDc6Tz$MTUJhw>wI8ZlsIK{D|Rs
z>H?4d50jl+U#cC4PrR4^<Sji1)}O%k^|S9qyM+CZ*9F0_y%~6}Lh$v~KKG~GV@lYe
zHv>Kqpo%DiO(c<KN0%8u;;B)Eyx&}PY+M`rf%5fhyQA!+A)ArWD*dMlTgMH&6NlqX
zT5vf0?yJhwAnv8z5NE{4Vmyw(Ye#+qM@|ARqVbSYLw#t<BLGfn_=zL3W7kRR$*hPj
zh763oLllYTe5pkoA0BjTRTv0fh~nLfdz+wktG1a*y{Uh7YmI^Cq;QSOf&i%SeOU1Q
zbkHtvED_o7uhTG|>tlPHzHpM+f-F`mPB)q0A$8aZ1*S@=JX`rlYasf}LZ`8K+*fb<
zqzG?{SZxI{h3u0$rg4A+mjfKQK^B#+Ve`&cU?*Gj$ip8#<vnn|lq)qD4^GQCT{*8?
zS+u70`{?uR^jr~+ItS1zF6dxL5p#f>CGaa#<Xm}Iml7*b*nCCwHEKp#HFDcIxSY=)
z4_aJ$;^lynk^hvU-BhKbIAS3svXMx7xy`Iujla;r(H?DOw9b68R+t^Dblzoh{xq+A
zL_aRpFF%)XKrTN4usIDv(}d1GBxG*_18;ZH;jmEBu=Z$4(ZXUA_rCM4E2xoB%|YNf
zb^NM=-g9Dr{%JeYz&<$R>-(8xtck;^XYX2@ZAqjgZ=fxPz_|s~!-E|30@zMDdQ!ZB
z=H)otKwA_|&HqXu;#9?ji2lcDBBfqMG=E+cIzLLkSxp(g6uOk+czm6GTjI2;ei)qf
zm`qWg-K1K~x$3|qavdlhiT8m{ftKgWgx<qET2_k2Sv3=sZiQ3<PxhcBFt>wNqrciL
zV18vSc_RmuXvO63rVUg*?QPB7+ty7|`FNmhF}~GN*@DHO(`eor^r+^M%qlz}Z0J~?
zeqf;udTSz<+R|1;4bNR2z!8ocwQrt?xNq*o-u$-F%Dh+lNM=L9`^5BsX>1~e-?`x|
zX_Z6hVr;!_|K(e9sJ3KmGxk7Ys%skJ6;WnLTZ2@vI027Vw&s0?3hIgH^u(liQ#GCf
zPc$XKjS{#Tv`2FG{-hOG|43X2L0zg~%F$82(cqs_6)`1wGb9&^+9kxEr(VzN37jv?
zd!JzuBX+7-I|X#dEy*@m*52k{xP(x3dk>EG-gip_xLSvpIimo3X~LhXe&*ROPgHW-
zakux(gOwWFI<&x6#;(d=W|k|8%8MWxqgtOaEr7QZvNNA44pjP~M$6TIxj*PDq4x(X
zi4QJHQG@(uH~pUqmAELrvHep!^%~C*S@JSa`X=e7?e+u33E8QL!;$*fSPg{J_>vy5
z+>}Cl^fKKOpHI|#MI6fMqyCl*)qW%|<&PCZJ_S9M?%pC#2aVpcjv|P0>P%R$HYe{=
z=xd%`I|u7KlJ|s4-tOG^gp|7zhIb0+M?JxTN99ZhA3iZi%rqQ_A%e@MeJD^_IvXGd
zLX7=tVRr^iLfl-PqBCBMKOi4Hfz-Oc^5m_4r}lZ~iJ}aa|3k!_lk%ga?Tl5nLb<*=
z`|;b(fzn)@>?=)1Bs%STmhO~$4P#K4C>5hyzg%To7THaGqb+nQ9NM)r{7Rg?O!xJ$
zBOWVJOcqO7Q3Z}GOt|sTQzJ;D10yH}l2tS&;RNkSbs!co0al-?+jy(XhPh+xr`#Ee
z*pyJNQr8Tj8}Aabd3oQ`>6DWmZ;?jZ;ud=PA4K?llcLjW6im<8D;G8f#zQ<hb&cY3
ztXIj$;%>2Vt12#8q)F;vteQpSiJasov~d$jrvJo$Mv5_}PWsdAyAwdId&sLkGRW}{
zx3tHzlp7^`aa!O+r#QddbCU*Z@t{c0=gxE8yTQ^2Xg3Ou5~G-qR1;1Kk^o7eSK!o@
z11ye*3G5n=gA1$Y_GH3qT3lGROqB@7#K)v^OAA&+C>pp+dd&PMJV`0HFLh@#l;?+*
zS02qXcS6?PZNY(iFu5@{ZLH$cS-S_KDJ=z?7aq}`Z>#iE#`fdpfg?A?0k_nza|sL5
z5Q*!GExyMANpSrzPFO!vg*SGZYW)lbWX>a?k0ne#;u5RejcovRoS8QAqB~XMZj%~A
zYOx5+-H!=xvhYqc0`22csX(2vh!mHSnd68mtI}BJNq&OqxmcOZa2`E##T83HTZ^(9
zPO!?jf}*0Dc}PGM?;`4>lhS$-ed#v4eSS}18qtgRn5vsm>?({hx5LAbj3++`fKSXI
zt`U$gI*5#N1U&)uolT@HG5?LlH}tgOkmS%|o*>EY7xM&LH6q*fWHb&^hpJPKpOy;>
z0<6CCi+*mUc>={nrMQJ)uupNaxO0UuWkvj@Qc+(6y{x{S82J6UkidsCCO`r5!$YmX
z;fUamv4?NWf-TkZw1Cm*EH=788{liW0IjE&KBh<Jo5+iEjdR*_oTJ_u-l1MR$>*eR
zAD5%ON#AyG$)LSSucN(5Rpc12%|xP{L!s&v;>^VIA@fNCP)%T3;ALFKWTMP^L%xzY
z5TKNIljs*W(DvE2=F;QCsvH?Vd&O6|Pe#9SK<1pCiQZ2IO4E(uSJFhXUQ2$zwh0=t
zJGuTLZHn}B{c8_oRx;0y<uUv(yn#JM58AX@#(U7S***BDrkJuPpg7~gO@Ye0u>j_d
z3R<#7Gczd~IJ+%4v0>e|flIYqbEiyqG`fNHC(F)N`IjE8$8E=^$&dZf^|>fUsqCF}
zn>OP;*UE4Ax`Kytvaxge!0mQisgJvi@q<~jF9C18v)T^O%}Nvwegak+6^VHfmx1-g
zW!TpGJ`8W*9QdT`c8IBYLtZik&!64~RMRj~xQgZHijPh~DNevJoBlAb<^rN4UIAQ8
zj9!tN8<;`&=NN>{o8eq#Hxrc$1Eo=K@Ya0uCN&g^_6I|zIyk;>Dju~KD|X|w)IudT
zXwNvY6Vxy13_W98vbd-}ok8V3q$9~~OL<>28xgbHEwT}SmiHrzwli#?z4)1wO`LwL
z%wM?=5&fibWOicKP^2PX`&vR`!b?N`Jee*_e1X4%8vkVVw8VyX*#68NxkK>>iVyzA
z5<Sq@@9P1US*iM3Pu)ObiScHPp;+XXJ!MnSw?29`Jr_x0-@Qdo4{DwVj2U4J&o~F~
zPnSc+$ke!J(9Fsr60?-5gekPg_AtJb0}CcI-_kFGQy}urCEcI-C?yqfEQ*kW?CE@$
zD+O1N@twJ-IzXeXMQ6e|Taqx;q)cA2*>GF~hGi;BC#%&!Wcmf)3)pS0jD$0`8A}|(
zZsiP0tbPDA5?jl5bV>mXm<;ZUX!-H7JTFss?krG9r)28=+6oc@ihK^HfF^34T{AsJ
z(Q3xW7W#t;Q>zoO%^HH+Ez|oZsTyn1&I~*szhDm7hsyUW%K={Ii%N(YcA6~{If0!G
zyy7sa#Fs6;(aQYFSW^GnsD9;q?GL3M!gSa$KH0b$@cS#920Ww|H*OTBzeopOK3TxW
zz?q5P>ah(x7v7rm>JpQC>{`gu*G`om6*u0%O_FczhhJi2F96V-N&j%{38{O8EZ04m
zH>fC{LX+py672Hp`}>EW04XbkEY>Ul>}Rf^Ny+k8bHb<9?*706?!W<P$S>TgnGQ@&
z1SU`T%U$ST<pG3&no{`Xn<e#S=wL{J_`A%iS`?mKOKIZnZ$UBM_Np^jg`4-8ho1Q+
zF0U54U7KwC)c8em2{dNhG>GVq7W}K4oQWC}3&8?F5xMV_FlO|<7(g=VnozTRva**a
z>k5K*LpRo%Yx*GG+U$RU!ClAv%#Ic)4A1MF|Gq7K{<SS#K|OP2?YxT}${PUYfDzFQ
zWz)}T^=C1CJIs;XFOIPbeJ|`KE_MOv=#F-dNXA7xB;cutLtg|ZuG}ov9vPnOD_QbH
z2R+=pqrME<%LN=D)Zj}1C|+ECx!nSIQAX;U*_@&Jkp3YaEE<#K%S$lN26I_HGJWa?
zw~~Qv?<0V`NxSCXx@j%j><sLw7Q`7zLb6GcyGdPu*J@qltxrW)$3Qv39!XpcSlm_i
z1{`Rw-5+S(Z}QVn0w85q8|?aI!w8NBI6RP3&>Cvm@gxDvt5F20jq{KDCpKM?zRW)r
zY^hy^s@DO1sO>!5FvM#PJY4nc)pOQoJ+Xd!d20b&PI&^DcRvjNy30z8-EP*tF*g%8
zYD9W4Kk5wGYnTRbmK)mhpe<KugJp;plCeX0fVUGPIt#D29B?-|ou!-vEXGMPqYka~
zPs#_tkm($9lR2VLgWCdn0FA;<=x1H(qTHh{F`fzoaRAH3jtbj}>&Qut_%F7Cddv=0
zoBR81uylB+RkSqbKnFT1>t_MK9Ah}-;Zv<TU^M$;eC$f_r^Hy>zzP;nr~gz)6ht|c
zKIu3}K;@swtG9p7oBIAHo0sBltZVd=YX;S!XD+26-dHa%&H$SgF@~8DXlsCP--)(+
zr{KcD`ZU3^gEFjW(*h=;Ij1!R!@y)_;<O!4hY_m_k0WvnIO5KO7;oN+o;q8<JAZ}r
zUdQxCL%l!eU%UT4$@ba)mJ;zmnJX2qf`!jE5aH3Lx44hqj|>*?>yCm6^Gp%#alaLa
zFb*hIK(O0(koOJcqzCDT0$3Eul0(`C&l4ChbC$IV4Fa8Sk;qe7m0ic^xK>eJ6Lj`e
zxqQ3wPl{Z(N^<{cQ+n54YZ(xT;5!ZjR@>eiAk^fEY8c!cAc>Af47aYE;}0~BDYDJU
zp;H_Bn!|wslB>|CA840lGCP)}127K}<KVY6%|iXrJpk$xXDjYpqZAy-mEI%oeF}1B
z)bZZijZx)0{fF;N&bVln+dD%n#AmbmMQwIslJ?;B<^$u>APAyv^+VmQZUGc+m{%|D
z!9JKa(7_{u@FLeQ7`Uu?F~@c$AQ`MX{)b7cXKo(aBX4GQX_LwT`x;1gF%g2*kYe-z
zZKgJ7Bg^m!2y7|Cb4KSXG!@_X1C(#@@B53duvK_~_B{(XAjOVP01iTWJ)%iGwi3Cv
z3fx<q#*<66dT;iY^mF2tw;r{23%yYZg$vtCycNb4t?-8Xk5ziFrC9()Uj6vfE*|>X
zAb|s5Iv_SSgJvnLTSI&0-C{`dGWN@38q1ue8;Q5?)vfw<r|`#kWa6F=!D5Ou(N6O<
zX_@DmNw3?b4uMywJ6b)gcHCb62)kXsE>bk}0Q~%pwB^ZyI7c9@{9ESMeKnaNya$S7
z2xAj}yd?PbuCwwCez67+(34eJlHOIfR;E_=8F=j3Ggh!c@qeE0fR??`7)36{S4VqL
z_#5SmtVS|Noq@0Km?k1!Tj*FZpcFmzhL?_U;MCS)Y;>g5qI~xWXO<q^8Xngxl2TE3
zAlB>5Of=T%Xk71Xathqy*BJbcX7Z1s;BI>*wignmXg5pchpX1zW1)D7iyGMd@KX!n
zCPKj)bbJL5c%ss8JYA$=y1k&j`D{f`pcF+FKVJ!=8m>}Iz(5X3YEj9gclc7=GfT$*
zaJo~XPxmvfmj~bg{<FWY%>zz%_g&-KF|r#XptdUm5ivK_ig)*Zoz`Ls4uBHCSMxuy
zIW-^HmfpF{?-kiH`fkZB=y!!g<?;7^6t$bHiH1A>Y>y|#9ig^KFI8q+(x?)hp5>03
zyRCEEg+}R8h#|fW3|lzJUX%vh)w_%=lMko-Wn5WXjnXVwQL2zM<5fg^Y2nxFTTEPC
z=yE$l!U=I~-Acar6d1(Mo~DA?sb@6wOl-Ijq)Nmow#|&mRs?c}6p2AIAA!ggoY}Ey
zxt9l5IQ+1QhH!P>pbsl~`Skk+y0{@M5u~{>0A#sw!h(X#n$+|!j53<7Y6K59IF@PD
zu#?`A0x+%-{gw~tcy@14!f&tr7_d86D5W2kGMCJibcuF$aPIM65X|ZbZpS0v&8;L-
zOf7r~JD8IWE_$__v>NmzNb8-+O3EtM4C%Tu$M@CMSa=YW5Ukuq0j8@~%Kfcpa6g*B
z+>f!XVj9SeoG^x`dWD)j*xl-+0V<hr=erTH0Q|j485!yYdW=deWd&4v$`e_nkgo`q
zip5WPk7i;iuYx|J$>UwK&*>-E51%%>;)&ZWE|c$2X41N2wpbD;rg}X0<2f4!$r3zu
zS&NGEYV=s%&OhQfhA$n;p0}4LoZ8ELe2tWw3R=zZVK{EAP4ngPheiAgJMw?J*Z-@u
zoosB_?GH{MQ0So;*Q<ZH{p-tK8rTF%%^xqw8(VvX$BV7J?d7;W-LflGPu1lld(3UZ
zL;hRg`A^j;_)isbz!Xd9Rekb1gKS0pSLV~V+GKrUNIS=MITPtDuHpM>H{?u`^pjrV
z!GW%ka~uck^>eS1vP)W%@^b&dtx@(eHgNi*t_GO^%C4W#-p#gY14%y`!PfFC$>qn3
zSR@*q-vGByd!rAR|GswqTtvTp0Qpy|*@I^cNkPAT^)K3>=a7XBQ;;(`xf<{o4@7AY
zwK3NkqMs3R{|>q!{tWq_XA}e}Ml=R75r39a+CG?Iy0`;5)4H7)zydHth9A1_yD@h9
zqS&`e{Li|005x$SXu>i<V*)Lur#0IIX)i870u||qT+H<O1iC$n!*+j7iABG3S<hch
z(rOV0p5HbHKK;MN4QYRSesB|Wk~}WE#3xkI^&|Q&nA@DFi48S<#tiO)AWW*Y5(Fwm
z-&9uo^9>eBVKjSy(m_wF3w%wjZQ`_U5}8wHPryP_j`akfpa-Zw!)O7fOv`T$O<n*#
zo37w07gz^2+(|{sM^`5|^s-Qib>O{Bf}y8-^edg`KaBn)1U>YVKPoE!Vdx=$8~VFg
zV<mWQT3~cHs403H#?*nPfJ{#uY<bJ!pbY{NO=yM@=&4J0P2-PdQT72gT9SnTYNo@A
zbR<Z1ngo{Z-v?owyUSUZX6r$CQptJ%I9Kb9UI8uldxtSSO|cd<;n`vE#$O}jzXsgq
z*ML`1I<Wmd;6wj2;KRWOyqFp20eAJVs;)o}c-hZHWAuQdFW<ig9He#p!+@jLOTZQU
zluP@d>{u2b2DNG{Js>SUTFfK-eZl<e=db8z$#?zwyS^|$97NrKIqUv`ElC4=2VbTC
zWnW=u<*%ZN?e08yYCg_Cjs@#q$3oHHp*!?j(&i5Uz3|p`ph(#OO8yN_8|EXhzZp9M
z2e+q!JwkfQ-nJ}%c^CclWz5mfb+zm6o8KRKMd@$r{k^q51B6o(ywKU=Pa1aUz@o_)
zLkHkj2dRK!juVW|6Qh4xOTQMX)b*|J51eWNW$qhVFt!0|WuE#laKN;!12UN-fOvW9
zqsjhqbNsbwx@k@WB-kAsg8^t;Ds-HNc%RCH-+yT6)pIdxwSI%u$GS7n>#EbBN9hLc
z4Ik26TB#>NVri1Pfn`W0*6xk%C_<LZc<*}~t<9fkpN-uEFPRh1?JL-^M6TYx>zqZl
zcMbdXKPIE_(UW<Brs*UA%_l)*1m)o{^8DLpuy96oVrW35MfFxAfsMa!7Fy<_fGN)U
zpN6@J7o*h?$Xql;4kujeE>jq0BQD?lhb)Cp%wSVBII1YcBNPh&eTBX=z%g(JyN=p0
zxz(x|A5t8|K3ORXHFISqw1{U_eh0%a6f4W&0FHwFcWp}HAmS>!|2HzlzwYe+`X@z1
z7<BUJ0Fz?Lh_41|k9Fk!J_&$nDTqUg(Wj1(ox(G8llFkO+k>Q}rN>jxh}GxH+x7FW
z=Ps)Y_9SXPG>tZ-e4ft=W<@1aza4s3IK2H>c^%}j2@H_^r(yXwPlSaLNww9X<@Yrj
zj4|k9=MuN*Qvdy_KxPyx7^o1wq$Iq(_)F#p^oFqk0t7tS29ArDo)HU?T^xqTU7l=0
z%OmgP?i%%)JFucTm)bU%lNIP(!kT}(l72g0{_XbHl=%f*^;exI8h~z5^ALFE+;2pf
z`QO9)^L+oZ2L<b7`HiqUcT>P6c?8Ie+l;|;8Uc99(gL8`-XVF^7r;38)95H#<cIUc
z=EWJ(9wT9bv;lF4e3i?Nix=!HT=@C}=u^}xT0i|IHtZVKU5|hJG=KSr+WZ)iG}<mz
zo#5Q~vzwo-guMW^E1D!FML-?pls5%-<u2gc_{8VG9$XUpNIeRAM(Y$HBWfY!+8Nm@
zz(qhHS3o9YsRF$V+T8sLYXy48K5eo8sa^j3&j0y?YzaQ;SwK5Bpf4G-n46RwzB&Dq
zUJjlPr3}XFy!3k}G@BL^`g4wie~M7ByI<p;LpA+IUjX!Hn)`R7a+Q7bKK8(Pl9g(s
zxX!H50sIrs6tW}Gr^$~Y4Ypb^T=+^>%)jmM|F?Jfm!tQ`4_00WbByOLlQAAXYW87;
z{%>RX=b-+-egLT_=o9evzboGUyW#zp!B0dl4OP=Kb#Q9Q*gY}%zkbjE;xF_+0NcTS
zvXeorTL17hX@@^;wf`8i|Ki{JmmjG_Z+`FJ=GF3%?cLt(2P@$}mZ<;L|E0175B{y@
z07P`T%>UEz`7aOh-+c+~7vNR01?&sf0?0G}J?tM!-hX*J|J}<4p`t4N!w(RT=E2MS
zbuRpOKmQ+p{ZIsW@XEI_3yt7+bSohLzE%9c&g#Ga3#M`4!42ld68_CK;6Fap|NZ;P
zs)ARUg#EqmCzy%}RR_OUb^ps(c^&gJ8<>Mm#~w*_*sw=4gRTGFRQS)oUi&6`4!#wC
zMF420f+S~T|No{p=oGPo2akTD_3z5C|K*opL%&KgLzx7y=C#Jgz&-DO`38Uc@x5d)
z2S3k`-`xO5ONbWjpK99w55Gh9J^K3hZpHRi6gI3!a!>8I4Sf2d+>_mTSY!U;M^9+C
zWjhr<Q>xVmea`qBsR>_Y<t@1cZd!6CF)0YVR=wJpCC_5T?P_fm5$D%FRavHaXw)aY
zxF#s+wCOyV)pxRC2a0XP!Pt|M{q&PMM|>4oOtQG=*sRwv{@aiLhqm{Qr~3c@ha1YM
z<RBw^WRn%gP7aby$rh5m_c&z4v5rJ2Gh0?h$SiwguMj8m9E5PJV_nbl{`@}IAD`Rx
zyKcWe*Z2AxPp|Vj&*$^8?ju1sIC&Omn%q_E6&Jjd2%iR^HFBp>|HS`lwf}qwK?4e{
zA#q}9(<|_VuL_lj|NQyiKQDd-KQf=GoH$v#ux7^B(tLmYk^eUjL6h_{q}iO<ulc`u
zjsH(;W?j5^1%8A19qhmTZvJyoK|Zix8#y-MZ@=XKT<G7g{r=@kB)Hr6g(Lr`KJqht
zuweKd%+deuKmP5m3cgA}iM{voIr;z8NB;T=EO<k8X!p*)U&7x$(tqx^8$?_f`wKnn
z|6?E7oR|wkmz0H8{jU%8e>}=<GOeM>3vX2Zi|gS(R{CEnw2^~T%_l#Y{Qvdc{d;*)
z=B%vpi|h&i{XO>YmHyWX;dipKob^L8?*8Yu^&h|YfBZ^p?%$U;yZn#?B+>uJfAjx(
zBVSj&FOT{}AY=AFbvq2b1ozj~*J`Z)Qy=;NhYK-SDWn-cvmI9#pB-av7*N5})9nBl
zW<Bkhu-p#l1U?36dy~D|mebM@G^iAfnogGQmYeRrpL{&3CZZ0wxvEb_ft2KPjSpx0
zz~6eJot@DH47!S+0uNNz<aLVIU(Zib=TB}*|GB?67<5+k9<+Xn@nZ|WsXo(*LIU9U
z*HRq86j+A?lemnLmQBy{;uk6A;hpG%`ffq1#U_8?DP3HU;m0#O37QOJ>u!!+nZVj)
z4RrQD;t{TPpoUdBH{FhUTAjCA9sfldV35bZiwqj~ORkqZ8FiOhmiVyWmS&9upx>vu
z>i3>>h@^W>+j5RAH<((dSdZ{qPCKlf%bjh<Rp*wjWE>&y`BL}pf`7QS_PHGHXt$M`
zhAMTS&k@j#0|>mx$BexlwYiyI3Bt0VgHx*7wDzRjuGHI*d#l(?AbqEFMzg66_objd
zV5b>xHSGs%7It6Eyx9`ky5pJZ|DOCe5Tj7vpxxW<cMfzPUpmqxHdgYDYYTK&GDuR9
zDA%sN5`P7?1bpj+J{MpIZ5Ms04=4J8tKsS_VwLliCu(CG{wSv`4nH)!%{1}${r<Zx
zDYz?+EFp9jn12+>^)xjt^E=y)72IBF?kdjIPqL-l3u)d5Dz>^WW_?ndud8t?S)O$K
zAjex@@ZdU5Z*NouX|5(42CdRAf=)@0I(!}sHXJK$y>U6u?EdPSSA$-PC4fVdPg~9p
z&ck`e!GzF+A7CUDmW7HkP9o$^KD7Em<Tb?=jH{>ZipQg<U7(%AG^s!kpx#Qvsf(=U
z&rOBHmsPu;!fC9)wqr=>?~>iGu$!_<W~f3jSQ0a*MQ78ud*cN>x;Pv@p@CawIlu%}
zl34$-ftoZgGHeyj?Ui|ZdVXCx0brs}et<*1xT;c84^+OM6qi)?er2sO@AU%j?kU~^
zrWmx6t4fW3P6Fc=!%~2!GwI+1*4?UmKu4*i9?V#$2dph$-O?+Z1Do_IRAz$K^8{?H
zl0aZb9w!kqx<O^+5KoX`Ns4Sk;+2QTGtW=ZiMxQx|MYmXqi<y8<giPw>{(nL|82B0
z@Lpx3kQ$QJk=3`J0j4FzwZOH`x5`oCyNGRH;fu@4da&Qn;N);3_uwm3g>~l|t<tbw
zJo(?(ZhB?b7_>bT+yfm$7^Qqk$MQ~OxDs@40z=q}vOS5+P|JG{k{rgmH1<+N+uV94
zZR_b@xd${(;7JVZVKK*R6TW2#B4Aq_i7i<lQ6o9_gIBFi4!5-V>=UhWTCLZ@dAHuy
z#7kH91A&Ke;N<XD{Me8b2oiw7uo1k}lzJBI2OivHLPKml|4VAr3U65*EAdhNc22+2
zA4%>eri7V>*^Q$yh6cdA{s_PyatsgBz5ql~tcT7!?a8jg#<fk(%^cYQV9LO7tR%E?
zZr3!KvWI7NM6Tn(2d?Xv<E-T>kQ-cU9nL|=Gsx4fdi#lu8K6?V?j=4wxjuETH&-EB
ze~@t1J!WBxn;}Z9V++r!`p5*V(^pgVkq3tj6qY`nmPAVT{%XfQd{#|xn;7S^Sq~iJ
zBI<V%c`(PI-Kx*Bi3rz!8RU6Mj`coXa9+sQu&amX8dU73OWS#*!uVaFal?+ux=*C!
z;=(vl_!N=;Na10{sD{+rhFJk3vJdvaw#{h<V9t;A05{2brrDHkDfCyVh6mN)ov0-|
zj;#rtwYY{++(Gj}(;BDTQBTULYZo2ru+t+@E~L%8Z=wj<;nok1?KmL8Y*VK5ZlW3R
z0^RT116+e<abSRM#t(+Oju8l=eQ=eC5ED+5hTVX@!oQJXfNqff&2Y-Iafp^Z_@dW|
zaKroBPI!HJ;O~x?wIg`ByxrIYtQk1upU*LRM3PYsKd;7na1aYDHk)Uvd@pvAQ9dh_
zqB__Xz#j?6Y*~Vrcs$@FI(?U!{O220fp_9`{E%NPqU;Y;#21fuYHFKGiw~1_gZ7He
zK1Ujm-JWQg$z=$?hQ4W$<ARKvbmYNAqsbt|Nif><v64w~e|<DAGE%u^!^ps^-2~5K
zo@r2>3HUD5m8jzHbaS^~KI?pHFt7%F&+QUyaBQ$=R%w>%4}39|f>Yroi^Y}c$$Bv_
zV6svIc0q>9rM)Gr)4{bg>sr8?tX6EqtF!3BmIHU!5(CHfgk;M#CixE79vi=SXT{Mg
zIc5J{axSEKdy_M1kkvUZ+w`RT3cM-9q<K<4sUqWm7ehCW(!I)kuoOs|a7YhAj3vh+
z_g1QPr@pUFq<~ox%JHxEm|Dp{TMYSKJH3~Vg+v=UtprQ0C!#`J`yrRtzjg}vmU=B9
zsv32>kap4XND{CO)qys#VLZvh7@Fupv*FIsZvv(TGk$kn@7KCZF|-9Y$6QSw#iJT-
zX<z$1Kmh!Vde%fTkJOAp<O$%M89o+!K%6R{KU0Ga-`iCrU|Dk)3)6>oiC2I)zA+9?
z^8~x?BN4SJMB$`*HPtowvCm+mRjFrn9Q%WMv*WqRB;Hl(G2(Iso_S1+>3v9;E({P)
zY&z_18eI`nnlkcf-azr9wsMsNW6!hF_wELrb^;fV8Ez!m&8yTZJ<AzH*50}{lHp#L
z+fH`w0}|G$pWfMcSFftyF!vZm6hos%uZrJeEo8+O9dGWaD`y=HDct;QHK_vf*yNgM
zY=saB`q1YKFh*pE9wvfV$^zTNTU#VsPLw~}{s6e4nIraR%6WUu+<Iz%15EBpY9glz
z23%yB6oa>|_kk|phm9d-P(HRu0BhsSkAO{GSxP@EMv3*z89ky&xT)n_*?(@)E`meH
zSPF&(V*4YjQ##-YI#PmP8fT7<L``-CL{z%1?kc@Uig6R;@`2s++6`wPh2q08yc(`_
zWbCS@Ebu`qwJa$C9HiJ|h$AO*QK?*L-RByvvnun)jTtT+A68+<`%5ELP;?LL^Uf|$
z3c;ZROKFk9JwLC|l|J845}eY}L85uDoJ58yToRKNDtX4A--sU9Gb)`15S_kRr@t&Y
z6v%AYG3!Y|9=Gp=^f!Q5=+skCXbF?+wYbO1jYorhULeULyTW&kO{D?A$7?HYYZ5lp
zT=Sg5+#$ud*=?}U=iP+<dO2<R`zN@2OEno*PtX@gSKCN%YZ(ei*>DP1ld<LlFwYqG
zM;*^SCSuiMZH}n{7&Sc*T^~j9L=Kfx5(ELy_9BGRR9`s%jJ7hvTh{j->wg$oUo191
zy4VFAxqn2x4=;2Ca8kSIA23-&u1m`DsX&LHv)^dyM*!yJHz^ZXphv!jal`bVN~VF|
zn;s{W{S&M|7Ch;CK{*jtT>zMlvJ%6xI82V?ktYRq{B3$AaIAip2GT;uVP{|!*-Qy3
z*tm2<e2vv|SyvUL^@HqaquQ`3Fd3b5OB%3sDdBuHgHHh{KZr(4L6Q0l!90Cnewtqo
z6ghWnZF>S?L42+cl|-pxxv24aML)+~n*<&(ZVdrS?fWnr7rxlSIN+tKJgshYQEu%U
z0CS74$f`W%1a+dSL4IJ3@;p6LIW-VrNKw=>utq%ak&DD7IpQ<bK}ly#l0NOAD$XQ-
z#u^ZLFS}AhUZ~Q+hQz;%N34SkLarP@woCN15A(l67h_5{++I09nuXoAs#k&gk2!wx
zZaJP!(u&(M<P5KVi`TKbW>#BW3M*67x`b^zY-Y({reow3=)UqUkpPbEpW8-yFJ;FW
zfT;fYRs!>LwuL>s#;7=FlofujlKit*uWc626z%xM^Ey&(3>Wg7TDIK~|9$8kEng;8
znB|-pq)D6mGhtVP+AP4;6X0xWW)~jumL}r5Vh&P%7tostu733xRiE@i(*0dA_w^US
z89GE3MqcheyKc92L7qed2IIZ2rA=*HxZh6gz3@jJ%R-N|V6V_v?K}PcdeGAh)E$Wt
zx1E(E6tlhNna6wTo{m`arem(UexeIE6Cey>KKy1rw(t_%HwE<|8QIC}<crMm0G8j#
zDY*<OvQI24hh~abm!_c{a=_~^Qeuxf*qW2>Rn``Wyh^DyH-(rWhb}ywK7^5OT@CMV
zd>AT_J6)~z9Z}(Yf*&L9epq^j12f6WTdQSdG3spfH*{<2AF6;n(w#T^_QFSe7Raq*
zARyf;y2)^n3OSG8W4j-!W6%_LmI2s~fnw^<y4ktkE8Q(T43^{22xNFOVf6@~L9j;6
ziSn(yIIe%l=`%Y5;sBigs}d+0)k?VF`3d7vsG|WrDo^d)tC58&QtHq7U=f~rpuOYE
zfk4+Aq%)UO){QHE%rQ~c_XYr_r=UF1oVk-^XjjBuXUTbh1Nk=#@tQ?8px-pDOKBLg
z+9Y1mnhpnNqIoZ4*bDeDwnQ5g6v`M>djbA=jbggJ>fwlzZ_97SB-L5Z_nUN|-2-&x
z%><aTQQ%=koH>M?O<-wyA&}DcEvI*P)Sv_2!GYSfTWy6cQ)-f5#iM%YuHM7xEd}3q
z)1u3V^OhSNq<7omZeholZ?(`iHA@&LJ3pelDzYk1wY=hXqcTilgL<$`w}r}_66H@2
zmOhd=xgymqp5bmwSd`E-JuMu^F_YtP&7zvKc@M9#Gs!xnxt!jHNoLLd{o>C{ad?q@
z%znmxK#F&6<~pf}>4I_CAeIwHo3kx#d(Efo>T<ix2nQY@A>-`rMvir@svxa0p`J=V
z9S$!|Zd}9?zK@CXi#1q0^eO#3VHcV7<*H|9&2z19Lq7WyY=iZC+BLeKN6KZVL^vPi
zqg17YmWX!y#GvRxUW()QXHDi>@@6G>+s<V2MrHoMbNPF<4W;smXEf87XMY_l&u&Z>
zimTOpYZ%AQ06=EqwhTPk!HPh3(-9!{9q9&zJFNq9RQ|5Nl*1wj!aa1@n$W5J8jPCR
zjRENo4JD7{r+WZl4HRk%w1rB{f2{%HwTTfUVgu^1>+LMG5kG986+#K8K-`kTyS9{~
zbRFEtHVt9}JJ4IK&syVanW7gi!lHlri?glWI*1JwTA)ptKMC*X)|fH?3O+EpXfYzW
zw70HN9g(qx_W1flQ5Mn5rO?c{E`2vf_yyX8wZfC!s<X1gGwmz*@uV1`&<-M7Xk-i6
zoZW&{K4Obfr!4;y;MQK3mSoTTI;sW`Jsv|ZE43A+SSwO~q3^g}Aj=8`P>L7aeykSh
z4ltg+VPmo|ZB!*b4=*Kh_GJor4bmDZ=lo@Sa$B<j?q?3g;Q3ju*qw!E%FpY51Iw_B
z@-$mRrw1sMS1$m6jJr*<FBsoNw}RdSry^tDJC{KiLv*;{5>RJ!-dKQ3f=Hs<G%ul4
z?4Fh{+hIc#a{g~E0P6-|y7xpqOsFmK#>!ZNgE2+CiRj#c0;vT1<|A!3h*lZ^P26HF
zbZ~~R8x?B?EKvB0@SC5%x*PDp5d|2ON9-K3XNQY3`^nnGzLRptQ-d{4m+|^hQ4YWb
z{Q_MZktu>8qFaD&>rSiT+>s3eYAMz9yRt98UI~^X{E^Mm8@!3=f`UCfBA!)8hm#j{
zoTVwI1L>!WqK6MNhB_xB_a0R+Y`8cjxHBTwZ0?e+cOaaWzO<6?E<LyH*~>Y6o8HG)
zrKU;fo0eyp$TiS6X(dDF$U0Cv<?<_BMhsl=`T}40Liv<8=$Tr2W?8Yzl{-yrZgePF
zJ|&Ip(t-FaM7C0(r;=3v?&4{rcA_8L8%Q|yL<I=VapO3*jS|Wg<z8~V1~s9@+}mdB
z#4nTL#~{sg<c$*CD>@N-!*ZaKNHw>2cv{f%wKn{qxw<<%;D`$EK-4oYgZ?G~ZP`U?
zYaUmaPuKb|;AB8nX~zF_WIJ7ICB<@q+hj7b1Rof`+uE;&S}8DR96$3lE4XtBOMmOV
zDAxO#Ccy)oIF?#4Y<-M6`=$2XRdHQv3w9G+18iaWD9KT7>rS@9bS0OM`n#4xCQm#N
zt}MyaZ?(}Lv&=Pfsgq{xgTqg2aR{NYo2c3J%gP{_EcObfYdd3UKSWm8GXT`$HhZ+Y
z_wcMh%H?FoX8FeTKTC8E7!3+jcNYvi$wzjYmKi#HIDKoH<~GbbRU6v?DcNcb-CjX;
zs!PD#$6sq4Wsycgd6sip4|IA)WjeiHvj4oBT^+do9Tc0Pv|IP!B-aD2o_p^f7~S>F
zh*9*F#hUw^ulh6V^?JtdzdU#)d+T(;Atul5yl^t*9!WLL)(&!8qkXrI^i-2IFxgS7
z`Ngc;k3b8P4E2N9%Jh#7FDPd~QnX!SlS4bYbE8`+wcWoLcGJ~icCf0?_r}gaxcn7(
z#g<Y!=3tBa%SNsMTX+YCZnt!qA##HTHS^PZlU*#F8-AuSTt@4?yQwu{BQ00f;Tl$g
z(1b)MPjbFF223M}ruur0lY%eG>xX$n&3d~E(ay!QiEpPbijSq_Wmk0mu7`@r{7L}9
zrUMLb(Q~Na{7)PJSbZ+d1*|b$SFdV2RV{~KCx()eEL5$&{F0Y}CYn0Ey(*yxeU*!F
z^aI3>aT>9!R;O!o(fp6}D;8;SE}`q<_p)W)F2wl=_;z-)hVC5C-waA&jf!V#t<xB;
zT=={wY3#zf+2s--a77bQ+lvT*YAiza3~4*AYO`1r+CUY9)^szY8s21GGk+>ZU;+~^
zrg-%F#kVWeY6IiLDzXD=aVh*64S41eF>m+ff-X>Aycth|@1~+0Bnn&4LB;O6u3vDA
z{)7y}B8<5c*6WJ}N$=>UvK($WO`88E$n}LZ5YUKBmsVoO)jGm7Hcg4;#EHPQ1AO<T
zJxj1{3PQMbgwAX!uqk@mqsrTs6kB-Nz(pYmVM}JLw0IkK`}+ez<T550F-=x$h3_XB
zEfhB1L?7#<sjy$BXId1=>*skDx1=qpQdZG<dmQA%8C6U^5;4D)fycHJ@eQu8U{-0u
zV$m_5EJhqmy}>+i;Ax>9W(j?)tWcMt_{lejwx8tNJ!I$DXNNp|HTS?5lG!f#&C}%}
zCpoz%Gl`+g>jIe=Jx-^-Y?@{{Xp{9~;FlMo(@{P_^u-h_9q#%C8H(11`^)Q<8L^F)
z{*;QZ5N=gfz8Yo%>#W!teR+*aax-5VXWPY2m#EWQQ;Bfhsm?CQ#)mVhuYh(+zq29h
zZTiKd=E*61xsg=mwYmPfY|>l&8V=8zV*gm&C$$-E(e@J^)wSq%_QKn}EJCKs5+3Ed
z<VlY@(rDu_dLD{l+wJRGd=p4Gth#u#?K6uL+%Ou5t2xJ0MC=z=6@ak2(mNak%km;_
zL$w8!Hu8mWo?*C)EP+Mx*_$gqm6<hHyoP%pN)cVHSCr2e7(LIE8}1=%SeDehjeCN)
zUiGzf)b}?UH+EE+HaoED;gINf(RcgIO5t|}DBo*f9=<`NcQ%|T%_q{IMmZO9Vnc;8
zxGiaZyjVOA)fYcdPr1MnUFH71F=peoh3wAyzSB(+(`=@fM0cXX+$471RJNdHwrnZ(
zgqm@n5i&vkYPovZZNk7>eK7kc%GUo8751$*ZsFjHY3P_LPLK#E*lt#xaI*d!wwrtR
z!E|6}iT03k%iW379mDMgBgWzx#+pq1Wb09Qjzhs(*%WL@IVk?jlrnOB`IU`S^qpg0
zucdn!6kYviRZpw$$Kj{`j|MoEjPlagCXm~+57NFLh~2hO>GYmm<6D);?D5Gij3ZX_
z$t>Kh-U=9%GMHM4Q>N?PJ2FzETHj&Z0?1CsOWwVGQ1Nq_=pA9#8z@@!zq=A>ZUhta
zU@RwgbzCA8C^QBU8QGzNNIe9^rMhFDh(*y0x}YBedwH2b!)bVx>;o}+;kODws80+L
z8PVrT6Tq%_w2*!A9BoW*>-wyBvebQ`*LRG6oqpRjkP}Be(Dc(xyp8+rH5`{}a|C_=
zRj;kUg9crywm*HXmxLD>cRhb}ez_)eMKx`5XLoV>H|Oc?yHT&*-a!i@=N~fV49^Qj
zWR+8{&37Ly-wpcG|0U+6&m)uwSNIfm;XCQ5dawF>ZBQ|{2mTach0-6Ywn?~vI6x1~
zI{hR3J?u{KVxUn3Ddj<+*K&t&piy1)9lq|B=un<F^jhfr)+id5h^O${vwePRJH=Sz
z0KL;`t?JQ4qt0$tMJYd2!z?J@9gN|lL$iI4?mWg<Cmkj#xe;6fN<|7kKJ{><R_9rU
zdhtqWR?Vs$c2xw%cz=#b3uER}qK92*Z%WqNx&KFHsa8j;(|1fct-Ij8-A6q!bpm*G
z4@~U41rS95iBk(}>n)3{e;xf+@0?UXm8>$}angz-%TCU|(HqNrGe;&m+l9cA=t>S`
zwTe=FIB+DxOJtgdmCkC>RL{^hukl?H)EUp7cRY5f>(`!xW~BG4z>F9Ts>*?<=9#2@
zZ{Ft5{9`xYbD%n~rWTmI+Q#hEk!|Bh0|?yXaPN0MFL4T#`3}aJ%{Nel8BCtSHM7+w
zV>bv4rMFB`znup5fIf|RYLx^j1|g2Q#=I7}@C4am^DSPTPYRu|55_}wyyC5wrr+d?
zsU0Vk%eUMmVY2N4W_Zeb4yT}MI$inG9|tkB&8fgV%<K#Q<qmYJY}b{O9KAK9e($`z
zO#1LtZ-=|Z;CBwg5*=C2pGu|~0Lw6PTngI1zJFdg6Y^s#4>o4+GkCtRbopd4-NB;z
z7`s=sd`7p$Imq9fSzT_x&UgjxY01_}2!wZX6wnnq%^IdVH~K@q|B&_ctBs(&D<qcX
zoc!v6U^aiYx<!d%ZjnP`vRJUNxzKxh^l*)~GuO?Hi(4LI)6G|KDJNlph5{76f~GO8
z#{1)n$QS2XVrw{>%%_yr2RvpDZ7H8vH7f;enqq)Dj^o<B4e`Bqff^-~V;8f3?}@yK
z4QUQwXufeYqh#v)yWf>yS18+9w!lfkL9RkzF#lM|<Zkc!J~?)qai~VoOD(s(YH=!Q
z_}8I7L3VRs#;jnFGh#1IMY=aqw)ePy22?qB&=kP6mC2FP5wb<<nOeg)jc+&AU<zYg
zf4z<M>qEsrnVPY2m2WRC^I`hB40ZybnXZrx-xIp;$8!hG6Nh%vGei0rjRWE_D!zs8
z+gtaluK@cek45L_1A}>gH_{z?_mu$X99jI5e)Wk%(m_dk7NA&VluF3b^?Iy9D?^R6
zu7rYiHJFg`**yswmd#cbe#4k&!Ml{&1%lZKpaj?QXdU`43B8aOarp-rW~Pt)@_6O`
zRF^G;mQ>p_{IU>I8(KvHzY$TXL6v^E(1gae{o!Ru?V<R_HsCcyn+*Scn3=;XQ1ip$
zR>)iV>k;CtUWzlIBa)zi0RUfpIOvB(Mi@dy7Y4M6aBK9VuI#4pz46gJmuGC*4{dWO
zv2U&JWB1<;25=D5xVo%NwG&B=B?D44&$x0LX;E?>mpxuST=7k=Q-`(woNB^Gz4e=%
zn)DqDk7hj8SVgQ<6h_}#0jHcOazKRjb%%OtFK}HRCU(oEui0&?nCo|$h6O+rLk&`T
zML-`R)v&-i{+WH3DsSE)f2LVJU5A+CFn$`Prw2q&t;TvLh#B+{vfm^O0(9mm3KaLD
z#jVmL1P4DX<D%+=fb?oj<o8GNW)ZB_Uf%v3>}yYlA{2O8zhIuF-VJxL1M@N)iJ6tt
z^d!&W({8p}@$>=v%FG=|Y=H+%2}_X^+TI9_DC8V|51OAncT8K1nLu0NH=$_kP(-LW
zYSKZ1M2E8U25^Y>O-9Xf^=56v#g9M<3*gNx2FCI&JW5#`d;ObB#&Irhafyy+)K@fM
zA0w&yIs@BX4F1qhxm<T8&t5s?kAkz!g+f&>qp|ktGvYgUgtu5C=tt@awupxVB7N=^
zX2l)Y{%|I39d-^_q$_pXGyKrMOqa;{xR3%He^vX!sd_e#V!Yo#rdS9%lq*is9k5AP
z6l`g8Mw2G9<kC>Y1jK3+WI2>NoRGGg_Bl@1e99XXV{$Y`Pz9U2<r=?~-}#*H9DFEK
zHxGzslh!@2Vuq67gy{}NuF8*v4G#nYx_n=RE|UaPA25j?=8D<C;)8$G<b(DH=^spb
zI}eiAb)5K=?!-KxV)aY1IN+*|Jc!@jw2u7bfmeW(vU0nH#j{=cxAihgUZnULaAJxn
zOgo&fWffgRbXINyFT5ACDW55^+Xojq&4yF>*bmBWyAT9WlJ#*-`jy)`P6fS{u(Ivb
z-qqd%I+o`rM%Un1an;(dJ7IjTAs>8V>#fDVqv#|*tA06|O{y5w%D2ZIB>ot9O}IgQ
zz{EM8Rjxrf!WYGNzNwI!HUs6ra<LskR*!)<nWo}%VUp~hFQPY?k9&7}k_!hsQHegA
zKD8f~bHQSKi>Uhxfs=oS^$Gq)tmY!mL|O{XONMy6JnMx}xh1~asj;}d^A&QffHN(v
z@-69xZ;*#pmymZ(tk|Q7?6zm=A_9>vF6<^=pIRk~`GT4?X35QOR`ppI9M_BfUJP_e
zWB;gQ+eH7HH8t?iHC<;Y;%lx30XYLS0)m~_@&@F@>e&ULc{~^O{bd%vl<ONJ>ti67
zRKb7xDMU8IYBeBEgwLW5Qx|)Iplj4-_+2Tb9FR}&8js>Fqe&>hB;bj$GtJI{yQ!;^
zU5x$?o}d--83~EE0>fu##VKZC>fyOcOBUi<Um>2Gq~SEA*gsJ<{~j0qy!Xe?gg|m_
zRi2O`k^X*UMhaWEXcH0g3tlNWihUfLR_`&}+j$&$pUsdewgX<$CLZlKm4D`3(Lq(>
zB|4pe??YO$a$3zBa{GOal#51K<|9}%eYGjy#U7B>ESjqm+^znG_meqpmiEl3j~ge(
zvHHU(tux_X>$joT7qy~+(bxK(T6c@aPz2XqNs$BrI%$`{GLgA&!0z{>whUgLNryN0
zL;pOSrqks;r=J;BI=YdpEZI?8w)2gci<Ug9u&#Y)tTb*~B9Kg;X*o*!wHt~pYC?$x
zQ+&Lsgm!NZm6*18-f<ju{~|BLi9ftbN@9}YUSi<QPS|A$a+hhyS<cxK##K9!dg_nN
z9i69B)(bOlb8dFZ8gw_8k8^C$(e9|90tN6NJmLpHj`}!H>}4d&)mtEfkskMd$fRbg
zd2zwuwfymu+r!VRXev)OU+;rKo0DS4@*FrgXyIs4?)y#=bvK(WD{d+st4WzmIvWD<
zEZwnRd=oG}+O6Q4x803x(}gpadP6U~$5d(m)Oo7022>pe-oZ*M7?mp+FWrMtc~W)J
zO5lvSF`>F=q@B{)KH+C9jX51>{*+I~n`VFjTBoYEmX1<sFQ6sx!$qzSn<0PF&J5|a
zKQPR=7@`|fs7FLc_?cT}h`Po0%PCX7X;O%6XHGf}Qelebf*-xxyIrLu&tw^;Q=%aY
zWU>`*%T;J#MOev5?bUISZN`?;(?$9j-S_0S0vZEaAP2kR@siO&B3v_NKCzPWX418&
z7V$BV(J!x8JO5}YXqUaUA!#Yn{xYw_E5<Sgh0)zL$(^&T7)<bL#4>rl^rDSsb*izX
zGZgo|Wowa1TJa2dNtR_s>x@H}<d~c8^zk&-i9#!0BjN8bS2#AoZ&%>y^xs`xw$k1p
zEXD}-ZMekPXn3RG-s_u{x-ud~i_GC~k-DYrcT35*Z?as{$d?E)LplxheUO&ahjak(
zN9wy(TVK>Yq|(w|jXP}L9G6x}684&sdWG&R;)Yr35_E-D{UR?v9Bs#IhUw+XjZlme
znj6=M3_vYo5G#0fF(Yh*`4u=3GueRR43GnzW~FtB>3&_|@S6@8r<t>nh7Y?qbp+qs
zzB&-#xzL*y*hm0h@(1<SXGPV%$rEgW-0a75m`vob&qD>^)2!o_#sq}QzKeaMx0S_y
zxcLO%g)D{}B~<RQm2oLPw=bzs`V1sMCzGNv#CTz}s4OjxEcM=annwcK)l`FR!Ka`|
zvfsd&UBj)_X+yJ}9RYYs<slA?LGK7zA|(^AY36d#KemlNq`qP8Fs)C(O{7*Cf7ikU
zX}(-#L;j(BRfOfdUQFC9burJAqAWhCkdcVb<#%b-$p#M_I({D@c}|Dod0Zux_)O=h
zms^Q55t`{S?wtxlU3-46tB>>Dg36SwGPBesJ*1+TB#96E8nZ8pce#mHav9R;4Na~#
zsGj!4Kf~0*7LOpr;aqt~N6DmTP0ds3Y3n(ozMS)<R<^C-h3Z7HYkVY^g(vGh)c5ZJ
zy^Nw5(noSHiF$s70fhdB@!1M9d0f0IWCzdoQ+as*2-EiF>Ng$H?AxQANg>Vk7@xeH
zz}YZjD+!y$l!q5)g3eEkAm|Y!^*K=gV&__8N+6DjsKTe)#$HKFu|C0QO{0GQr`9n-
zK1^g@&xmwZ<DLmVaCvI2os#%UGcT)LV1eAXqvxT`(@5F08-T_{ZJHvM<AY@I+}amh
zyo3856Z%QxQi;c(j`Ico^e(%^gq;@(uy!Ap#|fHC0R|dCCY^?#FIq7I<Xymrg+MIE
z`kmSZ-VXI9d_mns^Sb$&ZDX<l&69~OVaM`~KwM<g&YD+%^bB_y-IpZP@nImFj^)qc
zo-?pv?tEkKQmxs(?HMAw8Sl&uM^`#`R%is3&l`GfkRKm^mOsxR2fzc?U?ozaM|T$4
zuXkFZGFxbw3#}sG1|>(qd0EjDd&7$dtVym9YCc%WvCMu~J>PCZStXlv!VF?<jsFtv
z`LA>6IbdGV4Mi-nAA<Xk!`hX;z`~<4O-w3MsX#Q%VLkiVvU{(U2NhqO=S&&JPfu0U
zDov|w8p2h!pVUIaky{Lbw3?;a?~7Ykls7Bjx}Up#hm=|U?&1_95E69k5u0(SswrfH
z1+cl9RI7_k<*HK(mSn}*lbHxi;~{}U1CjJ7fGe7z$0&1Sc&}(wNc%FNeds-1ah#9T
zs<(k&vIYyZn>6ys59-R++`B-ypNu^J9X<TxM`t`$#dG9Rdo#^&>_+}XwF5LY2}0vN
z<5)XQ^mJKfD4gu`2hlm*x-mtSY|<`_G`^hZGe`<>iOTQiDUEp%K}7y--AA9|+qFqN
zz<x$x&|R6URoE!KO**8-RNJ%HJJJu*!35ik4xs|mZVh>YBAtNAL6IJ-EpL_6Wu^x=
z*UXT{NxSm&rksD2=pTs2^VRgO8tjm6mk{0~;B@f=!ZlsbzqyAPtlAiJ<QjwVez(1r
zfR#kO<G-SyS<D!DdKLKC>-)3)CAQw1K>F+P&3PS8NuGZq;CkA+WeehfUs+@mo0gr$
zu1pl|P`{U){m1$N5f0I*<nhx1*tFRxhBY`;a|*o|*dl*mb-l|U=AQ0AiK54aZ8!8C
zG<zH>xBIq3l&f!90`;^KvbW%raS0D{(gPERaX<)86(9AE-g+vU-AOJZW&~;vR7o0#
z0K|*!PtM?*4(IJdX3*HZR{u8G{&cyc@FQZ?#?q3GQPHzPW;Mx7S}A?u3ryaOJ3MOD
zAZ|;a@(izhu5QJy&dpQikFRLA`4yypmTx1sV`C1CR)Vt-TT&a;7V+k#WkId%*APU?
zRsg)X?fY$kVarE3C%d!I1m**ymsP`;t)_c#WG{G9@V4Cg=ddNOKlE)2SUgmEmWL@(
zn{o7ifTh^oQ#QHl*6d3Q&YJ(6=A5p>d)s{5<C^8&-8U7M$k#?u=L`3gP7lKM2Xh_@
zB-;;JjkXkk2QCk!MoI2;V9L=I4+Yp*t#MYl^Uimrj}J?kO@5110!hy0<JJm?Z3!iL
zRG5RpYJ`g>5hsM}41iMyJ5*KAIJPU|;3bZke?Q^oD9kEPy-b1hPd4!~coq7XqA#S`
z>>M|Fi}XNWPUCg1vgpIGl0ZigXc(aN`UUn7t5x88Y%dXNRe(xsP7a;TI_Z`p!ReX7
z_mHHYo7yjaDVa@z*|({bnX&B<$D3DzTFLTi0PN{7o;c5mZm5gf1EL+DmPPybxqw<w
zM8Z97V%M~FE4(ey-hg*=)mw)|An)exN^l0?WYfKJ<;?ncw8^Mc(m@4Z4)+czD-Ty}
zfa|aR@U|eTM@#V8q|n6JZY3!D3xz^G$>(k*0nM<l20%w103{4a<agxL20`XKLut#g
zWqE{G#O}Z71|;58Pb_XvxP}jCe0E44=9v=bvFE|?DtbeWD7ee3g-PZcP^X(23z<56
z`YTh8bn{k}MSI6;^~?PgT)F0S+oS_^!!8pYED}-QUBi(U8Y4qV6g{6r(?(n^kgT1;
z>gA3E9EhVe0+Yt4u>B>YnLaLS3VhCq<<Xqo)|Tp0{-D79zPv&9m`rXP-kDv<95wMs
zNd3{tNKiV=UcIFO;f2sJmW=AMU)me_)-oyHFz`iI=_qm`FCf;9_Lu_}#<n)`$)VR?
zbjoR@`5q+4vwp&Fz4R$WyC5}oCv|jHa_n;UQl~S%^PUXb0ij~kNRQ-4B=l(%cjRrl
z<4(|g{el?lbdR&bJf0g2+^<q89AEVrJq2eA_eEfjL&*R>Gc%XOTJpkGX|(ZSvZ9W5
z+OTQ>1<DeZVE|=9$*gg10BnJ`f6BIww76MLk8qTijS*QyV)kLmO+K=!w+|*#@+WbL
zteC~q<L$VhpRMtzMe{W)aG)25CSJ7yh31uYcj!dFq_#dK){4caTdWlMU`Tu|(88e8
z&A)M`sQdHDoN};pZFk+5@1e2{=4W$SlLE^-F?LfMTioFBr3^LRQ1rO~j9qSF>2HPW
zX~#!Ihc;dB1I__1;St2D(FrJW*(%M33%`3zuS(`BgsiILfRl1meF#*<&+eu<{35&l
zix35-Q?qGpc$vX9ARle_mN6om_66_-VImuv_wWRnU@d)N<v*51D)CIvcr27t{4=Jm
zTf(>?Qb<>d^kkM#sVc=b|J^i9KYbwjKqrbRhr%q~eJSgN3n=*fx-j7Wu?q9J@rOn9
z6Czd|Fm7|%bXp*Intb;`4j8z|G4Nc0^OjcY01v#Nx{Yc)cYD>()f&m@4Jz*%VgeO+
zfJc8?)h}kGmO(%D2z}%PvKTI_?+$Tad2efnWMi%dlW8DesClM=xpY3|#reZR0yy5w
z!4pUb)J6{N?0w;+$65OBR^WlM5}PX<8V_U$F2KzC^hiW^&+3=6taz_jBZOT1;=*$*
zXnW!cFT<F+!!Kh120TIBNHy#&UgtfP0xsJvb&H`C@<r?55pTyWXiU=Zn_l;;vreo&
zEL%YRKw6Qse{z+5jF(MjK2e>$-${%|9>qMEYxD&gUdJDmp9J8*Q5c%MVT~@E1bXzB
z`K-R9#x2kI-NIckYPQ+%e{7mFDu!$q3NhD+px6YOVB^VJuPenM*w_$N0qd4NaL8_g
z-6u1E;>%`&Tp_mSF^nGmoc-%VkGHra#$7$Fjr%g30e5Sx2#>Ajc4O*NB|Ha>TS+ty
zfC}sHG;gkW4D5a4(*I6zNz{sBFxKHL!9D0)+!obogvUY=3O#pt`V7UL{R6noPjlQ~
zp{x1n))yN&ByI{S@aw16xG!oCzqV*lkTa72*kSYI<MiKX;<!qF#Fi=2ZlahEKyXb}
zyJfZ-clCHnK0Pp+MH`Io#rFn4rvH@WWR*|S1aA>9GVxuzwCEpo*q9cUwK<Elj0x;g
z#p_zGrRrRrX6gOej-M8fnRZlH7ht{LJ$~iNL?H%?Ze6WwS;Zf|oPIpUmpqA!K5E0R
zWltCx?QXK(3D?s85k`iMRhFxuyN3K{D64m3bpNO+F5xB_?OU70^q6g^uQ|oe)(y`M
zodV&N)0`9kM?iYr8VNz~?(Mu$$=_+G$9|?E9NiRNRd=X5>=aTVbmXzT%HcQfAKY>9
zW`nP0W1@DA1H*RH;jyEQM32ra^xgU}wak1Ng}&nyNN<wEHcA=F{*u3QnIZ*fc_Jt+
z#%gGWS>_Bpo`Y!BuvuSx&)(-Qr&bs-w8ET75OE<sz4%c?jnAz7gllE{6ge+Lz2tJl
zg&WpY4%&6pTQZxR8-gcLIZ1JE_m5j3JuG*c;MRFi>%JhA?G^ARlu4~YL+%tCwey@U
zgub=GCg`Zo>|-dYB-iGnY@YTfPzd<S1{}Kp=I>^V>r0q)D2B(763a3YrEw^o%6Hg}
zPP7o)x0Ywp2-b!rb|gZ_S%0@3L99D%hE;+nfsyI?czNJG#q6j_`Lu{SN2J6GqG4u&
zdl8~cKU{P3!_yebwr|1<n&}s+_!E-BsCx#)vT+Xm2Z(4cQOQ{pxX~?HNf=$eUM%%5
zb0_NbrMP&-vn*jA8U*sYT841lnD6117(DbONgvVf88TF(7XHHqoVsJ3Tm$NRuh9>t
zEN7i{_Cc@rDx9Nfh~+_)c8W`wvXAy{WT}uH!^Sa?fOz+YWhI9pE__h{Rjsj*mQph#
zxh1Ln<`Em}PO`Bu+`89tw#GLc%w%$YjR4##GIm;`*-W0P^)*nsYkUHO0^J{Q-qzWD
zY<6b*Pbta{z{tFrRaoB^rlUX;`u5+2+M*{A#5j0Lq_iF*37dAnqo(Z4QY#}{6P)02
zX+gDC3Hy$X08kR(j%d_Xx>4qyQqZD2Lx%@ve#EOl5VK;UhXW3iM$XI;1Big{OXQ8>
z!+4R=vew9yno%6rObFM=GJK_J(+W0{ccatBomK&+O^$k*pAhm|nG(Bv@&nbgK=aZb
z1q|L-nN}g0yXi>11AMb{AvJlXRq~>FqgTs%wHfxah}i&|k?{0|262ci$Hfn)--kmi
zmk$4lXxo7z`H}FILkG*mW4G1}eX~9xUf*_Mq;gpE_8MpBak&wbmDH2upe4h=ExerW
z8qknaE7&8q90?pU>*%fCzP(&UxBt&s<AEig1I#Vun&M{8pva&!cco*2YL{g1bNZO0
zL#eAlMqdy2g(Brmq~0FGmr+4(C_8<5GoJg0&O>6=zbUq5eu5_7gMo8{QVJ{_>{>>)
z2}bv9slOwj1uGG7kjOJZUf}6ckAHzsm2U`;QeBjc1&XY*zwRTg<jp)}p9gVr4q>F}
z7>cqNZ!g?vH}#}S_kC*Y+CifH$w^PL{w`32k;%c$UsuniC}Bc(n;*KCa<pC=*&`=>
zou4{iG9aV0DB&esM*T{JsEAB)KH%XyO9Igyu2;_ikWvjd>;@#HA@Mg0I&Bb_s<f_E
zOaku5C`TQuBPEsrfI4oiR*n;21FWsDB+6?xV75652m}g0)9H5rOm<~Lb`7F&3blI?
z9Y3^20MEpY6})-LRj~EBgu;+@rLE{@*M?K444IH1NkcE~*?L;>dRUz2ByGL}>?Mn^
zR`p@PR1k17{Y=b}gNf_DPIM<*n=n#%YW-AMkR+LYDq0$m6l^0(*GfoVnPWw$vL@px
zeLMRTM{B5$hOsj^Y4F+IX;1{~czj7?3Q4EXlooQ%w?fjZ7@3j`7FH1{MEb-t49#c+
z+FnMF(5yy2jFbogs?H_x?R}xL6awFxW&uUVF*%$&{+^A$dv$8sOCm<?T=^-)W6B%W
zrp*jc$w|e*MSuC~MAL90QKh0gG7$MgQ*&uKSk`cznAPtk7pBpq^=f`_Jlov3%V!4n
z3)2^mfu(m1`7K^$3vSNqzYNedExKX+b61)Nr?PEIC%vg%ZO;O?10a3l29%ut_xWVw
zoSS*_)a3K#h+}pqrcSm3xT3Z69mvhmY`GxAk`B;_A{Zfuu*jC$3FEE<7%)XA2BbKX
z4yJbd4?mih@{IQqzb52nNN|J@=sSBzd5*5OdaR#q*yX2Euk%t<sv5|?JqC53vS&9-
z#8Qb4d$!vM!Di@|V)YN}sN-V;CMzQ{#ZQfsSOdS$AG8A=dDvw??mCDrR6|Iu@hMH`
zI(N9oT9Qw@b_&e$)riA_AKdRbjBuNbEL-~r>$IH05%;ZEcsI=^p|Cih0kb<qY6><o
zH+LGa7XQ3cEm~t4lYC5QWhAx|CLpzj6NC+xP|SJs<gb6wo=_H`%%rxwz#+K$T>4sP
zZbb`OW|!bzWeMAC#^r>!(g5yXl(PIo23e}AO~VPR7%K$V6n0ih$hkgfY%jafX~|HQ
zu6@E)ZFpmgG+vG(mMl`rs8Q+YH$W~Zl%Uf&oO_qM`zkk9z08i%G=elELz;hB_n?>W
zY*ATOJ4oOJ;g!<c%E+<zqf1tl^+}U7kB5{|7ShvNxC4CaUck>#cnX55bEsa%@-LuN
zZE}7T{5?Dqga8+f0px=`-Dl1H7-+X_b{61SdaSy5*l?~fc9Fpq+14nLQTiUH(Hdq@
z(<b0q;t-v!=to}!w3Q9@65f(J`38N7(Qu`2i=?)mlWdcqR^H}g<YEZ0G=(y`XHJDL
zPpay_E~iM(N>#eogPu@)Yhd;Ei4LqXE#KH}P`cIz-{UXFyQ~w34gfH@PvL`;P+eZq
z0qMJGp{mh93!K6=>~1VBI7>Kc%;G&J?F`g!mND)!8rZ}uLezs;(6V_VEmMjnf)mwc
zf4#tgxLg=bV|9Os`jt7XpDpNmL7VUhbN1u3n?-Ceryk!tim3Ap0J5aq;3VNxc_5>h
zJJOHn4k!B<gXxrjIxLo`uoJ93eJ5wp3n+%AM*HJjy?Q&#A&rYaQNV?En0#p|1nXBr
zgp-U6bZ3NeQuGIV6;Up0dqaY><y5tmDD41wvS}QMon}_4_2I>0^?v_MD(q7NZm$OP
zfDgkT?>X?sFZMF8<GrmH)n8WL7vlh)K6A~Gc01Su!R#u#X@z(~cG+siuDEo%9p<@Z
zBsUmV$(D=g+|6q05IS8eE=NX;NOl#XtB0#C^R`POGr}q0UMm8l9y0^zQ4JU&e1K(*
zV(TvH+BV%7f6ETdUef@WWQH&6cyHt%3lC~Q7T&KUj4FOGq*`9Gbd9u0*b1adc5wih
zrQ($!?XkBvgv_X1CL--@s$9>Ikp66!kCWcX^_>l2@p7#xe4F2`5U14p!(!T*ReQJ>
z-fm@GuF`?YNkH+7q{3Yk=2*==#&%syu$4m(geBwTb4Gwm978U%SMwuqIShcR#NUN$
zJO}3H-NoaAs5J0Z6sQlW<xA`0kcqFH^=K=Soi~z<oZgkm%>ln>V-fTF)xyHOo|Q48
zDX-SH8v`emr>OogAW&x0-UN@+agVC|0}`rA0da(zL<kkeb)nQ>lk6&J%O!CVtUd)0
zBE<f+oHXB(A(fNy)mg?Djg(#Hh%rcL()R_=qAF|;B%6)7)uxRmXIz~MGGh(P#qwUQ
z=rI`b2Cb>wEe)jD%}h|%R9XCf?FfdL;PNvis)tx8Meik_=5>FZNqLLa))KP={Ebss
zuXrIok>*ucG9<x0G$WZ<JIj|hdvm5Pmf;-J&R8zA)*{}%|Dh-=&aG3pLI#7<Wo6ni
zr?2mfPIv{&am?FF;v7OGNVNB)ud$wNpFl2+m|9lHxw0mMdUJ6+TzG|xGxNhw>4PiO
z#)3k1nC$3IwMVdY^ML&S?${^BW^>X%cD3ky$5`M+Pm`l7U5j*=h)*SB=$IOk2&cIg
zd_|kjCBMU0oFw}Z`ZER=wJ42rgx+09c{M6&k^8&alw_mMl(Vgf99<D0u<GK{q4L&k
zk-bxG>!)0HWJ(zy8>xlCZ+S+<0GY%udTG8E)7FrHAfa9&4ZBazRKf~PA%!j>yjQ%e
z#>xZI8<(0NX;YBCgeFcZKI=w%8iAQoW8_-0b<>uYsWg?MnP+^j#k9sc09=M*!Te-4
zDaJbb2SLPFj7J6q#M22_BY#gF2g=lAdM!qTrpEFq<LW8icp5Dsy~};+ylu&NKZI%L
zvHVgy&F4ZR4k8Oz%%)Wi*9@Kz`?$t*askRJ2eYldI2MZ+-B#D2OLze^j8(U+(to(A
zEqr2=ovg2yW4QviV$aw5K0SVD8k?NuOgWgFniWTv!+-z!(3sCti0dDC{qGJ;XsAAg
zJktVCb7<%fFpDjd;7XS7r5mILFoKhb-P70y@^k*v#soLEjnsS}eIq!==rSXFlv+YP
zundI?jcr!40Knf3xDkW>Q=GwK5B!?XiJBtCl8*@_Abau7&%Fk=0Kj0yn8_es?o@dC
zA(SN`dW-n<0znIXTmEx-R2GW9ZqHp{wa6ZEp@`xKV0+bFh3D%Ss|C?lh@bB1xI7H1
z{oNI>*&?=9^9vA^or~MXOngQPJ_Nmb7Ipl=jG{xAUg2dTf3w2xMH%zo^2ZP8?xSR*
zw=9-6xDa-dW%u>9HIRP*osvJ++s$`JIEk0UI-K_I@LYT06W@|6BGc@wKVCqtFEdA(
zBe9fh^sMBTF-o&pH^N2WNa<CQ!E9aFNI=kRQbK)J+<sz@W97F{*{cEB-D!)tIkKJ3
zw8x9Pz>&O(?+QFETD#z=Yp+NF2!;L$B5gbjqLf}vpY1HkrM)YU;=)B9F<v!5)~ECI
zk-c*C3~AnB$lLsrG0gdEHdnnDveHc_B+~lRW;TNXeRKEVz5ZRx2Fh}?TG@{+1)#rJ
z)^R`~Titfz7%V1tZ3j9)AbE}N(}iMN@=%%{I&G$NAgliEi&aY;*RKG!ch9Z0<e4TJ
z%f%8~M3_Sy?W*r!rFL=+)dxk={~8GdW5Ki?5o;TGWPRin9?+VaPgoqT5R)vMOiw+S
zPkft`x(tVYFa?ud{(_mt7UPBLht-k7k2Ky1IGqD9UlG6oqB=}uCf8qNa0*}3zz$cA
zu75fKS3DrgmXQqP?M3CvAGw@GUAjPzK}xO_3JuHjHm<WyEZN-U%nWc#wKVq?SRhPM
zB*bt@v&tYReg?&4k?&3cf$@YzZsh({c2=CEDy@{_?qb?4J^c^-Yc$7|SvS!?0Ckvu
zTyFoM5>Fv;grq#w%x%@M$Of&BH4ZD<_R6d{_tYp{Auo)%9nzQ_`_aMEVq|9;%0qDq
z;dGBdtC;-jcOvx&P+VLByGtfN0LxQnsVCri!ITbx5I9?jIYouepl?+?2RKRS?1DlI
zkjcNWPkpNm!_akztlq)o$l!UWI-<=rbcIu2<1@VJ(Ee$@ISKCSJIIFjAeX7TGri0#
zMgk6j@l|eTY3s3p?YxKlro~R_)~aI?^fl{zk_D3t=z!^ZJ|X}Mn&b^CQ;~^beb04!
zZLc4BlsvG9a6m{JXyj|naULKFxuS_AWbc3jbS~d4krn}Y5-%%%S0O;D+BD2#iX$v5
zPDPmP&u;9R6_FYk6(4}q;cd<2Hc&_~jzhPfir(8q1G{K%Beyw6ju(inXGS5(VwiG@
zD}tJz1QA(r&TcWRbf~?Z_p|>bt3+aWAHE$o%bKE0Sk!<X2{CB4S=C~SJXibR;INc8
zW@^l+*|ml7c#7FH{RSd&4Ofo~Lt(^<*Kd`ANaV422ScYQCp4$rvc5R1(IJc_$HpgN
z{MA~uNct>fOGNgw)GcUBXa1=)XQ%V@!vSOHMCJ|&&L+WM$b$Ux8Arm6;C&#0v0H}T
z7H2){XdVTI3p)H2?c@r?fQENN(FXCXF<JX5Xh77~1S*%iOAGv+atM|~P)D8#`O$MB
znBFQsb(dSDbFn){FuVFG5UJo^xP^q_8GP$5zA`V=l?fhmUI#ol*%gBcqj!zftrCdr
zlq<~x->!YTIsB)Wi^xjWa|Yo{_MDl%q%YOog>Nws-w8`<bN(K^GY_M(o)`pD^f%hd
zLBL&S4uG*!Z(S}hN%NL6Fv-2g`pBWOha+9Xc7yX0`Eqm(AhbS#3BsPc0qaxFt|6Z^
z&x~<=7Q#BlV%J>*G)|v_UTDyq13x4eaxuEnBKMm%V+Fy0y*8!364eq7>7muw?X`74
zicILO$6I}tb@T%5<)LY%F~X;;o$Zx1<c)P9Y|P)u&J)8&nPbZ8*FM&`y(vr1ihFqd
zJ5COvD9?m$yIAm5(wt&1v^!vID_^X)5{xVy$MKdSbKiDwV`gDuL|#?#jIQm>RN!ii
zyUewB+11Pzp#gHAbvwYpv6m8M{MY1W9JuO?)st#d`OUJCR^z*jQS{@I+j+OL3!Q*u
zJ-6LcX+7!>&iLuu+@Emq_UE71ftrZjD-qY4ld>wAC~F-~Oq+o>0RQ$UtEm!Pgmv^i
zDDipi*Xkba$cTy!d^IC6$cH&@V>)}ydTaTVC~rdFG8XIjYif^8y2tlZ$uS3d5&Co#
z`J&INXJ)z5Oknxw)Co)&WhL7t-(KN#Xx^?qR-?I}8H^t4UrR%&Xv)kV4cDNpC(E<q
z0<tel&rEUgYAhhkES8cgGsibl8FmG(z>OoEDNuLX;oGT$LO3Sa!93$?j$%$rTSNHv
z9KO}C?P4#JBvok(d~vK^@0WgW2AF5@6JVxgtLg>%Vw}UyKWSUgzCi#+AHb+ee*j6{
zRxhst*Iun5V4lu28{{h3P;)Xd+p)r5+rSp)kDjwh>h@%8-8$WNAlk0=)%M7U#i`AH
z7^FW|I)bX=Y!E1nR9q_{{Vxf#lqh8BO-`Ui8~d=tk=0{8WIe+XOX=MF%5MD@vPgUv
zTDrOG=r(L_VQS2v96vK5hu@xVf_{4>aCu`HZ&S+pY6b4wb8nS%GIsajW$R{nqVe#G
z9T|>tV0rHmgK<c)kExes&3F;>qsPYC*4j3qP!lHie5wyunhd7??2-YNK61UTp0)d#
z!gxrI5Wgi|b8VhbP0u35W-76#-xLU!O=ZIyhlT5kzNJ4RA|Zl(9w+KAC7*DL=5Kvv
zzXD4xt1=yQN@)?t+9j{`QXJekK1c-YcC(#i>MkhO`B4s&qQ9n*-t9#X-1@SVd_7Q;
z`&^{y0f2}3LNM86FGn!>aQ=*vzTDaF*?FAAHk;&l;#2eC6BLlpNA-*OsJFmCbT`AU
z|K21$+Xep7dj7w>JT#x#$eW!f{|>y0Y(-pkiBfnTETZdCXf21-feMqWUn?jaaq&|0
zP%?$07qwzS19jWvX(P1C6vu)=|Mc^|r21~(Dw)RXyAT4hms%Q|5M`yb^#%BfJI(|d
zV%9QwVph;~@zzC3HP@7ID{jF;UbX?u@D!wiQ^R|BtsQ|+B#lI~(=z-*6~vMAFx~|-
zlhg03>q%obSY6|(NnZgwYm<+mP~v@I0&1vK>t$=Bw4CFjgQgv4KYDA&Ir9v{Nwf_A
z6<zh++n$oWN;6;IcwJ<f<(g~YSZ+U}_;7U~2eDen)I`t<ZD9f}5pI8iTj^;Bsm4Jm
zqR>COPHT4@7XY?vs_J)}v9pClvUk+MyC7-axuQVSYUHx1k^z#q<NspsJ)^41mNihp
zErMb|1VK;`Ng^2m$(TiwqU0zDf(l5^VM{A20wPF|sFHJ%%mx|($*AP0<eY=#SIa)?
z+VtsjZ@=3=-Wc!vbKqWU%~`o>*8EDpAhhpN8RjcN4}H5fZ8({mcY0VcHp^|zDCO~C
zw^oM3XNkh9p|m8(ImO&_rK-4ac4N-?lRY`rU~s8LjQsR;>Asi^hRzZ1S*LKSXW|_C
zed_W(d>)>x$BhSAjSD&m&$G?fFE`KB*cMUUZjgU`wm@siZ}n|KLWuixev1uGIjqml
zu@z)n>^DPE)5&*ofr|cg0*y4IGDmlQJQS?5@<iWmEUO}4yjjyC&`EZTX(W(!N8b@!
zxT+t}Y{xY$GFR|z<CI<9$omuHMNiVS8f=C{&W8VXTQELr*<~5-g?;ZwIo}$ca#8Eq
z0(DQr;G7_)yr+;fIaANk+m5c6kuF5q`SIqgO}?`8IxStjfL|J0&)JP@%HN30UC5=2
z-qOiMzx+GBVa1VWb>5KA%d&l8ABvQZct%uy<OmGx<A@OrSwrd-S(o`wk2}qHXou)9
zq)LUS+i<fIV2F?c&4FD$6^CF*4bR7EqS}r*M(z{)1F1s$Saz8BUnBITA?SEeIq^iU
zeTgr(xkBy%PHNRiHs_Ok@`CQ~WVAgeq2IB5>FMC#^skpA1PP#DhdM}?m+3zAb1EW@
zogH0dHx0f%JS69%_}KP*W0WCyZ_RfABT%lHEUnXzdAydoxv?7aSmT!5Epdi<qXU62
zh=*w<n8b{y6bVB~S}83J`g};_b<C5zp^lIs?R&S7%B$+VMb-n{lOa`=zlpQ=r3ldU
z)&m~iy-tNi#`g-tQA#{=b@XFs%DZN@rbjd;hL*UCnKGwHp&i~f;(K`{UB#+NF#&1T
zftw(4k19V|FmYFsH3&JsaJWUOLfCh<I7>$(_0@<ghPh=0?u!=hn)zh(Ppy*`3q37t
zPgam-7&Sr!2_Sx-)9|?_Krk6>@0;{1jwKO=3$#rym9<nka{FI2A-(;h<F)eW`$Qao
za;V0t5kv||eZi!3@!5=j$M|6@5u~=tFURb6zdd#zB27hPbixAkyDK6iBNG6DGwVBS
znGhUnpE|^SXbb!cIW-n0KP2bDwOxfc2)naJIZW-U&nunvI1a7LfEcu=$!(vw%6H^6
z<YvqrIM_Za%2)q)Aw9VnV4D{)pYHv=^M?OGiY|ak9J^Sd_xErc{#wJaWMlwD$nf0Y
z_<#85-}>0_MS%Ge6h8VlTTcI)LeCxpugS2dXtpr%Z><UqBq%8eT*>=56ZrR1)xSo&
z)Hx42>`5Hi_BRi2|9S^`z?BwHuZq8>%HRCdaRxwhJWpSH5v_!O{r&&>yA7Ve6%$KQ
z=6_0VsK~%nA15>t_`j?QcqD0;BlrUg;3tu=Ccq)9w|zYSlnfO=21TSHyPWZlp;=lU
zpH7Eb`O-rZl7Eh<&x@yF<9q5GX#Z!9{<VDLp<IZO6pRCc7tdL3c^PK3TK%<=`Co_#
z=_SDOyuT(2Q1$=JI~x$IaWPQlZ~2w~v`{K9c;VM?c3J<F)rdd@-jh4%@;|j@Zd!O@
z>CD`{e@e(DkAtB+e3RGbpW3p{X?WqIzM-~%h{^PDGXLnnyci$@H6cAqcc68|(2|FI
z>j^;dd#Nul#wJN?F+D&LW>S3KjC6g*L1)+RgA@u9q5)oXQ;NDEcI!Sf$!%c<=qIme
zy|T^KqRj*Z!EL_dD_o_SU99PZ5dwHxg~gF@m)k?rSOOr7#Mqae)cUxvhQxIC2p}r7
z#JzrH@X9DJ`bfRoGbTD{PB|qI0^wWvoWdY}>h~-V8CaqY13jIZ0PsY+7OpLLT3pgz
zgsTQoGnYU2ysWBOUDu2kby4V;RBu><37N{!-Kk^+DVHJw&#Jy8(q;Ks5ds_4cg2Le
zGZ%CJ1ZhjoMR4A-XQes&(>{l_;s8%6bpe=~dM7z|uqxl469FF_eZmq3Q{i7KD#bMe
z$bnadH{f^O@It_d?CTx6Kl6m{#M?e74+OK{l1v~TgYhYxcRe_=RFE4LHc~U;=hjs0
zp?;!>K-VUSD%~OsM2?Z#A@*W(CTE|39i$*2oyx>3>0d?j08Z-$0)f&!@F!e~75d`9
zXFPP`!3cyCKcvzvVNMXhD<U_c4r~ly9&aLja`VC3pGlOHaVw*vfP1D@r&}|tm}0N^
z4uQCAqqE_TUYiXl3UQQf0brLqQ7@IAx@+RM<};fUBA1mHfYrR~oNyCDgr9SwWG7t+
zVL5`2+H^^L)~9@EVKu*zwVcCqw_qK~D&_nU5D!HqF$DqV;%bS(v_b$RRRUB~?u1wP
zf<izd6Nw2V8+v|juav|#h4<$7D)u?R@rruiV<&*rxZNV6>AXCUut%UZLZBAu0B834
z8u0+9c7Ae80D4t=+nF9XVbyd<?94@AZ;(_bH{Uq;iS}Nx`GzbhF+D=svrBM5#CB2T
zG{nO3NDA@t+03nwgYy%nE)I}k+<oVj|0n6FC3b|edeVKRGH4xzZoa%xfKk1tud*>H
z2-gJ^ks**{w%HWtu`X+UbRx{hOhYX5Rp+@BA!zZ#3tG5m$ZY$4F2y4VuuYH$v;;(O
zd1(TK!=Bmiq$A(G3{%Bm`k;|jB~w6TI{6cC)zjK=!&mvVHsOw&bP`{jzV6lzRGF+d
z|LM&CWu+zeV4;yDEpL6X<Fn72Uc0ZnxGHkL>@wf@eH|$4B_K#AuQEDpy5NdRQ(DE*
z;PY7M{c~6?3CKb;{;8@+1~R5H`KbNy>ruD~nF2^@VFl`Ev^{=1xy3-eBFlksv%<{C
zNV5b4O!Y_)97xB*7`9A3wY`oLceU?&enhH%qncCp$}8(rP{N8us+yK2YsdU!9g&)2
zdp*^Qp1sRS(=ZS%Q4Y{Ul-`RoV}oG8>~-PIt<f^po)B8ji`bmc9oH59J&(%vwQs~e
zZP_<be7RK!h;6wO++~s}GsVK0o_5~V4UW`}_{}blKF$^`^Gc=!5SrDZomy%RaFbcJ
zaPeKrkqn6`o$<hmi1EAjU5+?uJQtz!J9)8<_Y;~;xB{cNHt5G<422b;1ZJ`oWH{?6
z98Mcjh;Dc_wzu7k>gkF4V9;tN?fzy>-g=2?xH==IV30qBnq2tEYq|##<AB9#%MUrM
zb-^*BSBR+%>=U*1LM&J%nWfBw0Bd?$#v{7Fn{0nCH<Z5xxx3g;&h_!GdWlWPh30JB
zyLK4a*M6TC3}q9TLQPpP^GygqQ<k0;&Est@v%cYxGpe6I><?vFSs;HsHB&(+-@<tI
zywNjiYOaf9H-jEc0J0{7Ss)A)%lA4rR%aECuZGwsLEHM?a6vQUh`aK8I|NZ8pE3Y$
z&zzhU>7;>5@Ew3QGL?1#{8gsgS3}MtATXBfo7O6VUxzepx)hzjmdTGaYrCv}ze8Qk
zWpWzlacWBgb@UBFWPt4}+NjBmF^m(>IuymPCxK)nU8nAt5ux+pe|2%By0_R<pgmR%
z&`#Ct_*HYEz@C64vLj0Z<(_MM0sCVK{6Nfj41ikl1RPV_X~$z`DA=b47TJ&=Ed9F+
zepA+4vIXqs{trb9exh&;gfw`J7H{7t<NNH(-ERn{_Z_xjntq!TRi|rmn0or&&*tCI
zet;XlU#a7HM*S4gg|!dvGj6mJH9V_>P&&WC)4I}HA@1LdxRw|?LXx7o?Hm0p5~@6b
z$Hzq~(!a_qzuxB*`wC8{W|N<)o<};cvuj05&k%*2j;I5i(xl8kwY5r$VG)GzJ&{6H
zD#mcgU@HAB$<*m&ZI&C=q=)vJZbw?4CtU0=L~g$;i1)X=57cyTJDb7N!O|?%@cs5E
z9dbXWViW*+u07Oa?Ym0)^{5YKna8`$`KaS`Nwdrunaic5Z$6s&rLpK;H8|8;QFE`~
z&20?O<brK-x7bl`sRKF%mPniU>5IIBcX^fc2Ah!cy{;*M;QKm?wAgeMYC~DP2`Z;A
z7YEW9_4pYsb9=QzrJ&<eMj8w^j51SRGyXW!MI|%2C4hZSyq}^EhK-wi)Va5WR1_zK
zp$-2@zdIMRUhyZSXf_H(p384?M0uoxm`yKj$}>-ovosKT<@DE6`SlDCla{EY!Nu^^
zFM#q9{-$Kkp^Lvo!}rrQq6Z?+!)A`_$vZo|7os}XSiu+-RnO+@r3DZkl}4wT98Ylc
zec{ONR4~(i)QjgxHHkyO&m;|ZGdh5Aei5z(=kq=#rbX+Er>-$N+Zuj=<5Qw#%{$M{
zs^72W-_Sclw>`FWuL!$XIdns10zP3>x#*|wm?Ihv&f8$zy50hK6OpLnx`fkLkkMbC
z$5}~7uaw#yTwomXzez<MIa8c5$7Ey;HMsT<F8a8wFsXqrNT#>9Lw`{C{^>sDXZPJ(
zry$mmbu$kUH0vGLKkukRUj%s1EIvmi1bEA#<fNrsIALs>K&fT>RZBI~pl$?4c9Yl8
zH@sboBHQ!8OgF5|myUDJ)+bZwh3L?%n*k;H3ojQlnRcTZb*;}^(SXTMViDzUg3u<k
zTng|TO^>r&q~+v-;+3WJX<gp(r{%U!>BQ2;{UBnz6m8ZDC(FShCqI`2Z0I>!!?fr~
z4rX~(Jx~5md42f-+j6%EK!aN`II3$$;JV!j#n6a-SXy!w2Hd&dl-m&<KXr|HmAZaL
zM4hF2BWTU!8(?h+KAFhb0EG`H<e+aX4bP~XCM#36gxEG#8~~+b?(#gU(W`LWTXg0C
z$?=WuLoT27LVFI)osW1X-Dhfxz;P#lUq7i|@$IGAJ%Oqt>(@zG!TmY(mvZ7=<HtV2
zcu8!08=cIbk|XE@g#8$=+dOA>>J}^GW?!B--3gaD+M%oCc~2A(b+$}Rku*Q$`~c06
z5uA628+wEUgztYK@tcdC4jjKl+6{^Hw5_Be{zCd1`E2f5Lv|HgLS*ff<B$l)&?kt5
zl3`B4!+3DgLKlYho@asAR-P+5cn0dO$+4UeQdxE*5T!&|A^CtU&X=(vcpHPbbAtm+
zo&V%*yw+MK8JtGPH!;r}at;{)?o~M($#sOhz}B@=V}IKMobpx#ch5&3j0!cJ9twAH
zhFYoeyL2txCroS@yEQ}iIX1aO6)&P01af($WS{f%#TmVR=s*r>w)o9%GL&^132)k`
zT(&HOnV-<?MV+&=2m*(qjDT-SpT)`f(9C*=!N^Xvr+E#YvKz+Ur#{Y{8Y*$Fu({f>
zB|L5<9|nKvYht}(JO+oUaRA_98dXEq{<Wp8@(NZX$g5{>HlXbDeC&J@X#-OTV$vA<
z#NIz#Ej(CT=o{f(K-THhf1EB3oP_@<z=I_tF+?*{o4fZg%-(zMC^N%9AXoq%BCZ|K
zZLbS6KflQk1m-ij9mWiNe$jK-jFp+~m_N&!49x}$UZ#7Bp#eM$z>Gy#UJ7*8gY%FS
zN^QrPQ6P{3_=Pjh0f%pO^73qQHmNim+G%rIfR~tk;_-2>%g4Isw(;y*siGsD0+kp=
zfKsa%M3e^C?qkZ19n>>8<b3efBl!?%lB7CU?>3cmo?%9bH<zwbpTY}2puAux_XSYH
zoUGuU|GUuCj~8i}$TT|YUU^cz#+J%_;DuP{##%98^KYc_lN*HaWH1MpKfsB0KeOZR
zV|Af@EYUmAcfM~G{@;iB;WFZ@431qWcs%1qH*^b_S<SX_>|l)q(V!l_0ze$jhB5z*
zD1>8KmgA!Sl=VKLc)Ag!g}Ob`ZcTAxQkl8VF3sgfEUENjtu(GLpYnk6R>mFx#)_;>
zmnMLgCBkKa+!wJ)HH<5U*1LM2*9(nA`o?#d<Z~D^ywD@0d~f%70%-!w3@3K_W;J;E
zk3jA>?r#nzFma#>8j!+2@lZ}szV<d_@pT$>Jbhv{?7;NvXQy1Csf*F)DDLelV6W$0
zJR0t=C%5+i$@6qFj<A^o{5sqz7T~B2bZjevHRy?lJJmVuaC&}y5O9=cO{Lb?IQc*1
z=<9B1#-~MimmC)K=6OrvR{W`oS{vG32wHDkmu(V$NO1kA26D%}3A!FW3720vdY>m&
z4Xhk;Ji#iYWd7{=m^8<`8|Iv4*KYhNeuXkRNFhZ=3-{dtsHF$`_m@YWKrpiM4Qk$>
zv@@tjPA@2ALw%{1z8gjfXUDOFU$L38$LGsp?L*uL=yo94Y4iuD&9>F>&7#?tDL0Vh
zQGIV@XAk7^p?ejgA^bk&$+6sb_Bu;TVA0rCePt%}7M;CZHf5y}Lp&T>73^|)Lcke8
z5{QS??wHW|14gei;mUC)125e`@#7pxOL6CpYJZp0R*nHra<)VWSLGw(B1{3QKwd9}
zHvmD?y-=y+l459eJ}bY^OC%mL2M3v}52<z>mz#dU;#cBMUZRt=)}a|g>||<NN~3_m
zg@ox_Z|KwJf{467*91+AS2fMbC{9qFUlbC9%uMa<Ip%4`*!M7jqe~#@wmP#3e!k;L
z4DBoH2-Y-QO$?9P5b`zSE@AXvo!`xwqw}=9YfRS-pQ-K0u<JnUS4m9u-_9CZpUu1k
z_isWz>K=5@E9nIwbYBLcThMzFLAuS1(zD$ADPXnUMbg12ZqJ#(t+Zrppd+B=qh6ia
zPL~CutcVP^bqV^d5rJ4vK)7loPff<fco(`LzU-7xK?a&UiOIq^&Fp+1vTf!l1MalQ
z5M-gDZ*~rzvJ)yL<%<kD2V=Txvtp;0bMTW9NkVcFECL6;g7i~7h3j<Z=}qsPH(`0;
zAb3W5<D&&w^CY-@8=95a*B?Tw^MZ<bft^LZ((W|wZt2jYEg?P+#-3!QY8$CO5i%#_
zu5KM9esQnh_0^1?*+t*VxRen5?gaL3WUO`q(tK@HQ_;_5w$>)yko~RK!<%f+>?)&p
zyw%6f?-v{ciq`alBc>qdPLdOyd4G_EyBkyY-fsp5khnZ-xQ4F=E5{pT8!Fzf$fSCc
zvAXs=NMC?S1&cOuj$JwI6&$p|fi3}e*oQs%>`gL1UNpMsYulGNgjESUqwfdUUr8XS
zSt}4$eXj%Q5$Pewqzl-6j`S>6C-BTS2ns(28LV@V154LD`X&XYcFa-uChAzpz2aB3
z?py%VS|~i8TKZ{5PETsDfM<i?E_eCl?=8JSqRTg@D_O#CDOU%cGJqS;*BV<GdY|G^
zkvnex2fdT!{7?qCt6k<rc4#jk&Nn}c7xIM=K3;gbc>8B&UGZ{{R-|Y}M*S5)c;q_H
z_aVbT5(EJm(kOq;)K4PhMRs5Qi_{K0EiLXp8xasO1qXLY$Z_Hwh(V0kvkpy###*1x
zE?o(oXBsTE_bOj%Z^31?Lv{S~x94UIgu=|V&-~wJ&8hbTLU-t!f)v2>Zvyx@H8D7M
zJaNAioa0e>DRQsH<HLRZpqQNN-3YMpcU!2i?MdK?7D>@q{2E~`i6!}bsQqZ%_mMFD
z%7Sw>eA)U>!|9#f?Vc5%r$O#wtG~+O8J<$$@is4{&A#vWhxO#@{s1Y3x^d3eN5)HX
z2X`8%l@M!ty&beAUg^C+z5hV+43I!}VajDw<Vdgkd-LdeTd%F!6>Ui<@vUUvr_(T*
z$X>B5@syQ+sN^Ay`z52t>6R~RzQ~*ctB_;@6{3R#-o#^3WFy{fHmNZEBVG-r%v3WP
ziD;WHG%2S*H_6g7DZWvxRlzoiMV{naj+HfUkhVf4<Q$eKXzO>G?+`+>)LrDBYYZ=3
zU}$fGqDHf_UC@m=K+V50#J%^HDAk^wm2mBfZ=xN(AODb2Ew*>*46|9uF>-ZC1L6@Y
zK888rHFbb(YYn5pR}#8XVQ|#7E=ZG%NUb5i@~)f$OmsP`^@SE2(J=YqeZB?U%!Wa5
zqiXhb^DGilkmn*8`b{9t*LcWkAO-9a?lxvx_4QI~P}cG_JO*=D<l|vpfY6aw0I{wn
zo}{mq{XMEhhC|X#f1+4e|F)0*!c*5%XU?ZyO(Z*6u5wNCVe8c!(`h$vu)jZhpzQ2H
zsm9jbi}V7;FIW$sNyLz{VbaQWvF^GO%OFnwA-sNhXxwy4++k@lEZ<=%q-&`_OVq|j
zK6oyyurr8nF7%kB>ox*H%*t-kUBjyl?&3FYF-_l8Xr}<gx7IiLUhP~jDJm`*XZQEY
zP6ix45i#}QxHr2p=ZBhC)c111yjtDQe|tQ9F*Ak&oXDKY_7(HOn(5_Kcd<m<FWc!_
zb4|2D%&Sfe6jv9B_dh@PbRxS)oQsJNL%g5<kM9!iA3lB`MyODRul2K3bJW|_Sv*Cq
zf0bydmP`SJjMH-4xudhsb)lI}Beig~7QDdK5Xi^|-m@8HGhI#<(NDb%w;c5qmoiya
zqh9VBu4WenQ9FZ-!lBp+1KS@Y-r|2^d6Btm3Ae<t2!)p&9fkke^Y=wa`tCrokV6Pi
zel`2vJ1!j7`yl@{Zrco)$j+@bZZ_moJSh?B!cBe;5veky=c5<&ltX?wutGFC2QX4|
zTSr8~y%YLwtR|lu-2ue>u{%Dt)EkmL;3zba<bI&bCM7a#Upj9&Tm-%5m8)@w-gvQI
zcW+RpzPH#SD-6^ra%_}B1}gZtmfWlF=M3s+ax-;-GP2F$i1SPtmvdUd1{ly)bJ~$3
zllR>j5^SEGjNnQ*fXtm@tw=%Yer0g+smW=%H+0JJb{k!w_1*G<D?gWmAtvSGW{m(o
z)tww`U$3)<p!^WEO`w!y`q?E2r<4~37diyPn??c-zw3H2b}Co<PNXXi!44=x>|W<}
zpNmQcdTlyxd#Z@5*+T;rXVXP>4YZ{W5AAY7VC+2D%W%-FNP)DNdO)I{UT|^44wkRI
z20n?aC+2JG5yRHD%H6*R1WRMx!CP%0obqYJ`g3r1=H;u}O&_hGo5kNa@y^9s%Lm5L
zFg##J%^qZ}DqN(sF&hvNu3sKnQ{bBkrOQM0_RSejky4W*0%b~pOrBX5-ZP&$Kq-Ba
zqM=2scmZHEStspV@~zYT7ERg<r4p2q*pRfnPJrtNmk|0iSYk8VAbZt*NuJF3#pLQ}
z3$A(t83^*!gA11<I`8A{cuS~Fs@)4g&hUV&FL$jV-^u3NSN8z7Q#LRktT#^H=}xXw
zoJ)C=S-exWU*-gg?3tQ6ajv*$Esk)8a?Ptl0}eNyB6A%ET-)a7^sR7rWCq;m*1q&Y
z*~zfdhhklPE~wrwh3`Z)QVYk`-3q6644%)a5KI0LZ!c9@3gKc`+hXUCcy>-xl8K^;
z`gphuuLbno`C+jn`x8Iq5U!Zx?$DdAu=vK@QTW)Dpd3JM%S@c6CYzdv3rDSsodd+d
zwBh3v%(L-;YIb%G6d}|_;p-ZyV>v5PqnYZjI=T22B()E_k~B-djk!Yd1-#~1eD?0c
z7Ja0{V_~w)hdRI-R`0DOJui!Uu~JK^W-`4Gr2-gY_Wj?glyM5Tq88Zn)BQ<5$qcEz
zoCK*p1cx2+3t1s$8F%&_wQ9T;S}y_jtq&py^}<u1wW>*l71eo;Jmzv5k@i5!G#T<!
z9-!aSmu@M2txk-T3OdES6t;RL%y)L<Wr|}|rS6qIo4>3IeFG&HGZWQwFOa}PHR`qY
z>LdaQKjWWW5M@yv)W8%s>QNo}%BV-*%3YePFcV_h?uIs-29Lzat9yEBl`G;FyXeh2
zP$Ckz0~-Z&_1;_#-&A$nm<ylkG?}uk1ANUKzH7IwN5j7GdI-}(@7O?^(LN*<=2+c5
zR|09I`i8PB;j4P&(y}?(<AS4pN(>QPC&>+Dj9!|DOOf;^D69y5?m0pB(;(7w7ZfuR
zu9Sb|dMOh7*~G}asPQs^6TxLSmISU_ZL^bpkG{M;x)57e!`>!vPf}aS=)=9RuDDkp
zE#T5L+tF{uJN<)^DjY*~n(~onO&cRJ$erP*2qeqO2YEc*Q#tzu&Oz9fBNGVQ;=B0V
z^xJitbgD<^WTq9Y+O2lZ>+9TB>Xi;oI^3^%#Ftt)<I1@QYeZ3In0aV>Upjeo=J6%=
zoEl_|P5Qe1n0im1Erjx#CaZ9bK@};g4A(HWI~8-7n0fS4*V|saX0>8vI`<0kHb3Yp
zOZnl-sFCkN@N%YLkH@>I^d3OK&~}8ffGRgJZ@4KmahF{WoAuL$%5{Cm4mZi_eMD<~
zcO=6XySB*$|9A_$E|3O0>52;|=x2gy4);DTb(2R1MBwh6t-ccj-sJVY3S$UKn=X|h
zD5G_|oNRD&p&l$^BT}KpstaORbC@R}HIz~C0TYB&%gnqTI`vA1JPPB8#$f!Px8{AB
zcV~MvC(Q$TpTO;i2c<pWODK?N`zefxh&~Ooz7E|JfwcH30(+=0Za&H81n})_$P^uZ
zP{N*_V#8-b&0?B1oDclsa%yI=`SulSC=v7;$vd<^*zNgNJ6`x}?1cYK!j<SIE`D=K
z?T63HwTaF|f6PI=nP7ffZM-b`W)QUmSKjcOUU`?5>us064_h4g?MXsDny?vc&nakm
zb@+Uv7`LV5UpH-Zz(Mryfmp8#gnhHxzSo%(r8;;g+7mf_o){dcVMU3bq^smLxl}t?
z*h{DTB9l3KZ#$&hZE$Hnxd#7WeTv`!jL#t`vmTf^#e`(<JKrtPP%hYqryZSx0%b7f
zhkNvd(!EWYd%lXo$h{T{RX+v-tU3UwKh^Ach+-d=w<M)AO_cVXR}=nb3(`${hiUfZ
zW}vkBN3igVeE<0`*ya8E8wBYF#yLs;U%y4CiVzr~v5iyydxV5q?*Dcb1cY<P=p#(U
ztJ*YCjQ`IoM`_0&1TXg6fti=h-KG(SW-yu=|MPMN&q+%5IQnJeGyctw|N12M#BS0f
z4K^?zPev%=|3VVXE0)wdP4TaJnIcSCnMLpf@7L%4M23I!U)T*oy*<MsFt%((YhboW
zEOBZ`9B+fz!CkbDaLa43J9}NOwl*>Z?_<T11jhMTQ2{h-A}9#kt&>=pgV67h)W+6!
zudU$1=^!a}l9<}KA4{=<nYZ$_lSN4e)_V6a)jgZ#N-^}>86u2_9MD9^F77t$cvnQ)
zMS%L*+({4<=j!fIauhFhZV?=DJMJrl`UU}^CwW8GT@9ZP`%zD0E>bpR(FD~HpoPNl
znX4<V{NTf9X!(y=CKM(L*1l+d8TC?Z6ee@R|3G>un#8q-;5*LLmB-NEp(NkmkRAA}
z9#5xq_P{dlKL3V#BLQI&vP?~%N^bPy&HEd&)ZO0i#9O8(EVKFG%V%gV=&ZmpW3-9P
z@d!7t%dmq(xnTzVcp%O-t82bmaxYptuPRGQYHJErpt-HHcZVfu-Obne`oFvn#z0Q8
z!({Mj;~vy6x!pkmEL4U~f_{+$8$<@ygzv)0Mf7L5?O@}+g4R1wZzPscgI9j~y$)?N
zu$91a+3r>fyr)kP5IFh1u+l<5T@&HzRYJ{vBm4X>Y8H2r7<SZZ`sCk4{nFqne1>Ru
z69;<bf`r$14|jZ3ObMd7C3xl68@6Fp$#RzV)t@jOcqY=RuVhl`gcb^&N#Orr@`o;N
z><+@aUEd}j?FhF$_v7(~i=ZU}{;xzqG<60rnVZUW?JvJ?MDhq$`{>M5oI7&u`@32s
zL8!^K9I_eVws%xKdTwQ4YEyFP-61r?=o|K8!$aoFc;AXly|WSFWe>pn-_%FI&%=!3
zC^c}E9E0_#Dc|Ci`LQMt3#?bMPrc*CA*&^iCp+hlz@d6q3w~^YQKN_9CM<rrod~4o
zb9Gg;N^iqD1Dn@xI9^1*(;6uKE}@{`{}<~C2)hWqp(`Z6$69Om@Yq$`GpGj$2wwuf
zuKbP{Rb?tmb|e?-vkaj(77JToBauKWPJ=iIz{E*DA!d}67dK)oN##7dAEJNBEdVS?
zyCzJAR;Fm?2xCEh`$Kq7r-Q5D2yP(36Eq1>rlFX`t`n%oGfu+O`TApZs9y%s!Dp6F
z+_gc8PVqPi@o;b<uhk{=ZUO^g+fP0U^Pt{{QIY|XEQ{5Aj3ypA*nwcGByv2v)(d`>
zS#BI9&Je&I0cFa#?mzk4ug9^=6!3IUx(%9?HC`Yg<9<;hs3cxz1B02b)eg$nL%*|;
zvVlh>xDJb6IatLkO}(dQ@f7*0I&e%NbUPl+t$;h%xd^H8B+&u5lNFQL@Aqqw1cV+S
zH5|-+7U*whUV>MO=`+QkKR!&cAFibK;`s++cD4w;`_~TBpcp~la2Po8C!=@`{&>OY
z9^QibfZ_bO$G4uN7gS5uz|$I%{+t}WqDG))!pWi2=)avpRw<-^j0L?)Dn*DDVs#f;
z&|*mozAdEhcbhTPFKd97T{Q=zXwaPS+5xsCXSz!mB^pVGVBaeYmfKMe?^h85hM&AW
zfi6$%5A=gqM&1eyKyT+HON4Q@d2=$#CT_!;?Is?k$&Q^v**{60M~IET|22;vwaLMY
zAeJUi+dWYZhPxD$pHCyl#Pip8utBg&!MO|LM0jE#4y)|Tuw+KPkyu6=Ua2;lER8Y+
z*cp%l&-zb6Xg;QX0!dd=>s&zpu|XGJ$-ybAibvp0czl@+ZI&a}VUZJD=WNh?Slb2a
zxJ4uOQ>jW;eED!|FTP}e#W}3Z$cslcAK=bKW{Y3D1}+^wP9KrNbK%%=G<OVfpy7i7
zKC~#RN$Q*c35@#f9(v)DbMVSnTO*1x(8sMRv0vpJEc1V5+xcPb&Ga+q2nnNxER+8N
z(_8d1!To!cj09k5(S*~<1;SNRSc;-V5EIP>25+7B`F1oT8$ltJmFvv#UU?4K^C$n0
zn6X?2t8=Jd*1QH;rKQYMM_FJU1H=%?T+1Uve?~?NHs0^4iAB@h7h<;H%+xtN%Qy=w
z|5HS91@v*NPVljU)!6F)ZQyAO<$jaEU*Ew}AB4}8oVAihuN;ZFa!=gBj}Zny#WirR
z`S3{D4zK*oNzj63dGRc;+~0JV6!psef&TDLqsn|vG||;ym8lOOGFPL2IZW1&HAXOl
z=VLiwm3MExuH;2Oy&*3t8J<Fj{u41N2khIBO$05L5oxeUPmaUX$I+V70DnApz0Kzr
zEs)?D3{eh)&oMN2PAN-P=vY)=M7u(I?nGd2s=^}rm68^SsH+Q@<HZb7m_g2nsXQ7J
zR?ghT_TD#llVN;@^{)?9Q??pHxlcW@$T@9Bwpn#CH^fzv9Q=sD>x2Dx5>X98rXCjl
z7HxkTFraeGeh);^T!{!P?%Xl2Y1Oq*HWHwv6yi9;wD6wx!cS|YgTWpVEHFx|Z&s(w
z?lLST^GZJnsSso*CT$<6(e!)-PJu?L${Ecc!(?!gTj`PN_x+O_F=J2bwdV7Cj~`ky
zI6>3#YpLw&U-kyy(-3uXSEsbeZDXz$4<O%f_yE*%>*{svfHT!|7N^h)c|{*#-13{k
z^PRs4haqkx544rkPPc-b^I%Mf`IVs3K5q$}mRyCUpDT{%jPRZxZz#SBcJ`5O`gIS)
zKgXutdWq(`2Qm0(!als%AoDVKz6%eT63{zWIUliWyaVc>9*PM{9O*5;d$*_++Ar+c
zEit+&C#mgnDEz0`)!;zf+qnAd^3NS{n^WqYM0qT%Dxz~`3Rd!{m)8V?Koz{qS!Mbq
zq|j*u)<QQfkLAa%v2ozmg6+vZ6bFJU^P+AXL3<AF9Uu%RBJvr3>>Jr-IXf6OgR3!U
zNi9AIW=NKxi|HqS479<Bj0emVbiMW}u{&wDgz{a*?2uUDEdyRVC<{KW`%35f&n@q#
z=k@?H8>HRnBLA^~;zp!4QAm7Crv`$|3er(a)Ry-LK7c(RY|wmx=7t`UtMbKu#&g`e
z;Kk9dlP`WgoxueO@nW@Y_vF=zFMlRm<^<j{;Qez*IBd!^>HKrc5K~@&7%oF4e)7qW
z1?Y?-AJW3`avOk^*CVnL{y}+UM>1z%&%1^>rG9D><QY7WyLtYZ5fB_o+DQF)8ry~h
z53Bm3rZ%y9_y0_`c5A$4z-#RgkCJ`Brs(IEF~l`*K&VvN6_<YcZ0;XXC{+e%5|5kk
zksp*tb`%XtBe=sd0G-%?;ABpe|A`UBofv$Bab3sy*Xr%aTg^&?DOc|m>YH_V3WQ{<
z+Z6F3ap}d~&BiE3aVvxBS}1m!po}+`8VS-rKCn_p@8sq8589OODPwBUQrEDoeP^Lq
z{Od6tVM3au3^_B8g%&OWT&3{mO4oz~|HK|;OPYTPJ?IWMAIRnHXx1)b6yYoom8p0*
z)r^)EYS68q6ITS!{$hSHEuxKEqTc;fO5n}^SFYhEu;S6nbUM_hF|VM~lai}{{5FbO
z`yngwD*02bCJXGavG7im0!l{kAg=DjE^1WB?J8*uV*YM~7O$qW3^~AQ3H=E<^d2=~
zk8K?4HYf*Hyav{>R$ev#`VU!#)fzwnrDxnfNd(WRw1RQ<^8EA!&m{4~E8{-q<JEb}
z5nU9x!ou*&)VV@p3zlL_V%u;Ov~c3YNg<R`bS>6}J$h7r3;Ch62ty_yXt*n{T8YkK
zG5@w46>%J5j&onp$FLYR$aY)W-}suL_DF0AL8kCQEj-8i4$Om^x+SmNFCo~5eM#QH
zle0Ddkb-z!_DZuIioEox=CJaAlhU~2q$-8E&{AM<8Dxl*B3d2AM;#Lom%*&zuG6Sr
z@-YG_&pjMbLJ>Vs9r$gbK5v3Yp;8FnXuGSz@Z>5UMCiKoMI7pBI9&p(Jgu~dwi=Sx
zkv#I<t$q(Q(f1sHJTl_<=M-qNHG*?XAQ|FD|LqjwP0u_$b`t&Z*b^Y>%ps+C-<C&e
z5pkJaCsBLMV1qp-FpZ&C8OQ>1WyfZ`gsw#-BFRtF3oC8(XQ<vHmQh!j0`*2rG(t8e
z+W<VdqCza=<B=QarvqPr$vsh$&yIe&oe@MxHi{_){mWrUAX<9L@Dh4MU!ZUB6=^NB
z6f}UV#fudX36l(fRdSMY;r&nn_UKO;au1yXL_lttyt`ByP2S8K@J8v~a#`r7Q(pin
zl@&VP;n5edVwE|<X!2r|6u_-9z0<_wtuIp1q*lI-M<E#!$nxWQE%{IzuZaV3385_L
zK&Ls;pgRWfr@Kf|WV2ucmo2xu+Zeqaor9p^Zw=c};Sm90Er`%vgWL~zEV~FAPII9e
zhx+t>!)V}+fxm|a$_GhCT!W3%DAMx&EK!1)!K#7ky=$fY={b?VT?8HFk3W6u5n8C<
zfgNhnwZN;N4@!eNI=;-#M^QsZ0K~;0L!%45$~h*GEA4>^6BH)`WsxXDHF@$W`ZIbI
zV1Je5Snw?94Io?e?~`xPd!$ML`l<+pK0`l!9pMgz^&!-HTqX07;C5?uavff|688=7
z>4^hCUuC;W4itAVD^SQAi|OS<3)X%WdT{at?t<tA8AgFnpY)K8M(vS4RT5<A&p&j<
zDNzQ?;4xeu0@vik>;yE+bPfP_e6L!`p+BC8q*zt<`G?V~WJb=*4#d~tg<#ZRfKTh~
z7_~-;dZ08&dM>-QAnFLF_!%&~Rt1}P1ofRk1@L7Wl=;CZ`E@Ne1R1(CmoA9@_A6G<
z@b?NnKcjKvKc6AK>{UP^nivh|f#b656m}>F2SxPc<+*S;VKuXwSshC;?JwNykL2=S
zZnsM*qzRG;Lj4v*GYW*uUm$%aR+Um^tUNgXroU-J;SZ7xo(E#f?P@nhxqfa+Y;8;H
zDPid!SOA*Ytp@80PZyw9iG7HsMG7<TDgDU~MIriBQmjWxZq~P&heaIDv+fB2X8x5&
z?5F6$u7|B5=K$u1%nVS3f_OO=IC9=D6wj}mgcGc6h8oVZu<GVrfekV5F0k|=^1^)#
z+a;K<nl|LGeo(!Ks<9=KF(O4O>KGSFm}-)WcL*leEqPhEsD@aTz4SbF`r8AGYR5v;
z)zHI#NfbP$Z~{~254?~-t@IV+^cN0OEm6DGiKpsamTt`Y36R&v_CBXZ{b&#=V;1-q
ztm5&#iiB!D_kZEbzo!lrbP^r|qhV;8a#I59^=?`oKz)gLKNY=B!_YQbbVCAt!vVy)
zAN0G2Vqya^JWaTAc;<p13F-x}5KkF2z@Utx-2Ol&0ypv7ai_X=qsTFbxXKi!DI7{1
zW1%o6FR2|m<*A2SX&^a#%0i*R#23%59ONFhc>0YKr8JP?fYx}Dd@@2mz5Dw@2YWpx
znbDtOfrW+zzeW8L4ux!Cq3+q~EU0y2Uco|J8ejNup|{EUeWBz<0kbGGEqM+MGJhof
zEjRQ+`PhM?q=%B#QT9{E1Z>@4bZi$Z>PvcX&d(BIN~(Q4237vlQ9?57JC%9p?-QT8
zR2)%1==U%LFXlta9r%ZZ+};od!t;Gg-C`))#vKXX=-BIVRP+o>ez${kB~f-r617o)
zYzHU4rKFlE3DqoD9HpbIKB^{#HA2o;{0qiVQX;A2j3HeK2~W312OBl8&aN#nxz5`6
zg?etP4?IK3PR+C8vK9?>$HCuwC?%Fln>-6*95$yzc(reLe&zJ6377d{Xuk8Y_!Ags
zX__`X`pY86zxE`RNbVASGPT(=py+0v>V2#4yKG=ZX?{wP*4x<yxXXg8sSk6QR83TS
zcPF+W>7D(;=<AI-I*S^fGQc=g2%EPha60$3{bwo;=4AO`&S1R@r}KJ6Q`uBX5zc3u
zgwTkGKF+%P?$pIW`DlDGGXi~z@*GE>zhR>xIp;6@t=}6M@fnJ_f|@SMNU0J#NOBK1
znO7ztNHmvJwAk)h72=A!Na0?C$N3;amXP~rgVK<je?ioKC($h2uYyEPQVb$^Ps{+R
zJh>RC(Xn&!3t)mjWBPWKRAPvSspgNM1kzP<6)b+c_M=HuW<daN!)-?~Zogp>h}dGl
zYZ3IL#qYtT%7wJ<K+}WU49M|>9IxpI!wkk!&`kHxmZu@{+VXURGm?1WpDk=7Fc#^-
z+e4-{i1E7qFy7%heMG$t-X1Xi2O&RV^f`iFUV|+xul$e?DV|(Q!5^8PjHA-RZP;?i
zvF|#PRnntB4b^(R`z!kNJJG~ZK{UX8#}T}v^%3M~y@<3QZ%Nf43fUTGDp1NRsk0Zt
zr@Ax8@-Lv+Y@h{99`tD4gIXr{PGF10)ek0UHT}1TL#d8H#cDxIBGem+W#BN;q@C%@
z3f|I@Hig+ep?LM)&y*mr69?`)p&th`QK7SYmlP$+?9!+H29F<*6hQAMVh1dfgOw5Q
zSiATDB8%J7QN0U5)~9DHB24iPQ`(VaW`oxfkG>6Ab6Z6gGrzS{1(7km`@Qcu@tF7o
z;ldUf`wIz$_-^adi%8>)3w0je*$f<q?Zwl*e{&qR4ZIY-0@0^NLZbC&83Qx1^^{Z^
zX(y4og;#Mtr3IN*pT}kVGe9R`^JlK@LU?Lrjzn-Lhi%c!lL3V>`8Q(giW}G^jMu(p
zfdt-rceQb#1#b>G%BTI_1a+8;6@s3bV7If@9w>3DDF7=G#HA~USMm8jp5J7_JB<Qd
z$F|6rFp>Jz%jtL@Vgfh3RmM1AnRm||t7JfHu|N3cY|Qgo)YA9UUp**H<g^>lRyc!M
z*lP4#vqGoP0Rg5RD#9sSjS<M$5jS!cYxGx%Wi)^bTV?DoB$U8u>(gNLDs)z>8;+ob
zPqL#9-{?&s{&B0w5)b1m&^9a+SZ>Zg%UDP9%mqd_G+F+_+d!<SeV!MO`Xk6myfgI&
zJbm_u0@t4m-U^7*zY$wFncdn%wH>JVXNw1dx6ELTUS<sz>=5n2baj;f(-}lUEl0Ba
zGiXg<2P%H~v%!N`T9f36quJ#TrYPI-+-*GN%0!M8@-Z6WXb}Jz<7s_x_66F26SE(K
z$G6CsAuLn#&o;F9{q`0A92~wlqRkk3S!e<M^&80Yl#<l=8^uzP_=e_D+Z+H8QyU1b
zqJ;&|6``4o-r}!~VRH{qIY{;ZU`r8uF~IAjiaH}uv(0w?4%5i6g4u&MMXOx_4s2+5
z#(kU+0Z+GMflE{!H>QT+);ViR5&T<@pH6YrxP3X0?SY34hO=PWCbY#a4X}8)Y^Q#n
zm;L(qzu$6Aa`6%;F7ouinl@$2#&W%rsd?eM7S6?jg}r5=+3jzp76lwnC!6^qS01ls
zi_GtWq~_6V=Fejxct6KM(EZeg@+NLgTcxRXs12ngF|qXg&*b>NA}a4<3PGZHr=d4J
z6cV{>PkXC2cdO;Md3sb#6tqQ`iJ7EXrH$|&dO;)4ITo(3(Z^hIv_W~(pB7y<`1f1;
z4~!x$JWI!h7UKW#+kg9C5y+w7inSL?JN~9D5D=b2`dVI9dHpX5_3uRLzso>#=fBkW
z|0Fev)q0leY)oQZa*m@T-Cw5mf2TRvo0#U6p3QY_0;kf;ZSH8p#zB^o7OFgG3HOZe
zNqOo2tb;_<5{`x&bBfvh=xNE6mt4<q(r2#tm(F1YcE`G8<<9s2XozmmfH<?3gGLBd
zjBu6YAULuMU2(p89bSVz@3=z}>(Y#>G)u}7EY!cczxU@`sGuNXO7HE7)^)>h*S!xH
zUk1_sh{M|1BR1LwKl$4H%Q8u>Vj3knlnu;(bi*2ExUSp|mX`1SLC0F8&9F1w2d~uI
zc%b3+Rgc;;KflGlqfz4S??OKd#}WSU8V-S-KWhVx2d{1>q7HhnSja7iW?S*1kwuc>
ztA^S^l|MQ`|4ZEO68vA{g7^Ke<RZ%UU$X%I=f7s*zh>e8KMoC>2Hlx}cv=QHMJgJA
zURMSP9$NRZyR6HigH75yMFKK>b|7G`y?_(TrO<q$XDu6UO?o+o<?Yk}bg#0qL)sb|
z_9IaPCj<bzUl)L9S+n3`eO`<u?T@TR(z_UD+bFNic)PF*wB7Oz<9J&nyKd&TM}QSU
zxBa-!`h7aal9k<~zT~INKfWv0#f#YBM<lTgs}B(h=YRagYm7fd!xDllb{eoqJnEe$
zEiODrvj1F8W2(`KASysQtk^keojumoxw53JWv6cjcj6CG&xd3JI0OX(m!|HbT_`vW
zw}i87J36|R5IBg!;HS6Yo_85}*z|7<C04$d0Wcu^`>p!Z$Sqm~C~=C(#I9PM5-<vt
zXAqo)R)F$Ynd~ba*o`2$av;zkmU5OABHd}H0arqJs_<W#kioexfWE@P2^VB_!<`mS
zOj^c>G~wb9B%Cq;09KJBoO#b8p0;R5t0FJyepXEI(yxGr90H!9LDnBBAYhh`hA(vf
zYOKAA1d8b6FQVOza3TG969UiH3$UV8j*H1j2;!|7K(*F3#ygQ%0;t1!-efu}awB`k
zRL903Y+Q8)&?fD^7>~p4@G`*j?bZeS5W1Y6Selfa-lx3)8^Q{s4l`fxmEAUS?a~p&
z&M~c}!*^Un);9@NRp$VDaq|AHhqbE!01&#=<bhGb^-`q7cKes)nL5t_j_?msJR-lm
zJ}RxfqTuu8+bZ1H_EazAByftO11uEbGT*TINL_%}Ju+FIx3~iTtL!-cmX2csAX8rw
zI03A#PD}Glp-g^1AmE)$F4|bp1uS>YtWNW4emUpTJ%WC|&ozKK1C0CU((^L*KIF&k
z`gLh5Fv4|Z2xAud^R74C5nQO}zwG!kyyrdxe<0}<=Kq&G>v-+Bvtu5)%X~r{fUmu4
ztl<v!+Yf1D0M`4VLSV*Kb>DD4%!d08n{a_Bl<OsL>*_-;$FB<cdkWVPOl0b9pfCY@
zon+qi0j{xz=t7)4hl#mw@C~|UA;~X*du%?F9#EK=QBeqJ2j&;b>vnD~ot4y1dA#&p
zKcJ_&1<;@B822MLp3|mqtCWI;BeG+m_XsaVZM=-YHSXZM`8#qKeRe1{U<F`dRUE&S
z(sA+%mD-ObDKiG9Xh(FV4$)-x0P3P<0nnQhuuzgNiT09}A($OI+!M75^K4u4yA_k;
zL#;mm7}T^b;1j8)*0bZ{I$l4guXNlg(5mC+Q{_at2i0LaO=vY1Ze$ZVkqX#%UQWN~
ze6=Y078XR$xV}vah8!yj%~bX}binESACyd0Kz{FAr#+fwSDpt$ZYl02y=%T4Q2pKt
zx#XT!$sBwtzFsFXIF;`Q=`Zf^zrLwa$5#&6p!+VAU+7#L6f~XBsFYa=v7fFrvChy1
zeusQC#diGB!!SXiBIgvjnq09i*rQVRhDs=6>OHwUyVl&R7RvzX<rUmgp=lc2qYrqQ
zc0;RZeZuq*N!6&Vx6`;@wPaFveaAO(?GL2IlX4lEXSJ)Q?dl!0h8DjoI|8a)P0&C>
zIJSP&aW1%rwt5%e=7RR-JHz{5CQNMd8L+RZb=E$IZ&-e8T1{sF?8>8UmHCG}c9YUH
z1ibuhFHDiFdon9x`VhC*{m^%=>KI~4Rn|dKl-B@Bnbo_pxCsDOKVI-v@>&I`WGzSJ
zn*LmX@E3xewA_uQUr(iq10D}00&i@0Z!$r!IJuL%+{GC{p$}<)`~6T=(>9Ct#%mk1
zT$|=4I!|P-`dG^-jYwDBdGE5I?D9@0+`N=@o$reK*~8;8;m-Cm?^2wSP6K$A$S%9q
zu$3jfqjP~5og_r4=O6q?RH32sZKFhjz(;@&zper3N1+av*DVk@n4>EV?*7{)glR6?
z`q*zGfCWt6ALjZJ193O5IP__63}{z%ZH$C(#L=G4XynQI;a!Pkv~&VjZZn4^gq7|K
ze#;{Im81-ijs(Uh2b}FT*4pZwvmCQtkxaedebNJnGuooFKHjvpILu5a*yS>SAyY)4
zR0q<QS*-f<yCzHq7_CY6`u(hiMxW%CS+_`|^;&txLq6?1$QOPBN|baIh~*#3jMFv<
zDcyc_+O@sU62a36@V#s)m-oSg9g~j0t)^Rjt>^*JU6I|F`*WUE0FW$Vw6O*x&(!Ef
z0#?n|YqjfC%KDG>T@SdD4S1*?ZhM6d<GK!~IrY2Z4iyex*!>_j{Z55C0O|G0+2nmb
zEa>MocW$rHEzf!cjDZqhl$H;NXV19R)CFU!6%mBMgn*>Y<&WOZnb$wo)zSj4&;>%?
z1WN$kq$KH0_yk}=+?Gr@;pnBIaIk#6mGaKB53ujsSoZi}$OlWTZ1NB=1I(+!m8O)*
zjU30&@`}Pqs}z3mrINGQo7-1(onR)K&EC3Qg1&0TF&1QhK*~tcVG)p!6VA)k+Wu}_
z0el(CL_P)#SWF_5p%K&d!%WC9KD%`dK^J8N%3QZa(?+xM9%+Hns9mMrCji`T(FG)r
z!#OSj2p|my%i@PIO!FQ-Nr6>PLH#$Lq78N96;eje^&{Rx&A3O%>S*fi@M_Nd#I<pO
z>|EJgdFH2qWvA}BpB=ST9Z2oD#oooKx??qLso?g~C9R)gvVWMERDDxsTaQThj2DB|
zw=*LMR#%%6O{RR2aS9+4dXUCyu}u^%bq~C!a9fnlpBR4Y&A9<n1M)j%C41;Oeqy7X
zL_}gv*3|yG&&F@pKE*30RFU*O5@RO6!Wd|Jwdywtz<1m{oPUcS2e@tP7noaV{IsSY
z9GXAk%{NT-^DUTi<TV*R6qlUJTg9&8dX{HZ%d;cBPLHa%Kl8DJ_@JoO=#XRBLrH+)
z$Y=kS*b|J1QW-!geIyY**w7+hdHWBQ&m(->d}(FCTEk^7KtzeB{_Cu$Nu1?(coJX;
zuF=G^RU8m)wq>+mu(7YR2)mBBiir)wk}+R7;W*OwqZ{1&<Q(ch7j}&q5EsmNnzA2=
zRF2Z~5;7>EpF<`?4Y)qK<3K19Ad-KP!Jr2QEZ&G=cbNnb=dzylt{w`(+)7zW$L{s$
zo_J^KNGBM$uvavBm3D5$zSR|Hw_x_Y2HqfwwJ99aEK^k0RO}p8%4XwLpKQOIB4*C~
z`Eunh%b-vjUs-_@3NfJ<M>RDk0ie~!41ko1curQV0}C8{M483LZ|*y}(aW`=do3D4
zEDSAIARJAP?@$AKKfKY`{XED!WA^4d;V-oi6_^i{a%8o>v4}&kfb0?t^y?Upt6sQI
z$#Q}!>)hA5@J&Y_^JE1jHmT((vJC*PzFVR*{MJi2et0p_YvyO=%(a+%p+2*8wq_Z?
zi;PIDJ=8njgvbi}=0u74C`!ds$fiG7KB+bHbce4cdsN8WW^K=!Y_9rf=CWy;-^=%W
zW|g`pjNJiU(xjyS5#b8u=UJg2E@)uKqd1qr+#!IGOuaRT1Mg|QNNxSaeD82ro9I-=
z8?Dj6kqm&8bG{k^=;zBhGDslWVv7rV8IdqF6-O>Hd}bdT%exPemLsJp%q1W0xCE`W
zX0%=T{NrYJan)|p>zm`XO|kV<pD(#UjA|3+V?e)x;IiEVQ0@UjYSqh8S3G17Uo)Os
znnU7pFVWc#=AXqT&Ab`(*eIogitUh*2KOw6=46g)$UgSpw`1PQu;7od({n%7JMnat
z=!ykFms~vxq_jzC=O1v}$gS63!R3lm+#Vx9(yAd{jARirDsd{DkK*Dux|CDF@hx$+
z76}-{ZO1pp5s;jj#M@#Ct8aItWb|G!xKjP%W3B;_YK(axeR0i7YHP%V<%;r2lm6fA
zbY3Z{&qnY)44<f!7aq%34nT16h?1*bD1OVky-@4gqiNk1_2JMO;QNgCi)lXtn7X>q
zr9!TBmr0iGl+FM)@Aq;Xf!zC`&`Kg+&1R_@svmfx#W2u9d6m(XYx`h=pL<P^Px6(*
z1pcFNyZ*vqGOO!KH!6##NE=cVL=gm>J^WOnAG1>IYVXFMs}ARKXsdo&v@upR>F7@}
zsH+{7_`r8(PY|{6zNWcC>E%d`bYpO<oJCxvbq2ElL;hOS1(z?R-K{)iKkY_WFkIzl
z^ehyBEETh=&I~Mj>F=*X22h$u1)qv-hN%|4;7g6u07=(sv36+`z@)icw9Q>q&BaQ*
zsrUO*xSMx%zdY`n0+5aQqL-YVAvBu|;o9@=6lsDb<O_u%wNWjcXAZjgE7m`Ike0Yn
zJv$Y2gn3h+TNXDrmY5-D6|_UbL2S0LYD!A!eodi4S%jaeQkT@xS&`6H+Jp1CUkJMq
zXC;}i1BuM*BB}sLuJH1<Pq8C3J>M?gwko(WPJUi6$*(9)RcI10Z@DrWB473)fN=~7
z0xaHfZglsH2?l!lwG!4eHvrHVCWbt3WJ=etTN<@3T$LYnQmWg4`3T*rOx$+1?~fl%
zi%1S%QLHLDEaS7HxG3eM>;H-S$gVh>7&+}I1c9#X$dqlxsIh7oz=rMgNGu5Rqc{{k
z>lb60<?yRPD&hcD^hVIo^QO7M#v(oyVK?6dzo8Ih!d=yn^YS<V%9$<G;@WYI^_lAN
zL6l>pw3Wra$gqlmg#_XIOZ*_<cWr;P3I--4+JDnL(2QPnK<LCe26BWi-0m+|Z*g0$
zO#2>P{&0yhC^km~;JUO&Wg<d`27|bhUSYWOwx^q0DKKYT?h+(#P&`pThy(^Ugd3xz
zt;ux*Kbr>OQt6{47tSGSJ?5x7LN+OI@1a$d)p-?~mMxkY@}qC5gN5%^y`di2l&`XA
z9guSjF$>vTN!e5jOIeAjvuGPt`4_kXc7~WeM*I`k&c2mk39q!LTy~YxRWGr(jcEG-
zMGIi;Osgo^^LDeHktbO@?5FTihFkvS0lxtH*WG?+SB6|RhI#=wda|q0E}XiSa_qD`
z0fM3#>Xgy0PDH}EEnbAFm$yVuQEl`nFQ3|jrg*<LmTrib50T7|IkBqmAx&OPa%K*k
zp4!})s@99Iw|8Mqod$R(wVBvUV)1kI**4(=hZn+38&=P!ED~u&vy-bd(THZxl+y%w
zJ{iE=C_CNUbz+aZyJ-KhE;^1CVd)1*P_Dgyf1qm?;5~f~)tl|As;N`Fa^L8%Sd-+7
z(YeWRp-V@5S@s)Wz4Ba@@T<Z_k0=e1eRB@i3Ag1d=hZ~avYUpzB+0*xsWHoScXann
zztW2gd%?EUIFC5mHYI2&MuxnA5grBR7v9d+9@xA!_&a9jVmni@K!WjNG#pCf#sfc0
zeTeX*aDEZv%$GZF&v><uZsRN6Q<z+viNL$JEM<~7Kb}G$32_}x-;W0}jz4!z#1t!%
zb3(Q%zMVyJxNxq<Rz2x*7MOp3-|OrGH_uO{`PzpovMLk_FLhc}OW%XZMFZz|Ry(;0
zd#y1ya3>sP@eaB{PoNoe|Lv6^R(jv<jPw!<@uaqai!vo~4!5oIFjUM@1OdX1HdW{M
zZqCqcUX!DGUY{Yh5PV1S!V=EdQB21DY&$0%0HRfORF6K7qgjfU349@Jq(Gz|vyt5z
zN!pZVR=*Ur>Y!kp;2d2S(vvS!%zLkjC+=>vT>g!tkHliuf5yG#$B`fhQhZk4SfLa3
zD+szgKWBH>v3clxbdDs^$lG~gNa!XrBMp-xpD?F&jqdsZkBRX5CeNKCtv?z@W4Vyv
zIW7aBgNe@EW|wA;tnI&DDpwp*X?#6Kw(5O-+{s0+mzTA_<3rvGkyTJ?t5)%doS(bx
zct)Y9X?50)IXRiRw^rpL5>3<XlNk-)Zg=m;X>%hbRn|ss$n9s;+974MFBH7THQyf#
zcL5wWR=~ec>DNaJA7-9qN>zJ2)~@d$jqce*rbMQfU$ksd%-DbKWz4nhE4+1#9>(lS
z`|7{Bj{BzVEcGJ09GY{uh_}l2w+pR~e;hffv80BqT%M}+8Z&jS%kNBkN-d(op~Zbi
zEOrT(*CU@V&I{ocn=o~Mdob^}1;{uSFJ}xah8^FYp7BbJ-%WLDadL^ov0Cy|_qK#?
zR~+VT;Zot$gm|W7Xi0n_)`n!|$`>WmDmFTj<aoBKwhiyiZ{B54(&D$2>;dHW7QyMb
zCAFqi0O8KDh5SRcaAJ+t{o2s2a|Iu^58g=SqtCm-BX4JuGVE_Goz$puL}K{1LF8(-
z%W8HVXHe9nMfhsx;e9v}k$F3xojrh;J^z41^=tC0kz7Gm;}S3Fg33{seGaXq&8;Ur
zA0ZKIXq$9%>+2ym(g*cZ(l*=Q3c7N5h9v&3vV+YsoMGv?9Jb~|0ENXPSrW@?9_npk
zUamODefDmPyt}O*%;za5Yp}UAO<iAdE<CE*E0%EA+T~FGQi|4)h;L8!x64Sx%H2Uo
z(l3A2LAZ8KkU!z_l0#ry!t)|UA<hIGv!u97K!W|*Dcs3*ajTR>&LxP=5(+Gjsze`s
zMd7B*8)f3FlX!>0RYSzeofIc~lzf<Oid62hrjUz6#?uQ5O6rrzZ*M>J#4O7mzs#9H
z#5~QnCpj{4+kyJT56V%ePG`vTw_PGKzBT)BQgt%ksbkS1aPLEznimvYzV^yPW4NB0
zgPKXqp%n7*in*Vgt0Dpt*+Ui6<N>GOya-dv9A$!lXCv{w3w2I_fFdCWxlGWkRMk`u
z5>|~v?r?!n)#3VRsol4p{Igf;LM3Yx&9!8$KRP7-QFK9y7AILiDfoPIqml!KzaXg{
zE@(`vjB8=3k<VQ=DE0WlYNX{24gum@X>$YLx`(`xF;=(Je?LRuG+OQvxANnz0yj1G
z=4iNEa)|h&XF`>X4kPMvUo?t_d_6*sm)KGVKR*@Q-ClS_d+9d{1i$<#lJ4nu*)}lV
zu`>xCT9E3}({7ZIKJXJjQb|Uqb>()~?PpzX1AHd0Us+}^5nXqwJ>3z>?geu%+~+6L
zhgUH(1IxEpmNL#-cKPuz*)<2_)}Bd;eR17hRe91J=S<*4y?X^oxC=BWMJsJXpK*K`
zYRVb!A33wn{_6Jv$*O*D_Omk+T5HGp4)P{P^yu<ZtH092LC#@W&fv$AHxted@U%R$
z*Z(NI%+X)X7XNgYZ?tEvyU(GA%O+8G^r&j&$rEYP)cb_*-7$B%1yN?_eKsqk+_~Dl
z3MjBc=G+EsGI6n1`}`*yiZ`7Cu{b;1Ad0EH>ue!SckBxj{Pc_N45W?*f+mj6b%*CF
zEj@K;3wv2~Ag(PK2a|77(iAQb`59QfxT!Q;?K+;A*&NJk$Ul^`*nYSA%B`uzZu%HA
z>IJRaaZAg+2$=EF9Cf0P(LUUx+n#j$GCxksCq&{M>sZyny!y{H#Qw%4_6bD(`v9mn
z;tRWC*p#vVp{eXoT1rmip>0BE9xEP?tCHro6-w}ez(!*DlxY0Ka`J$Pa+LFOa@N8|
zjoAA*?YC?t$wr<1D<kKJ*F&#ddXHP3R9}PWiF0=Kg`c8L{G)4DNr~10er@vc^u=2F
zzzCzu%&zHz1Dhn&^Wkz&y6@`uAmylmME52$H_=br(n3+bQQAVt<mWOdEEkO4_zu)q
zOuD<@^oo*Kb<b~@aHi8KBE(5c&7Vg?*XOdi7W^B{keYmVSM-Ls%g)cSRk!{h_TDn8
z%C2n}7F+@gP!L2wK|)HpL8MV6r52sif`~|Wh+=?9rvf5KcekW;F1nGB?pV|}FK^+#
z-}l@5dG<TL=g&9B{^xKk*1E1a&w0)>j`KKfNFT(M2QxJ#t%i!FYaW!i&W=FhB@OD;
zC?u!Cfp+YQ^A8um^d1(sljRdpeHax>6=DkNGYSlQn!h~rTnujEs-yX7ZmffPg<UQr
zG?y2=_Fy-a?z*%P<Dr~u*!M>mmQ^Il5>%H*`}RUO&5d&>;vlZwPpT|j91nKzh3s4H
zn2l#=aa*!XLs#2h6wwcISv7mue_xIh?Uq{)Mwx|`VKyM*z%s5qH%;bEkjVBwJBZ$4
zL{P8@ms6&Y8(X2mbC#OGb?9XO<f6BuQ@qw8L}99EOXu?=jb1Z>jj+s^`mG)BZ0Rq+
zHq5ihwQSM9sAHih8dqkc@Yv~$Ss*e83K+#Se`tgV?#cbmv`0Y|?`FZ_jgiged#AK4
zMYVcGvv-|+_JTQ6tugM=tL`*G!YWRn>Y?k=f2e>VpnkDni)<ST{mmj4ojOhL(VWlg
z(lLTtXr8^3gzL^T)N;u0a#_gSZh6A0fFWv-tHWs!A%8vS0SjPCeHdmWyLH4(cx(RP
zNu!i;#gg)!8BNYytwt>44L^VH4vRkNV{{Zj<Q~}iZ=)WQ-k8@nQs^=nM3zzXoxX&2
zPSbDc9o);-XF+Y_I<s7e4HfBo#ef5ilD$*c0io}Wr-yxi%?_|rpx5<1bfJ|zj+t<E
zTRWWnpei@%aeYflrt*$^O=_!-^K96I##hPDLQ&GyeLqn@et>#>r0Hxq48{`1xxTNr
zSEe3>EX1&{pda3Zl-K6%NVj8X6A39bk`O(4U+o;dly5nltW2C(70Qyv+W{G8hjDc_
zPIjJ)3J@dw3omo@BIQVA2Hp_c%dm9PzO<Q5t^=8qw1Q6IL%UlktVCTAM6#TT_Zdf2
zW#SQxxy`a+oCX46v#Z!9KBVPl8aE@9?AsrE(tTXN#O5T{%@)r&{{x#2K7E7N$u@V%
zv@il{I^gime|Kb@Nc@`76Y_m_?3aDEtFD=sl(nK(p}yK>X;OsP+hzrfBQv`B)zHIs
zgO7}}o)<{h$H<Q&CcTb6AYKObzG-oGx%jwu;Rj{#S247wP{*W{(*Wa(@vA7P95vhG
zbU<MlM(}gDk*_Kjv{J~^0$Wa{P3S^bkMr`3t#7YF<@~!&WdiI>8dZ>P(zt`i?(h!A
zHf;&(>#;khct2WR!}T(i(a|=`M?Y<zzInFPAV>g<nmDuwi^HW<pfuk*C3f+2EX$tx
z-GFHOTmzk>j)}6POu0#oJ;o9w`Mk9&`WBbAW{(Hpo{bk)TNwzAt7K|2;SK7$t43`?
zJI8HD;2#&Z-_w<<yvT-pN0d6D{@tKi|JJ^mXJW6Zl>0rrXZbG-(O18PDFZ7aQ=NQ#
zOLE?|$6bGjK7;^~UX%1J#;=rAuDhQb#Hn;nQtrH|<tyE;ovHKgAR%TJS@rm#d-+D{
zuENWGG(0EKyJ(Ua3Xa)Tx+@_Qy>eK=Vxvz~97Nu9ekG2wXS;KdMi>7Bm|?G}th><4
zx4y^e72i%!{$_%=%cXyp#h|NJ1<D8SGFbk4-kg6dcN7+iG8C|XO)9GVsW>ez`BwGR
z>zfY4Ovv3MxpKx%?ezJYu@O||ts>j>G6(xB_A`I;+|?MvK+B*lzZJ+w%$hVp!EoMM
znM;w-(`V&o*A#Sf%HygI$(qgRIAs_0#AuzR$oY4A8t5WgfA*?YyQ5zrbkd;R>HNp{
zwSb)M<Zc^R<7)G>^27GSG2KQzHNsz47U$Q5c?tFDHCXS=<Z-E$E?Q^rwsH;myOVPd
zH8{_IQa2WSrc6{3u|Zphv8_5FbBi;`u5s`)s1bd#VE`}EW;;smIGU@|y#^#Nwbv{7
z(0o+MPoOY9kJCY!uJ%XdD;R?*vI^4Y;JQLH!o@0Nq(Sq$0myC*o#it5#iRB$;i`!x
z=q!u8Xmp}8VSc=bS$XGu7E~Hyf_kr)V#+R}(k{!V2KtkAMckC?Lim)FsjH+vdzu@k
znZV@DhqSnY^f*S(DN{$J@3wY&hzUpWuv`t3zVj2@y13jO%Q&i&h|-yW)31sI?ERlO
zCku-6!oz6FUJ~18U9;;^Msu(9R=#VR>$+4-pogB4CqBg=5bXE37h;yj@U)-s+{?9B
z>sk`7I<yyQh54B9epFJfuBm|ys9tn)ecAJ3b(OMx+>-Xk7O3Ist=rtVfV_ABOEBqj
z4eNQszxm^KdNGcN>66a;80x96w51*sE6Wi^n<8Q3p@QlGc~4zID@G@4ytiC`ZI2Nx
zsnYvZ<wS28h;By5$U~#IXyS7wCL@pVwjZ#$bo(sRDGQw1$}i664(_pVQ8;u;MqU^n
z4f#Frw{`m?W{fI1dw%zBy2w}RxUi6rO+H4v_2|z%32|a*a9&xNPo{wHJ;teMi?R&6
zQMQhbi9+rBfhG_8&d{m>s;`0Ty%x`R2|C{&^;l5vOcT$%Mdl3@%iY6s;(AU^NWPui
z)&jaUrzJ=Hmy7yJhofKrQcN+OY)$jkv@*`^6M9bL9fO437<(y6KpC}Vk0&gRZ{nyc
z5W@deZ=Bq2gq%6D%z!81q2ukc30z-w9h(PtZSD(@a%u>y6fAll8}3cno_B|v;ZQ}f
z(uGdEHNU@ag;P_O$BPEd+1Wv)<=vuYDI~?|u||uk>ZBq3HSxokLN@8~z2d%rh~R=V
zTp_!jx4=#(Bc`*dru>CH(4Z<AG>Onnlb1NnQ1k0Rdq{5I=QIChjJ}z2T23n_R65S}
z@JIRHipQrJP9<v|y7(K+x+{q%n$r7cuxAZY@7X>PKj+3s4i==b*)NsOlJExQ#&2pf
z+!H|Hi*3PEe>u{>sg0!fv2tUtaqL6u7Op{^duVQ?F)9Jow|L^&W<<-IERDaskKF@`
zp9?yiA%>u{DMOkbjIzlJ3q4Nfn1a(~2zNRm$I*_Wj7zkRXhXCh+IHs7DT=?2jRJIR
z7z4{Zx6)^b4@*IMczSf2>$saxq^e4x7nXbwl@d0{!&ZDP1jb~|XmRI@VYC{fvgYfk
zchPU(Al?MgRq3>Hb5!@%TFU&S@c3$wY*MWJJ&^NgA<f)^LEJ8=1vRr1Wo-oN3HiHv
zmTR{CcSp+O(8{Vnb2!s%&2~@2_QM3p>6XZoN6w0;xp5leb3R*!SDcxtSLm84NM*a4
zBq(F4UD3RALBalBxxf%qWkgiozA*uKpX}!_cP|sF+r)0A=3I5?k)}ryiA2NPjSXv@
znJ?3Sw6Ns3&|-t3klKG$?Og&s_C^Mkq6I-h5DT?GZmruv9R@sjnjf9lhq(5+(^Z3Z
z`ox8DE9WM1$39K2*M;qu-a+<6{`}FdKfc-Jdm<mE{OjoXFsh&)kem9qolTQDa6Gf}
zkvwpe(^ajar<z6+p)<)o3VPmw%;cD4&qXa1ctatH8eu--L*%{Fa%GRWL3hxWPhL|A
z@yg85irDj8jB>?EWPAkc2AW|AYOP$u2bJk;ViXf2%7PpV)@;u(yscTWO$nwUT|2)R
zr*&r8aOStYOD2Ts_sQi27ejP@>%V&!zPsRYM?3@48x<nH=!sQSyRR#D;lqaZn@)~j
zwQ(Q^OMsLt7eSPhj<3{UyHomek4S`KW{;TSE~62?v8?z-bZjiI2kXV41D#alc74Bk
ziUQKbGP)%3wPWs=C+_m}MQWM4ls++PA3^>|$BUdr8y+t=lpkK2uXv{~u#SP!kAPpg
z|AmBk69%6(ntORzAu&Z^<lH5tGKr(JmC-H#a^888hiIR=!~~h9ii8v-vn0fc7~>)&
zXU!NTH^=T{{V+@ahbRv76>ruPGiNiJiwBS#!Rn)#mWqLZ>A-nS>R-=+>TEWN{)38#
zc~|M4II&?FxkVUzSoPb#kew(A_}X+XGgxy(K+<&j^NQ3$BR{RAa&eY4RS$gVRO;vu
z62b)HcDOQ)5rLk;{sL)VjCI?!Xs5W2ABu9#2VwEY4RoExzG^2w*9(eQN1s8(jUpmA
z8bS3fzYNp>d%5r>q|w5}!ZGt9*^jl*66UoykEkvBDHz;UAK)9f9ip3Fa6Xfbm3bx?
zM#g>N0zJL-eK7=-cPTIPZd`k_#GvLAK)rCggJ`0t03_#`Unp_<L)RmY-)!Q~Ki)^~
zH*4sv_#HAfg$-nAst~VGhP#E9%-Nxx{!!4=z`?THcsR|`ble`eg~>8ckS?X<53#PT
z-ViYWrj`}Xs;dR6+Llk+@bsran)%S;B`;RoZ9_d<v5Bt-w~cU67B{l9FL;Z}<V7ky
zrov#|;ypk?IH(l>c6mZ97H#|%OQMD+g8<|EpmSLM?Lfgi0U&qY`zt9PZyrafn{s9l
z>W5I6*6aca!;Iq8H$eY4qHp9)F?CJoCo3Oen6umd<5GO)U3<+A`b5?NR}ikJ0X<3^
zH5H#i_;N>sRcLbZJ91AN*xu`~48zlSTJQd(L|jW0X#*B}Q$ptofWM?&oRk%ta0MVD
z)B^?)dVRK6Q9#D*X&SFI#xJ4W!Q0ka-Mkc1op=OYa;00N9-#TCj=2AXAZKIj=B{D*
zUY)=ffwViEISLrzKw;*2<6a=IJ2<mZ67j@EK$T>G+G|jqwY9~?Sa`~W&~jRACu#p7
z&)WMJ>&}hWKipYpg_p7d!7Bp+tjR!S)em9_54Im^4_;r!xHS}*{oPmKqz&|SMY<K&
zldeWg=})PDJbrqxIg8`biRcT+`w7YH;8NgZ?cczn-}<gdpmg5{=ja&PlJ9Hxswpq#
zW8J=e^Jdi&JunIj&_qm)nHt&T^m{`fF)A5D;+A!@>bqK*97DbA3+-1`kX+m}`4vlo
zZ4Lv9CRw>4O|wXI_SM_O+fyTR7Y{(IL*-TV4;=dl8OB-nl7FbWcd`9GO_=@><3Rr7
z*O*_uYmo0thyBR?V1F|Q4mUNrFH(X&8h8`A+yqWKZW|3dtB}6Pm+5`>C4o{RG$g9A
z&fG-~{p>WfQFz-NDP(oH3By;|=T^&6F~5A5J)Og<AA6X5uJ;k8&U=Z+H7zTZLj3(<
zn=L+{5;n$;h@aAS<him}r04JBBn^&}&U1A&GuF7*;-m^9$9BVGs$CDM2o1~MN>Dwc
ztAF0j*BtXU^UXBt?2)ihF|n?ll7$}jm4{ci!F8*I1S9OE5t$M7$@tj;SL4b-U%ju%
z@Re`%fWD{uaN8w!e~wc~llzb~?if^l^{(oFY|6V&ai+uAD--`-1U;toXs$RnfaJT4
zA<^554aASzo`Tk)?2SZ*p6dg&($}X?NYq>1-|2MmJvsSNfrUaBXTGXB=6QN7jd*F0
z%8kx*)%$Viz2qCCCE@DZOqbkC2eVU8LFe|H!P*fCQnxlJv8B!^P)LoUYB4R{)cBJ*
z;kJ#n^Ci)GpEno}3lEI@9IVi9O*ZDgqH-}Zh#}~QMK)2WkkNmPu}%3*!u!D~h;xj?
z>5&M+C&6Nxp&yaSaylv_MnPQ*iN;E>dAIPh@qKXe7je*g=d@a?f!MYr$aytrt$2!i
za$9F+&Cd&^V}5rb7U-NC3n$bXYY=CR+Gd4zJn*8$^I1z=cGkHaE{{Yb+P*~oTlqBk
z{RLHG2tuQNB>4=;ilt)TY{uHptZ_bta5+|`v*Cm|uME^MNGX7RkzF7b9-i?2WatB9
zj%U$m+P=>egb$3%1jTZ3U9NB*&Fxn<&TA~oyrH>UQyiGq2w}Bk&~t#$Mqi+!nmElx
za$&cFJoUJB46R?B3HTmD=uKOlBA%1I8&^an&HdBf<9Zog<rYU0j%!9U=S^`?u`?Xn
zy*YUyM&VA?*d9`P{}fjB6vzf6uiRVy2=kDarudTi4>>>YS5#dMr%%fX2^@i7w!M9=
zs857+KQDu~mbRySQ&a?{4T+?<a4`Cn{HzddVBrk^v~Ur~!U>R5#g9$AHd7oUaZnuq
z^wX!WLeVz$u<fEz^tVqAS}OV2`cAVgVLL|`0#e?Hv+qgXwBlYOJMPq!8l^f{$#bD)
z|8Ppm*Jt9-L7eWs@m@azKwXOY-TQ+FktuU>9AU*3vn=_;{`oJ_iR+K2`Hbq3)9(pH
zJ3XdrMF8Fh(L`q3=D!L|Udi0oX{HCla4PM?5^HdBFC>itnagG~Q%}}~mksZTm=-6G
z`2+c}3`G~-1G|xyEU^6}))$F^u6q<zdNNmR6Kk1asC`P-83OyM>iO3_a2t+FK|L_A
zCK<Af!N8m0mxZa}D$b#E>=KeyAl0;!q_-;GkvB=^>hL45Q7H@0L6U!YCF|Ns!(G^(
za@1|y+^hOO1)U5c<Fiv;v{|qFdz3S`%6RH_G>`oB-#nIIf^q#+#d@*@uwVsRdB)OX
zu{HZCU)9!_)qePLSn!rBm%QO3*W`P*iutTI64$9zX^Ji_C)7t6z1Mk&#MDi_*@9qq
z3eS`O61PgwCP)Y0j2W@ODXE^9Ud!2VB+`_K7MP=b<Js3zou0>X{8PLd*g8x*WQCTQ
zc(}3Z#asRhj~_f!>i+gKA)f8q&$8b>2=}(v<gJNIaYR9QLk&5gH^yx;N8+$*cxz_w
z28RxA8MR<NvF<WjL&YZ5%Nm3NT}lx*h8?`30dxFmemi4cB(o{){rf@2Iqw<$5c3RN
zB}$(zid?0lm889o5jSvs30Sk?z0U`rwM?<Uuy0P&T;o(d46vm=E)!z6b$)^pXu8y`
ztE$&~U*(?$`M;{Cgw>g!IaeU4`2J!1`_rM>)gO^RoUCYCZoeJvNs=v(9Zr=O)~vfn
zJQG=$(fafH{L)Rl#08FXHCy&mSmTC>HPB2Ag?%V*CX5KMb#(rMHBBbv(r;{x5m^6&
zrVY})G37~A#@-+b5{kv}cU~<V(Bqm`*(BT%vAvNc3)FCb>qmQ^!2M@D8+GkTjoMyz
zzHJ?JU8|hg)|6$GZ`%Kr<9wn??=UE~@gB%aF%{x=lMMh#S<D{cV9k#P_xPqCP*;UJ
zbwS{pU+X|t=Ht<jn_N-g1s7JyFV_4s+IXDPk`Vf@kox;pyqR6~UzdUbKDeWP02mOa
z?!Dbk9yJaWX%Wp6wQJ-*-s8ySiS%f8Jxn6>(Xp*7{c`joM(NJ%^n13?-Tv5kScz2t
zwKxzaXfA5{_W6QoEW5$Q1kw-J)v!^K`ww$?+EPA-t0z((Ues@VAre;q!!2lpU5sL-
z1>612XV4Gbo7J_`ic^rZBBp7*q<OE_V1#BIy0w+xu9!!#HC>&y@o=`<1^5h;OwHgY
zepgErR7HHhJr4cZmVQ6nN0^h{HsG=3bC*z@Z{$SMB%z)_PCi`$2numMvby0F*Bw6?
z&$eoF4eh4tHRp{fRO!apthGgqRgtuzCf()F?z=;zs_Q5q8cp?@xFLsCr#VfWH!SDK
zl?xk>Y5S7cZ<kDi55uPz0~d0x<_fT!-|B?c3d1cx>n!pp4S!07;Hc&zUdA{OtbJ6U
zseG^wsnP>6zGJ4#ZNd1r6&A%lqBR}J&wbudkvhsHdCiYUML{&117Y@3Td2a4>+lbP
z_`NN@lhoEh5Wb(r*SaUZvX;v+u%~>$x}HeAM~QaFQ02b8hTc05Dj&(72l}TJAr}Hg
zPMl;&pc+N;vZsyzEFWluyla_`KEL8W)=cA@Z;3chl%Ad(_O@?yNT64WH4X|k^W}`5
zlJ$9n9@9|P?=?6nJdK%H;4FQhk81pfXmw5PRq)5H_o%`zJl2H?s~~-iAI}T}Widn(
z6(rAh(MTt9uWHf<r^n<?u~-UmqfmDzkSn}Zo(6n9PXi_Hjr4%ms5r6Or1J;TAVE(=
z#T`I44HX?Uno?FlyBi5`sZowf*@VS5kP_uesoJNkd+loL@^|*Vl~`_^a=UeAXnsSc
ztQX^>ZQDrDWF~jPW@@hL<2Pv#=-)(oZyukWntq2fAItH_$I|m&8zX)#iP=ub+k7Hr
zCPsm$td>9GIN=^b8E91J?={>5FI%t03CVxB&Nf}ej&aWh)|6`lb7!kQdVxLJp6v+4
zu+^c-@dqY>>^@ldGPC`33li%jQFr*)K&w70UGwL0y+9K@@(PIkdX)|8+&JG=VY0wY
zC-HdwJ7clElcx6bPVqeVC7e^JaWH8aAM*ITZ6&^{_lVl{p|&St?M+0OxvVHgoP@@m
zh1(>@AMUNDK;pLMoR$E2;gNsKf1Yx}q~-PLyvS*4ibv!K1U1a{S$lXjhes9TtHAi?
zZ0nY`^b#zsNknFO0i^3VioypM)H%tnq_;`**FsutRc7eF4$A8!rxN0`B^UnEb~T1U
za-#3h6oT(4eqzhci{;VYR&~2`6p}(v3=~t7VZ`q}j2bQrf>a}Qld3$m>!Bn+<L(B!
zZyXt%DCIfFBlP#vFN~}RDCG+0q+GfXv+E!Wk#D{a!ogCz%f5c2T&kA_c!3vwJ{zW<
zjO`?NPZcSh(`5`{s*j&t)7&>FX4{|%rkr{Z^1$>H@Bx+fyD@oVfcYpmPhnouriJ%_
z*OTpNMKORdfugbe7l17W@!9I0$G=hrXz+rAxC*Y1C0C5d)(PYG>ubtyi{12YCz;97
zWqcBHH~PLgB-*<3OTp8{V8YlOjE|)E+N^rJlA7w$74z#)IFUREJkNK%59qt*Jj2!7
z2=C$E>}2k|Ss<@-;caFv=$i8%5A=oqe8D&RTteV}-lQOvJu5cNU2=G`bnHmOwn({R
zUckA5M~eJ5@UU`!j$kYlB%=E5a*TFolV*8G_QM?k0eU*GuY)iBHD-Y`=Od@{0E}$K
z*-gTjpSa9V4Y5!yu2cz_!p<}wXZ9fgv&BpRA8Sn6pEN=v9S%}W!-^*ed>~!#Fs@@r
zbZ0#m&??x$vAx2GUd{~fKZW$ldg*L3AT+g>zLc2$s4VlyQBV-fou89$+{tMBQ#cfu
zTk_yA3e+3B$I2nO$U4i{s>%gs{J9Jr7vIY`z_9p8-IeTV{cGA8*n~dB2YO)XB^le3
z_|!)BnNfUTn@;Q*hPl(SG2#*6ju`X*gvHFiCM*Ro`8KA+iqm9!A4ByF4J8Wbt_{Y|
z9S_RneoJ-NZKZJF3NA*+BWs;mJOPY?M6|F8tb$vnuxGdD-d}h&Obv=4WUn75#RneO
zG2Y$8yr0xCy15I>C!D<X`W;_YU@lvD==#qCYJ<2ne(%mVaA5vn=nCixDN;Sg#<I;!
z%)7J;mT8mCB8zf*+@w*4`&ADy?hh-Dt*0b@kZFGiS^nFc!{4&)7toJ(i%fELnU%4o
z09rW<J3$!gOD2i-n6!qo^s^M|@nyS7nM?wbD-j_HnjDQbnKve35*T3{Q=5!_kOX;%
zb%|+v^max?juOjV>t5`s%YE^!zUXXa6%mgSGHgu2`4pZk+4XYN@TzQ$ih4X8FH-fc
zrbA-VEh&7lx#bemvUShXJ)jHQ7dGi9Z}D@!1WIrJ-1}X4-n^$FE#ovVYEm-V?zhC(
zGwsI)NqsQ3q#=5ee?-<;e07&+2!g?Ovc&?u9wdMwjrL`X5171o--v4D;Z#2tr?2sW
zynky9<s&Z*tf!F|vIVWXcx~YImkkh-RQoQV|MjUmw&cRJF;EGu{(9`3$GxjWi+}sG
zx6yVl5*T)H-lx9}o8q@&s{+F&kaz+5Xvz0WV(+DV;f6;@#7fFpk&Z;`^Yw4YlHxRw
zST;CX-H1v0fw_@!wU@M(FWBgQ?1Swnwzut6AIK8DFgI=tVhXQ&U%NSCb^#<D5F+oF
zOHMqa!xKcKFu*n~-A6tj2TiON(`SF8p?-@qzm66wn?Zf4!U^N)AQ{_Rc-OFjv$vP@
z?n6m}ZL}|7M?g8@n{p@2hL_av8%X(zz8Ff9h%?D>am7nNC<7><r;RprTX=VWel*u(
zlMTYd#6NvN^wH*$JAVnT(-lq(g_t~45gY{q{IJ;N3jKOjio?OCP2DHtS?iY*3v3ar
zu?mvir|~8#JH^S1zOjMNv!&zrI~=RH$Zvk@fp-XHY^~u8+)yhG@)X41YIqM@;qd)p
zT-ur`-PLsbWuFD}M{3Q(LNQM9`9?>+&nrMUI)>_mhU5|l!^B2U2?Z?Uu^XQFQ+1Ex
zQu_LId4~of^{<BkMip=H2bFJ#aUztUSq=n+1;I8I9i^0jzrh05@&|WjSqIpP{`2pb
z>_avW7*t#iW+T)=l`?fCr?z$e0kJNky1qdy&zw!SjN9r%t%R69zTICbiS;#fL_mD7
z4&v>I+g7{ECc4Kx`muEdNJ-o-2Dg^Fq!yog+L7lnDj7rQBv8hGzdMLK0mLgO_`+l-
z<dR_~Zcz;-6StJw7Rp+EYnieuIP37*F3No2Ix5At@&G-#Sf%6A&iEjpUzoJ(7buKc
zXk_+wR$BoTqr@))#dLogeGWgYY(H*F<b@y2ZmlC5OLlw{hYtt!0fkW$)%FZY*deu8
zQ)cma^mzWDf@%m<UULCGmNrwXihJy=Z<WwqO)e(ejG}x8)XU_abqjya3^XQ2H6GAU
z|CMS$XKP;sm{4adQSRlSpHcf^Aey8V87&cRJN*eGNO<5cuIU~)18@fbs*AI<A4hX~
zd4g9#>bs76TS#xw`(K_83C1_i&nUYmOmatLi^-AP=qoyRSbT2sqN}qqd>G$0w#iBa
zD2LmVwScHwQOS!2*G8}2k<6z4>YW;)4-K5zl=6aDM5r>7=CIpYv4)J0IIMp!3OD{+
z4lK8h#PETeT`)$(E*QDW{*d@!v>1qUhRTnId;LC#8Aw<e>UJGv!SS5!D!ZpKYwkM*
z!HxU&Am05pa=RluH}}P7;&!6w8{FI^!+8K1z?!L$C&C#VAIPY~aKqMH(WQ3a)UgH)
z^J|U9eI-~UwSxrN>k#!DA-|k={KA;j)j*kg%HI@brR<DL{~qrYjD^WjCs&}~BqhT)
zd=<d7Fe70}F!Z1r%w=jnh_`<Nii^V`?P!nnpYfMLbB8-wj+N<OTFAxnlAPYNY3hwV
zts<G->n?aicbW0qtdGF`82&QYIsr_qh+pqki)+W>ZS!d;g$R4aYRzT-3ELglG~zYM
z<%Agk84l%S-V$;Hb0=vK{MDov@hiynD@ylS5(CSJ4L|v?dui>HU0a*A2c;JRix58#
z7uBn8e4~BjN8#Zlq}Yk()J?1@Nb+%B_|8ZGJ0~B;uLw5%&kh091`-!)WFI`M2wmS3
z-xc;5IvA078i`qQ-<9slCN}-i#aHWWe8SczReJ6_NQ70;E^WDT*Iq3STU*-KA0nYS
zl)XGAEpcv}%?R(@YtjTY60fRql$;nAa4h~RXiWx)drctRNN}lyRHrv5=A=D7Ky_*9
z{Zk;|jx2sl(O`XGD0s1IDJeovD@Z9|i2@*Kq;)hVMwgiO3k#@HFMO{m@ID-Vrb~ZK
z>Z6b6n<?4grzW6Okx8;|nn(G;Ce|?&D5$<h8oOfz{4^9|Z9Jq~BSDPOa7`{E(|gP3
zpb-x{Hc+mR!(MoF^F-kv@zQq?J%|AGUN`C11^;HQ@%MobDY<7V4m7Oqpk{;^71)VM
zzc>%R@OkZJPLRNXp`c9ELEHRr5CTtg$6XzH8LmrTyd8jf^(<QiXKKMul=zgZnzNZK
zW!(Fd^ITMZaL=5Tk$dE+bPS{mD_MapogQUGpX@{8oq{E$J&n6N?rN>pJ7@_bzrB4s
zzN{S@*X0(zv9c}l()aJWQ4ag>^Hshm&?rTQ;h{$YZ(pZ+YexGl+Jia)Hj}0gF08~S
zT64(5H|2EkLd*&djlw9pC(r!bp_?pYMuYzm&mzF;zeB27;;gQ5d3}z#-Oo>;qVmxQ
z)T(4pgg2<LFjM*nXFBSOV;#qxGdqnh{&l!(h9%?EML3sBM?ALfI!21rZrB(RbvLo|
z$;nYk#@;rvX@e8qyOHZlH3v3j%ya=sH$*`SoO|62H|G%f2dYC!4!Z8<#*b#81+oQh
zc)3UO4DPx>=#@)`A`l#)XxvF&cwfqRHx27dEUS+&<26qHd+&mxoIvS_i}7_EaU$!U
z_r5>l+e8wg*~y@rVY?}t9BEgd%EMQGgbOYn7+;@yWe5mbV(l&N$iyJyO)w1wd_-TA
z1MrgyZMB>J2<qf;Elw|TdJPbXkL0XfV<WTjjCr#R31lN+LPl8JF^r0hT=iTL%b<d}
z;t62?$zBKi#jU}va};Yjv_jZ;m)vz3MkIe54&B=e7vCa&&<54q%#|Ult_-D}pY=mi
z^m!T+tj`RyWn7)&GsL>R!6Y+d@<FDC8BcJ%CbhYs-(K7sb_EeI_sTVr$0Tx<%iP!X
zd*@s=uTdWQ1aF>pap7Lcd0g?a=7?}%(Q__@zixFpk>c1K!oNC5h7@^OG+tvXne-u@
z=eA<!9Vx7J8fa#5pP}nn4f)F(0;hg_75wQ7O#jOvB?A8S<+=oYg$Hrlj!0E?Re(cX
zOI&KCrLG38j@%DHkE=ZUD5pV$r)m~!BRN)mVir(Kbneh|C}ENtBCR)p!ToS{4x7)5
z8>D0PVeEtwh-H~px8=fX+uf63tRnNxC9C`_qRhm&qREr0iwkcfl0+qGE`B()&?4v*
zybO<z`8?+SyMDty2EOtkMv%o}`+NoOZ)Xj#Z4<$!-&XkIa{1qN&JKq{am)2>wCFT_
zAaLIQ16`1F^)o<L1gMVSzjOx!SZQEl0FAZV0zr`|F7eFv>h=azm}<)c&;jp!M0#^`
zsosTHjKYe-V}B}iHuGA=1CfJZMmq%loljQ7ZJ3EiT6O(*ILN<u*A3hF%TdIg9?hq#
zJi43YWAqG457IV+)|SdT_WT#0>6Sc*tB_M1%gtybZjx2H(Q!A-l=`NA0aHW;?7Oe>
zbW*}1Z5N4>Ik6?`9?656MxK;gQkUv^CCr;e+?*R8bR<MmJgtjnly1ybg1NupIKUK<
ziI${z8UBW?*-OLJmh_dMPnDS67~5YL{zBjdHZw$*Watmk9A0ObWRzn34T_h0&JDp%
z)S!G7@4MKW6x1Ea*bbG~ReHiV`9~@eHwXeIgcyb}*REpShn!qz%IknQd3Id*!{{UX
z)c_bX@Hg3pRdoCRFb^ClgKMW-&Gc~@&nh+=B`NT#gPy*VpUE*1<(U>@4EwnYii7IK
zqc3DX+1wS8oWMHJa42+p@t#8=Plz|F%7&0=@9*^R1j~QgQ@Q<a{c|u!a!-JMX=U1z
z^Yuab#4&4oP{jB9h>3eZ)tV_-_5K(AH(<V@Iu)8_37P~o)Z@;e^&yROPuvz&!vOlK
zxmia3Hn-rEZ^o>mZ%>%Tx%cefsWN@yd_gE|adW@RTvjzE?s(bE-ackf!LuGl1~&ON
zy<Nx4!$45Oe;3&``l34eK5>0iXc5o`euvIRlA9sF0SAQ*$HEu0Eq%$?<~Ug8bkLd$
zW1wc(ohxpj;`KyeS@@5i(^4=%cz%UTjaUoMgRHVKK=rGqAucGobu?0=)e|VEMmW_%
z^W5CoR>8+XxiEJ@QY8CxuOIyVKl%F=PEpus;Sb4d?kC<3FTTw+e3@_bW4+AFG=HV$
zK(C%WnH$F+?WrAqr1Us|Xya!DNl8Z`p1?x9CbE4Y8yEY{qvz;%>5IjN2KqBJLl`@{
z_3va&yIqSnesx>pF~~~V%VTqe)g`$EEQdtjL`EdC#;CwAqUmBW9kk*1hM`x08TV&j
zDc>FmyeY)D9tRNjfYC23d+SMT?y1874`Qf3lXq=fiO6rQIZRzIs>)TKzWX)WN%ES`
z_z8dx$cq<=0ovXNDP*-??wUfr#7TlP+~O6<?}BWE=rFM4N~hm3nQnaGrT=87?DE|O
z3{9tnC<xzm_-1lvK_b@;MAzTmz9zk_KQSp>LQz$I6!3~ib>Y1c)H!Q;E%fHn@kWIo
zkWgp7zRdwSQ?aqX%U}DT<tE24X*5l|`8odtwWJO!&WVN9&CiJFEja5_e=JL-o}n2{
z2i3ka&y*Zg!+$bwnO&)7bIW5KQjWjIAWpP?z~0!sVHwp&MPdT6b&>hS23oS%QJ@-e
z<VTmzjF$Z^RE$3KxH2vJCaQ*b4A5ev1@MiRtjV%UoucA!e1_hiC7tY3u<LcF#?N9o
z9vK#HutZU?z3gpjkepzv+*aPyy(63lPzeDe$orp~BDpy5_UHCz>rJ+vh7jzrE9F#>
z{xK<P!bp&t%?H=x1YAG4DiC)NZT6WdTnSjuC3l-dej%(9MV-wFsAP)H?CoRjXK@+K
zA3JleRTwKi-&sV$x#^V-jGBXK!By$7`qbyf=Bzhi=W7YvZy@p)P-VEeBU_;8Ph#h>
zS~|}Nvs}t7)+-!y7)k2<axH1+Z4?_X(c!*F<z@?0YTB(2yc3xK$HsBV*sjzgBtB5C
zgXBB8&dUfkAHn^n_*0h&HF$MHh#8!M?p&HNdAG)PIvceY-9c_ofxL9JT8Z@4n#9qk
z_2LNiJ3kv%H<~CEPR%#aG4hicKW;bSw2Pb`bZ!$!#B;|TlwiD{kMk6EpZN8%)ACnO
z(nT)KKv(k4zi%jOJM>P$Ejbro5G@pKC=Gvzpl50H6?euv%WR=!T@DZJxbxRRhP(3F
zw$re~VcR0~Q!4AX{?6vN@c|&>bLTL9*!cyBBW=*(Q}wDsGn*D-#iUz^v7Gq)$IbrL
zM6iJuz<0sV4%mw)&`cNoeea6KfnPZMYB}+brUxx}j^H;V|M(wwg09o$kSzDVyAr=>
z<KwMeL=IEkUR-ilImkWiI{&!A^q(KYTksW^uD3dW8%l3s_=4xI^gz`)FvJTcgN}-$
zy;}T_|3T5jpAOOM9{x#pfN}Q9ii%Bad+e$l{KQ5jZyN0-oPWHgp3t0dxsaTVq6_}&
z-#j!LJQTdFEcHJZeOp@eG+&`?_wOEka)g&p)vx2_Z=20T+w!wbs}g^|^;`Nbx<Si(
zdcyI9*#GXK2^hf3hTlKS+MqMG5aM-W7H7AjeSQnbv7udZ?>!t+m<?fCh-nEN^ZWa)
z2^Heya|RqL+(hZG$^4s#rcnet#QgTZKn>`@HVXe5E<WGvNIi70e0%wEE%cr{NcD8c
zf_L(*3%Vtdc_yukR)3d_fMw;7z=mP2R!yz04*yw`agEbCe|>0CBJt6!d@74(W{vn0
z6bWqpUK^=+1z;QQJ%!f)u%KYDg8#;*YPJa!yZ9>ODi!`wPlL`EmN)Nry^)Pgj#x#8
zf3s8{MIpQRHz&ow1<tS#Gu}B)A8~#Zp;JKTD;i^vz2bC0@`xey-xz>YJg|1nfAiYx
zm9T5u@@r;mA?K#}`)VK*|6?Jz|6vQ+Yd$|<(3^R~ss7mg>;JIbAO1Xd8^9Dk*7)N8
zIn8zd;nVE?;%_4YYiA&i^_N2ooca&@DB;rc^9H$IcB((esOkCt|BlhV{)Gz{PER{q
zbZ(r<=PZxGN9}CKMAju}o7Ga8HI)f8aj3LuWeCKR6z|If`-&*M=MAM#^(9CkxTE>*
zKFt#`nn0F&L*mBm$|rmCf}Pf@w(UX#FZ5g8+j+K5D)m;Q?%K{VuT4bRlG}F7yP1{m
zjl0HWai5`ue-j-<a(jaiq4e_}fh=>k4e4vw>SKxC6Ors&1vTh61=X`>WJrJ$#fJF#
z(cRUrbmwH7?7m(zom9^&&Q)I7SJ6L+G=1uWozZ^6lHqN<E21j#oyq4kOAx(^^feOJ
z9KO-RmG`T@;^l7!&c^`=?@aF7QrpYNfYE^8{~51avtjnHKG9W;j_!Uuve$n$bq9U_
zEh1nM5AXF`oDpf2A;8x}IQTdGHMIuA+D-w_errV|{p{fvl+l{vbcZ>vpGlIjH(Y|D
zlASQM-2S;>1vvZ(U98jR%TjY6oBt$MO1GEp>;=uiG7bH!Pi$v<iRz#3R`Q&Yy+UM!
z4==?kE_8n=)cUg&-mFC6Ym&%~a?j>{g%~II^Z(^L$=RTJSKS%+zW!e*%2&`IF%-N%
zpP8p*4+5J+uRKt2zDdKE$UKYw)hCXbD7yx#-L2ct{{#zvK?#linoEIE=Jm6kk_BJm
z@T5tD`OkWZ27;wo{+I8hq9Yn_{mwDI;U7x@M8&)au~(@7=7o{_3ij%4ACJ}9MtD1b
zz4}+55Ix0ia{w4k!LjP}|MJv-7}?I&^xW?LcmJr#xQf3J!_zT$PG)ts&Hv}$<j%@I
zauH83UR*!U8m9J-z40C=C;^GqB9WHiTUr;(hKzGEbWwHeQP3~gd2#n-xn(t1@ysYb
z69S{K)wOta_5!*rz$nton`+MtJvqZmbRu*`0O!BRMRtHQ@<|u>T)Hii1dX^r&Obf%
zT<XE@I{4<H2F3FwetQ+1c8;v%GqJ-3M_^nY6X+QtfA$-i-Vve)hVkE?$0s~%^%l;<
zKjdEvcfnQ(4J4_(`E#PXkih)P%UqeyXafyeu(7t~hqCe~|G5bZ%;1~<+w+LE)|W&x
zp8dS28TjV^_EJr8gfkXtXRGBwjK8q7$5Mm(n_-Bu%N8ufe|sKQ^IwarxzEr3?EfzE
z|7?*LhD9fOJWK3;RFyjHsN)?}@%_V1P<CZQ{&{$-|IDSc-$iFNk0NulJ+sE`ba1>q
zR-Bj4X+6lm&W6l9%R1W;J!m6$ZWaAg6MXv<@S+p;r9@|rS1!#9M~g0OqV4S0-RZ&E
zSSuUFKHGje)tZg@ubVMv6hHB1z{9%~S!R-)*%YwRV@6<j*+SOm>Utr!Cw;=3ZTQsB
z+*&o{)&@QokDonWGjvq$uFjnw6?<@0^wu6PogD&*rDXz+wY{Soq5b&J*VKH)E{Qi$
zK4?F?x4Im`!8pun%{<esHBbf)^$}xr)Y<d9sENjBo0a#!J3H*a3;f^Z{p||>Gs_#l
zcF`Ra8$5OZ-pgTm5>&D(wagP<r2+2KifBOOCh&nP40Fzt6XHpAA?gQ86H$6iB@JvV
zvuvKug?-Ah$WAk)h=R}#!hUzi3@>s_U@1dIXs@)Boc(DE@u%cmz(1-AdY7Gw6fwbT
z$6!1>3+e<9(?3cA;6BgvEI_r1ngy()YL%0h*chw>7II6;GxJ&)kvM?r@2Y#pZ*qp;
zHxME4ynHrA$!Cn~l<bMX{7tzze9s~$gJ~ErpV-VhXXb;|#sGpVNkWOU%LCTP+O_9h
z6{e<@%7{54VM8}rc(mE74n{Pr3IQNi6@Dnc;{b|tYS1uj<qe2IDh*I}878a!g6i~Y
z8joM0xS;yIeH%v;?*0+D<P?PteunJiyv1@p1`J*qb>Zz%w*bQ(5`2D)jQi&n1h^jt
zHyftEZTACW)oiB*CJfxASH#&a$n-S8OImq_X&m5&GgSpP>Iu{FK!<y038G+xz|UPS
zYznWJn!vB*w0J(Dk8^vm^O~<B!DIEWDKD@r-pz#Iwia}?J@X0MiJ}Jq?r&+s7n+d8
z9e>U%8hXA4a0JX4eD}_dbQ}&i(p#5n7SGm-P6`AjzTQ>Ir|E!vXfgifl{AP;k{eT@
zbe@oWK^q{g(_7i9D;!EeOr359dOtH3eRsMdWL|>aN8_9Me84HA2FjZ~UK^KC7#OJ6
zOAo!P&|(7rqJRWXnuTwUf%0(M6GAE#k16Tytl!qneR}<zgYnPp`LBYEu1?@s14Jhv
zrU919?Vik)DNwa{v<KBn1Q0=yCzoqA$_EuX8zf!K(;hPJq|bpoU5;S%3jqcQo>q2z
zV%)w=47<ss_Dxhj*%|PZs>!mJ{>%wcm@R$e=)YPls-L!K1$udf>ff;4>vj@FCvsw<
zub$aXz7C~#m0z4k+3;2s=A{GE(kjyPSmHN@;y+yK`MUr84-DYZEnz4ClFwKKqx38B
zS`gB_5Ih+IPiR|Fr~@+iMbNp<nv3&15fkZcxAy1him-QG|Bkt&Dz{Tv3TEqS7O#L8
zVBciW!@a&R0_yu|fZoG(@D6mQkPyLSozwC41y_KiNrQ$;93nyEC;jeRq>)$i^J9iC
zKCf1sGXqe^qUkEGBi};|h-?_EGY7u<S1bSj3IX@^fy0l&+J3bRos04PMm{xwu~A?t
z0caRRss%66h!YI;T5D~}$3Vi}foARU^#yI?pa*GSQkf0O^?Adeihc_+<*d1gpt&fs
z&uWVK6X@;|)bg&jB_7unZa#WGc{t=Z3Wkf3;OwYZ0_q&H{bcpC`^ScWf4l5vn)T>j
zr%iNp^Z&u}pY0;hl8pdYlN<&L9pax;PeoTBhHy_T73;OLCQBO-gd1WBUIZ|jbOHid
z+5*TG%6wb~EFNY}EgoPP&(Izy1F1sXcj*9=S}0=@0G)Bv@tUXpX@rbZHsFg)-x<)a
zhEBcq*5dZh#q{hfR?u}6D1ah#L~2!08)qxR+I3{)cj;VUF(olgXm2$CWZ^eH4-ieK
z#L+WE*vkQCS>)pEag)f2Qqq?(Xwus62Gm_u!(k~<w6DMVuy2M1w3>FftwAel5rPK`
zG54~4ZvsuC^k{%VUOL?cG|Z&64vD!I7${{HjwSo;?uP?g!pGaa=>Q;J?lJx=&N8bs
zSqmT&mWrXixyxNpp6@~JD0){x7ztHo(BKnM2a}wqDlFg?ke0wHSb%m#HpG%^F%F*T
zu>&ald8PrXt<$W{{WSH;G`)^(i0&|^Oo1yCAWRRCjOiK5kb~Ws!6kfx3z3#Ac~k<&
zyJW@my}`}l6_Hws9>5r?gE!YOTK=WLwf5UFvi)|9d;}hC4$(z`bPm}Y73J|1Z`&lM
znfK$n2nah_Ic9dKVHCiV1~bSf-?hCr<x9A@W?wY}jiiXEX#64BHbB2L?(&N$ZA%_#
z<DNOZzb)c_Z$6SB4di<#Ispa=5G>`(d71B?eX2Er_ds4EQ&ig(GwIzZ#93eg94^Uv
zKVaydTZCY_VP3-!l`Q6-48yaUH@$igxgoN+YA16MYD6DutD?IAY$?4di5^Bq*<zY~
z(s$q7o@nnX-ztI=nGrDivGj2W_%KcbT>{A%06kw?S8r_11M?yrsy-eRfRmQfyvqS{
zYybqHzI5EJRfvg#Xkygi^GQ}<Pyh*lZvD{@4?`d;MM6x>c*3hCGiiXP^{Z-Twn{vk
zdlhgT>P-}$ZiipBvU?+-$>em<*(&8>RyYA5bL97G!Oj-|7GjDF*M1fiL9QLX0y^mn
zxNEfdk@2&h#{-`2>w31LMQ>q}BO?aj(WOKHHDrI{w_#&(N3Pz$cF_qy=c{NP!Z+b&
zd$_zbkSZ0CGhvWW_km(D*BfLc{RwJ+DaF{O*7SJpPsFI&{hHQ1OQp~5^nZ0Nh)!VR
z-dvD1x2QUpyGhigA$)K2Qp!Nq$)XaA%AudyjR)~;y-tAGTODH6-T4VdjI&h%!a~=w
zXXSAU0G52qiVtUWOifeM5_g3hQ+h<zc9MyrD!M!A$1eR^`zNW-ix$4TS4&M+du`%$
zSi$vl!hP=(Ra<zPN9s=pKssv{N;(W)J;Th$ZzZ921A9B5Ax)@j4^UVaUg-^E9|Aq5
z;ao#KM%rZM$NhKQZ9y9n8TJb!z02|RcyAzVt;OzTL}wDg7&yerx}L{77X=BK<u27j
zmay&I%c+X4lFC9=>x&^Po5D3!Y&rr_>GnRlmw7iu3vB#Sje+Lw<{ZQ$w+Jo|ac51D
zj(4K^wy%dg^2bbF<$3CQ2uuNe=Iz$jD%WcZOXaM^wZ9#xfimp%;c2S0CIi=5M)JQu
z0{=KBDX($2-2vPniz(t0@GkZxPfaTbrk#6Z#u4gHBcNq!HkY_lQm5-BFyvi@kUj(r
zYfU!GwA7?DtzmFaY5TRv@H5pxRC?;2ZgX5qd)5FgUWOWm99#9Hq>BCUe%x)7!6kq<
z7KknG630O%3WDGY2?ZN!25V5rX%8G|XrQ{fuDvSxCgVk5o-vH7y>kE{?xI+00!K5|
z$Ol)A<%f_bm4Lq~LvJ@P0{J)}%zfCt#&#z?{q-#_O9TSu7NYyZbBSJpKhN~=%urkL
zu7YjlJ=>6$eN~01x3u11CE8SuT7_uap@{2o7q)wQRY$=Uq3y0k0Hd?E%!3mo-!Ot_
z(iZFi4_A6+(d4UXe72i8;6q3i228f9;NmqCVtCvsw*_?LbdO!YWuI0BY%HQXovVZ2
zsHSWgbj5dEgy-*erZzc)DVxQ$VTf9=-;AJ4=kY5dt!y*8t_Zw<0*V+S)MhDyv2C}Y
zg)#l7$=D;|gP+4G6%g_~Jx@O3@yy2ln&)Zx6}e;=<6QD*WkXkpH4MC`ByX;J6wQOB
zEjF@mL%aRRHE4x96P8F%-4(Ip0b+{ud7(Auor>sFK_DrB1QWo@Lcot(U%9pIjGm!c
z4D<4{7!o#&z)m3bEV=HtTn(`WBTOjR`^-333YhmGM!~+69aqiHBfemhJA|&_GF^JY
z$MiaUPw;nG<ekdOK6<Jkzq6Bn^mi2W4{Q2g#~a?MjO@noY(&ijb~5qQVLv5rU`g{o
z0fGuCn_vubz8@_v>e7u>^uFdR+qBH#GEPsd0j4D#Li_G)1$rLMAiEz@=!z&9SOWI$
zlsLXq=^BW6-9pvj5TqO0>YT0o?V|cVN~tM(5XI}k#v+g-NFnH83>znKyYslzbhgH~
zANXl&d)zuVfXdDmKK`XpoP1B`H@kWUbVK&<I|pMs(XtnT(R(#$hR<sca7T}+#b=lL
zn|#gHvd7+rQX5OnWEWkZ=PN59F7Q<Ku^tjuseGOzIsuyH8rAyA-OW^MwCrK;+>=}k
z#l4_(<2ZL0JXz2Lh^1O+v;!e7@13%N$}EphlxCWZzVQS{%tYuSySag?0N$^^WZO>P
zZm$g>J{XAYm?2U9sMY7kgSQrB3u|a2bn@IPb_PdzJO2cmR!fz+!qQG0-Ysr?GP35(
zQ!VYlE5ClOpRpAMtLfo<#z@=f3~<5BAt}&oXqmW4)yo6nBt|n-nJ@V~5La{=d6JzR
z>1Bdq`#uf@1UuUVGO|1!ZYMw(AZ?y6I>&YwXkFXNS3ZlfPP-ev^FuIiV$e85J`GY*
zJUQW!CrEY$DiKPBn1h?gayw1T*1*E33CHEx%ZE7Ea9B&?^#bSCVZZwIJ&!GN8?IYk
ze#E~HX*3)c$IJ0q)4}$m2P%?JItnB|@BReaY($>HvZOLD`<$n@W0PAuIVxuXP{EEZ
z9>T89knOpd<p~angoUs50w(bgm(wUpV=WX%LdPI3u`zS+#$VO^GF^j`)40Xm*<!c<
ziFxV~k#GW<`znCvNDJT{PPy`yhKvVv<8hw^iApspe59TMLq3}lA4CRY{y0L2!I`Lk
z1k4wvLD-|6G-jZRVb4JK2!4^4p(phiEN3<(q|s-c<TC+)cKRaLAXcNYNzW`=+};q-
zK`<WD8svkCsD5NNfdL~+Ld^Fg2ytlv5}2$Ao*6u%1=;m1(i5g$3d?xw47E?_?wUou
zFCf-(<olsptg=4}eA}aumoag+$pPL6DSWuBqQHcN6T)s}%vJ(WNrFc?EOYY<xVQk3
zkcsF*svO(Y7g$zLbZfYU=aPK7LuB|0<M%@4mCRd%`|}_gAjOxGMb_rYw}HqATuJKc
z(odO{m!5FlcL6lpY7pC+_FBMfGl+nC(>_H0qUlo;w{!l)uEEF=?f`;)wE;a>D`!o=
zgXqWkFcM(rv#4VYGt?7!yD%6G+S%Im%o|S<&=Ze`=ki1(xF1zK@_Nn<9Q5GknCyB6
ztY`RJ+{ywJr(@D*4C+)P*yKv`3NnnU*i2$~HwZs4x!VIFkAWpM;TW6V7Bi!`v`Bap
zZKxwi>x<fko7(52`-k+wsrGD{RI_B2{p>k;k-}WHs%^U(5MZjN<_)BVtx&cqGg{D@
zN3Gk7yMNMRjH0{7)HfjA#117-krwE6$ESS=ZHm@2lC}S}EBmkc-~tigs(07yM##(9
z7a{r}x&^Zo2UT_g9hvppGl7NZ1J8-$A34OYK|J6)VcBj;dOHxkF{;f>SVet|F{~^+
zQjInp_Whm076)<%Qb_23sZ7NhHVuUfh;hc=^ogfO`2e6@l$K3v7xi36Jd`Q3Kv_tQ
z3%`GV(CZrnrVi^e7zU9&kl4sZzSrCSf)>~QJ{N{kPRPg@G0()n*`{4DINlu@XpIqu
zCPTGbKQF$8GbH~t+&p9wfjR);jw#W1go&KSlN>0yP=!*Fr4{9{9MT0@$~8bmX&KOa
zLi0Gi;K6mxuQEiX1*VGZTb(a(k<lZHTdw>}=<a@GFMfFCXxQE*TxC?ni5bzhjqMf;
ze3;giw5TnfmwGKCX8_#OdP#;xT8srI_ga@A9>tv;1eyF!m2U~B;7mlzbLd8|jI+c*
zZ6^2{sN7+FTnQ%3t3fE8zW9WtYJXbQr}9bayC^Fv7UYStJ;&o3kY{Je@N^J(+_VHy
zp@KO}YyNUxt6J*oh#W0Ax#{O2Yc^2M+Hs-1A4O^HV*`<l3+rK`6Kb`aO+&=M1Bb!B
zKb0Tn_@Wasv{%gn>g>n5mqB<5#B}<xn@lS~AtBf$8mjIrDmw1!AF#26;mPhaEAOzs
zJ`fJYUrzA5rN#CTJrdrTd_#YZaDFf;Gq>?ZX3K{IZ99-oBJ1=wgCh`t308YCM}OCE
zDDL{aX*?T-O^aQ=V0uQLjMc|A>0R3=LlJb>GJY3B|3BQXUqtoQR*9?kAN<DF%dFq=
zPJP5O6$!iZh=0mgAhQ2SY|xdPf8DW3dHvG#pAgv{5Pf1<nu<6JyuR(J?4?44vaIFn
zT!Khh{#2EvYrq2mYCM^b^FV@RdAqwm2bQo4j2C6kWnd3(D|NVM5>G_tt*(7DX78$v
zoEywt0~x}R20Dz>Vehn4u^d1`Ibdl6P$b3ZPA@N^dg3(u3lI*n@#ZON;<lvpVLRcG
z*Qvp7M^Cjs>iu<X;SeCQN7E+vQx)9PR7vxC=ed@RC=NBCs$8En1sIIj+2dtv$t(23
z&i@pO?Ep^=_fi$|esap)E+ds&5%IgCcEZ-V^~nhYuc6<#3f^LGyRTU4r`Ktx$`rTr
zj^IXG*V@C=8-&~=MCZY{o`&Cm@OA>;WfDHBBE+~H{?jOv@YK5JAeIm}(teuT;MEz*
z{y1$bNk%`-#8f{@ExG$hwDu|dN^-!%c(#Nx*o2JiY-b)@voK|=qE=B8G*P$HiYSx)
z$k7^Y@~DPI^>CO;gl;`%O=VXC-rPv9szG>Nj)XUkiRJJZc0nKLrVKnv?a~r|k@_*h
zK=;XzdIK+!Ts~X7V=3Y;1nSlD!*4BdW_H)9^?_fcyAas~<H{F#T4Y)s28vqhbYCO;
zGjKJ89b4{v>q}VhrGaP)(U#O2|J;Si!8OlMMR>yc8}j}qB`p_egL;tt*fDC;V&#EZ
zJp{-|M=$shuv=o(VfeF$u)Qw7V9$Bc9e(t6x(`%D^s`mr9ARBEa#<EaE~z(=xZAWE
z5nT%3)GD_0QoAB*h5+z&)nyyj4W+V<D052<)H;OTJ}w)w2I(Xyql{xS6bkQqDl`%-
z+{8=&4sN;%Xzu3<#jG<r0stBvbT_gPa7#SFD<BEDA_9=7X&|S)8$wpVQ)dmaj;9fU
z8m?$O+ekYobqg)b-@Am0MpDM8Uc!u#N_#+eAuODI5)6tUMB<1*O6zXapDK@tXA{eL
ziu2^4>Z`yk0QZVeimHXrLxl6XK`dd{2sbUQ;nB2%5sWtQySy(SLux>UVE7kr{|5m1
z=^umeFT2iWh;Qzm6Bt+QpP^xoOmkexM63fxw5HkVqLyB39Q80?S<mNFdw<5G$me?E
zE^rUw^_q+JJhg}hOKZ(j*i;W^B{ZIpndS@Mfn*CZ!7W_MNQ6By0#;n<c;TT)8YcaI
z#B(e3*Qsrq=tb?Fp^I-CmZ9pXt~VR<AWuPt2{>+kw!=2%mHfMv${X3B6bUZqGLd}d
zn^px%wtc7rkOTB!3qH&Fj$x0iBbDU*EA^J6@}4^Jw7|}M_uzk$EN+L^$X!C^%|Wv>
zg@E_64;g(0FpAum0g_A4!wg!X+CZUt<&DgK=-LChlO~*P$r0q#14K#60T!TGl&<a!
z@aw5n;I0aE+KGr%Yi6+m-UCd8<uQ5}PWc!`spscG={KSk1vDu>cAJ?S!M6kakl=#b
zmQKA^45s$fQ@<8XcpR#ND@UsG7MzPNQcH`g#1RzYk>3+4BGOD!N5jIcDpc@OJZy(W
z;4aC^+E3u*Nz<D7;uFu0dq8<R%^ci+NsL<J?h=Ivi9=AnQ40)SbcM4e?^pmcntJl>
zlbZT;y88C%iL^O0a-^pizAgkR`8y6>WM(<@w9Esz+u%-q#)Yst1Y8#y5V1$5sJ`>3
zoDnE-{}94PRg;T>B5<>um;i%PuXaut5(=5E^J%ZCr@ado)!&_`l^P%`?%i^uTMCnL
zOMNl;G)kXQ#Psk{WL^QHp2@q0lQ|GB#|nye_DL0PMFmhH$DWcip#_E-)`o8%gC9-V
z)E_qZ-n`t)`H5hm^~D5`tXm0L-ggO8RZ9i^namGsq<_W4$?tyq0oDGx7ylzaAa9U-
z(H(7n*-S*+9fX~k!W(ZDKovMt@yBZUnqCs;pi|)#>;J>vTLwh6wf(~e3Zn=LQqmxx
zbO=a;s30I1bc52Rgyc}7pdcJYN?HV@yJ0A$hVCAe?wSE+sCRA8(Zji)bI1Ao{=ezW
zu=ieTuejEA{X#l66Rei3V70szW@%&5reFsn1a3k+>Het-gW$~aKx!4OlJd}(8|D+n
zFsOL~4$G;WV&So9%f@TbJ~<rs>9qm7TQWG^Q>S;Mz}J&Tf?#TXD_8A-h$@Si3Lu=B
z4pnx5fQ|pcL>FYtr@@keirMM_g9H$7tF!GR)=3_ZfxreSXX*k{rqnuWC=%Op@O~Da
zHe8z91$i$$8<48Q6hf7*g&=|UNTYF+nujV9YFVIAmG#<`#rngx`;W9d5EaPqtL?_0
zpM(xXRHK&jD7>cp90kPy&Toaa!!Zu+TFbfhL?_wNMH-bzGevJZaD2_qhzQRe2WQ6n
zEOVo2VTM}5csLTrHfKIA#)iaUNOje+_7p{?92NwDx*7|+AoI&7``d;IkGmG8xTGpe
zFm1^{pGS$MerwHe^m|PBgyoh|GE44taQ5m;Yz2j>w3Qx`IV*K08qUsba0sn%1NmMZ
zA4Ios)WMU*4~Ea`US0YTmPWSx9>nSkBagx+{u;9W->??^S<)G_$qPj_gg6gb=Y~h+
zFf@H{C-5-lsVa!!=8|+WTtN#6Kg$;Z5~6y9fv4-fXZp+J2L`EIz}~KeNM@!=6~W#|
z)Cfjmp}GMav+%(xySWESVAUn4)je>{)bg=Z$H(P+JSw`x-Wf6P1-50Y7fPg+;=mz<
zzZ6OqEiU+B0uOi1z-bx^6m5W5B}NlA&(ehUrKn86cJ*&62^(b<Lh?IKgB%@=m*y%q
z+r=_W!nIgK(?`-$<FrS@2HF}`A1aYm7joM92EihdG4=^L`n3&Ge7hiZ{cxI(>v$Df
zCEb*Dsif`<M=xW0gPf?Ojrtt*odg~uq-wm9;S+G^&oTw(hEyp!?2S&c2|T;`sw0Xl
z^#GLp?4=29v$x-xaqMr_TW>A2wpHb=?O^(>k3X#5hmk-lS`HxL`246b$f;0@hVe!}
z^QUt&r=<Z4ob>?KHpK?4!hN6wp(AB?@DNw7I)~gogvtmJ9LUt)!#{nB!MSdZbm};l
z`~i&~+>rwL|0&k|>w8Rzsrd|!B0Z4z5#~(9`5u>SxbP_)$*~`XdQh-$tyYLEw*{h5
zkcx(gVG&4i$7`$tleh&VSV{~~Vpd~U7aW0ddKw{$EO~_x14tVv)GW+CI)`StIY>Bj
zEmg0>a99e+JGzHi>agrYd?URuXZ3RG!BsoJEaeHjCl%ahSt^Z!iVT+EM%26=mGVws
zRadQwXY%!d1)G|$tVP<V$O$7~Jqy?ZEr{uURD!Cx4?O_+k^X#~l3JB2p3;wSUf!wZ
zfu1eM!GBg3o&RK+mK@%qlNHWx4%jJLdWqa8iLXz1v-cqtsBIE*EPB%DAx@dP^aijP
zp}Te^VzmXN54Cud9o#vP!XFyZ>rmDJm-NFmlWeM`Fc^dw3*ca$bMhH<EKo}?K1ZIy
zv0A<Mf+w@OQGgnx8&OxUMXq`lhZWX_Cvxr@hVUsnwW{V?ZP_(cgHT|xM)$-EEtDy8
zTXN*C_Kh!KFxrJZF;7RC>_PS3lj-WFmHasmp7Pd~d$0k69``P5pq62a0mt!Vx<atH
zX<&)INcxoLt8h-HY8$Z6Bqsw&V>m?1Y0T9_%)+EoL73<31F8?gi$<`)@J<^E(D!Nm
zk*iO&VQ3St?LdRQk96VQb^@29ePC3kQ9Y1lfO=oAo441Gj9ksdi@yqLS4*eJqyCrl
z`yapQ969mm4EpZnw#Rrlbv>uP1r{FHJ61&xY}gYk@MINsvMb9#rmSBL^I#}_Xd}uG
z0c^6Q3QvPv9evJ8Uf44hM+_t&RSufGuuXbhF<>gH!^3pnEfWM_#1F}G@jCgth&?7w
z_Fb~dyzg)FAdy|ItYvR8N_jVxDpz5R?ORSEL_i^vPRGA?aOd-t#N<L3s8WL2+)%k8
z4qHyn0+rJ~D}&wPa&6so8dipD>KWfoz@zMZ10ibmMxiZKZpTO8r5Y#95<CfamFjDR
z^o02>i<+1Md%slyx^$+3Q@-jMX!q}TtFAJh2E|RVvmxB5Ep7dwbG2R+XIQ5v7?&D=
zrK4#@MS@c#5KPSJzD+$xolVas0P47kEDhF*u+Ju1NYO?KnOsnLsr;eFj^F=4LcpiJ
z5dEk)MAcVOei=%_I6<M}s)<gUa|%@_^_>?8kL$I_RsqH`4+=$>Yeon>57OI;i!Y@O
zicwH{X)78O@zVX66g4sF8QJM~^D}`6aNvSMW^%dVmIf4&N{FwfvLr+#=xf*cVU<6<
z)7L(MWSRIdFr;4qPf4yD2T`&X{+mrvWG+a%Ph032@g+{Vr2Ggq$H5F8=pa>YMX+;y
zn}_II7*7@yN~yXKEa8iOPYd*>#6saVu&<_GkqB3)^lT=vsFlSYVRDJ?2QaBd^&$lN
zOJ80sl?y|p4Z4Q^gW)x20Se6)VJQ!stUuYrxz3~!0ht9+f@_0j4+J?G%S}J$^pvKZ
zcm~!K4UlVmvbYR_DgH!|9w=}FuS05SkJPsekfIh{Rc<EmC_|Z3iTxQVfDw=jTe~-`
znR%OJPVq&aDl>--D9PkVg4#qxL{?)WC;?qHidW^O&Q7+Rmcgf0H4(dNBMwH5I(q<Q
zT}u7TUGM2vvPFiGcZ%Fv<cXd%HVJm6o-s?(1}E|%_MerWVnU8z%|-)+rV!PQm`%1c
zh)ZSB#eoy@-fE!5A**@Bs@)0KBp5oGWE66I^q=DI!aKcbP-ooMf6fQNsRH(;)v$1B
zop3?M=0Z;Q8|knHNUjPanGr-?QAxkZ@0n&BdF;^*PZ(7y3tr?Q|FZF0*XiBJ7Ec!Q
zsHZ*tFs<<QZi*;Rx`;?}^}C;WjJ`mwZ_=pgg5)q)L5>}ew07#FGK=Zb?#J8ZZ{v^m
zcve-sW)u}C_X8uq+lNOo=k79$>QEWoHgv+&hbR?&Evft`$<jEox!oV<HhmUv0Zqj@
zE;aOnA+Ej{DCk@O##ifxNhL4Y80<kgV!$U{*{2ptcx9gm`%S3;OvEw*XK+NZMvS>3
zAZ$t6y|D0f<7MP~?>6Z0kX){|yH_CT%ClGnR*wE204{lum9Cn7DgB0l?{g$)fzm!!
znGCX)%Ugg_YQTK#wKq7gONwh}K-dVfmr6xNs#r@qDl?EswN=wpdQtaaKSN{%X#J^z
z7`i$G6n94e9%J02!qXF~_{Z-}u`Rxaw<)-~3NV)Ag+9>BtOAcY<oTg*Zx}dab=_Jt
zX?qr78NAH3`No2sK78S^Neo^r<knlRLRx5P2jy{SCzJ-)pe$Jx9848;s2sVE<1oM~
z6anN!;W#q%8!nFu1nE`4;gWg{yTSA=BLsMRVvyo?>JprjL0f!kBv7?q{a%I=TVnQf
zgaJYuyhmgLZXKYkG9ml2c3w~RE*nr2%~~CM0`jI3n-Y<rz_57UFzOC&21u}5sfBaY
zZ^BhjQ(~P5QT**sLSShrqNZu0gEWUgmLQWGB+x`Zg|oZ1h3_{5cg}mn3QdA!=M)t+
z(Ivo1*4t2L9LaAQ-<zrXAz=g3{Z>z|<$r@WxgjwM=zuK=ZYT!;2Ga-;=YJHyu-Q>+
zB0^HAIFXHaQsAxMsfR;6^(3rHifX+Od(AVBVFoJOu#<6;9#c}3=l#~_VL*zzYbe@>
zr>N+aY?rSmtRoqb9t88U{4w$pDDXB)q_`Rb*v2s13!kgUt_}{IrM05{uZ5*wp{>tV
z9tZ=b)%6@^#3CMX?9K@tSe4@$-MhW{qd@&L>x?O^goU6t%O*eT_QJasUogqM`yoR`
zt>JhRs-7I!71n_Fo?Nw`H$1B)Qb=^`v|%BvO%YG@52({6pkw=*73lj6S*ywYDj@2>
zw)mTNO7!bju+L5oiaXavoY)6U48o*QhFtB?22OiIiLBD^fMj8@m(F9YmOateqZ$D6
z4%|i`+{qL1^lgGgn^yV}15b?EqH$$ZO7#XDxlvT2x)f-acHjz5I{nD-&h?knwu6#x
zdLnD!AD>BZZIWdYx)SyHguh3A50w(A=Aw-QZJ+)GisGdtNSCGQdiHev*>6gP5ayg*
z2>xUCL)fwu@IlJU*<Uk$k_Dh4^xvh(g75AV&;UO|0o4s^Ykt^J7O<v%UNxpsPYm?`
zhPh<wOx0=_KD{<PP*jY}Q6<U3-}$}NkA4LptnrS8eLwB89A5zpU->6y&eMOqp9B?v
z;oG_+r*rp@$B>R707$iW!GJ;b`?LDc0m$289#N4f9)S{<52Rh}2CPAD7j{Ca=pqm<
zg_w`()6YY27$=aC>dhzv#S?V^64S*1$HD=!3y_qen@c(dXsan#DI5LnFD5MqHkx40
z)4vYBO20wqT7v$^zdB76g26T!GyEJ%HFz+<gkaDN{h1g+mqE?5JN5(e{_y?;NC4#k
z?X}Vc5Jk_I?f?(2f{eRDL!Yz(6iO6f_xpwB{uMhr#|@zBjv8K?mwt9Z4g}Uaf(G_k
zApzKRy?rB#pRccq31E0RBrGJr2c7;WpmUcK&`47URZvYqP<v7X5G7|&ZG>DF=`A%)
zeqZ7B0OYUy#qW--P_~DZ2LOLx2@B!>8UphGp_{7X^3ywwfp;CEQj8DM{Jio06>#H_
zD%;p!y#$C7(4HN)qR;UY4Lm0bwMZ{Qli_EJRG=2kuN2w;e5u(G&fiJT*yrbkB^bel
zNyp4tetrP`lK1~T@gIPg|JTQVo@)Q!sQ7v0|Nrf%;2i;K5o?aZ+0B;*#`vd<X{+XZ
zWf7}CH<2GlRdhzp?8<?h6OvQ~y||!b_j8f^xt#y_(cc51qz3?F$H`_u^G5~v=g13E
zp78ajOAB_LNbG_SrXF3We!h^OKm9+KWmPgPPE9R>MdK)0_r4djQ*ZQC>plu&L~XMk
zeIA_J#1^$(W^ufFvI_Hmz37oA0Go&pVr79?5p34KF6hH|R(!HwC!KZ$YX)^gkgN7j
z-1z@_z$4fCvk|<W0D(PeW})yaCIJ7KJL7`~(1%Uc>5u4KK68XO-uT}x_D6H}068$8
z%jeIh1SBig-=tAlD*F|~2P<Ca!4F!r|KMHK^pXQe{P4cn{E;n*f%m!e3AhD>_a<DW
z`X?R9!0QUy9BO9wCzu|zIr2Q{UYL>K{eSXck$`;Dpk)4+Kl$7D^y`KFn}1vZPjst}
z`RKQ9Hvv;dPPE(dAN2kI;j%$5^Y2J+N8k3JgW*>%c4mR+85<S||4AhM@lDdj^K;ZJ
zF(LmXGs&t6A`)MpyKzTHLxDE<P(3i3{%Cvuc6k1JB5A|N;E|Xe_<vG0e@uduAc!eA
z&wlnNZRu~?^5-RfpEu|zD5P^L?fV(yf&NX{hFDb9Az6(7`xEgBg1<>It=6(U`uYh0
z^-B248{z-bnj=&A_=!;HZ_@TmkLK-t)&aYK+Uw!NpK<m7G75tHL3g-Z`S3IJ{&AOb
zKm*AM6s`aGzg*$xZ!Uq(K1Zl>biQ?-1)o)Q2KN7DV*lrXG<OS3D*Yw3#UDOZzyDU+
z;4EN~Ic@8>9__cEH~#y6O9%vgU0&+I`FpM7kGFiA8T8vF!Fa}heZ~LbVoH^ui@s1t
zpFH}aP1C@KoYO)A|N9ep*93q^x~*h#^XT6wodfMn>rx4ScXYV_mp+mLe{-<tVD*az
z1(<drP#?MA>qs2E`hRE^`09V-pdM}Z-}kk5&0}!$yU&OtkFsR(8nnQijgh$<{GVGE
z!@o=UQw!iP?dDYm!)FrDg8%5JbL61yw(~ILpXTa+nHfia`)g`bf&+?hXZ`Qikbk^-
zC0B5vtEd<1|NYN@T?eqgeGo_K^WYmmx&|F@htm1wNA}-4ETIA#oE2W2v`2qy3axP|
z%^v^Yff#rd@$hh_u!5+gZ`!aQurAv~tsH*R{*Qk7@2=hoEsU+^R1rtt)<bFVNM{`1
z-TDV>3hn_|J$Nd{6poNAI+4&yFj~?7=IGV`_FetzY*1%B0e@Jw%_(-YlSBzX5GKv{
zlUMz}%;U~e;6i$xF!H}FLz7p*n27t3@>2=n*M0xPKPaJzw_b3K<>(x9d;xX`4aevI
zh&<KG0PcSB)$yZU5IGDrxZ;pg;%I~a&6NA2A3H0cjvXoK5BjrXeI&_1a-=7?WccXS
z|3kCDSJDGu20Otiw4{$Vtu+Q36RuQ0NzDICZ}VnABQmTIdvwqll0zes$-Lm7EWE9s
zz}=O6seYRD{kZ!WH%JLAOFYK@=YBc*q&}Wde;lDML*0u8iU9^4&$7t=K}4NW3Lfa<
zs&t^rkx2wp8GI1WF612D_>SKF@80DYl-Ho$H@teZ5xlnGRZiZvxbeTgHK0xcnntam
zOnS5xHB-=P>`AYA;^@0P`Z|t0{?EsgW&_PoW`B7!QZqaQUFchT!QB7;NYX0cZ!nJA
z+9F48j=K+*k%es>Ao*$S1A7L_TqX#pRWxWEc#d-B)X!)5a|r*tPk4*K;7U>V($YS1
zbpk?&p~ZQBAY=`)4K4>tZTOtCz^7gfqCuT}>G}`03MDSk4QYJ_;o(OgrXDQu=M;M}
z%zraS{$s!LvV+Ur&%>TS+Msd`ut2)+ntoDb?Gy$E$Va>UZ{EzG58{&sC0~nh$nIYW
zN^o~WO>hsRSLL2~K8Wm9rnA3<YW~|O#Jde$j$MfU=r$p}4V`lBP(hB(AXs5H5kK0|
ze{HLe7UL6OME3*WfdQ_28jx)W2B?9iy7d>tsYyVz7JUfO%8{8{m#G=_G!~ND)`rke
zeE<os4#8<KxS&Ns)AcV0>mgpM04T8gn*&5E%Af$KpuQ7lIq&6c|HiD6C|sa74j{5w
zEbYaRXJ9_Y?UM=}!1yf8MBCZEkpv73Rp3YAxQNMgvj+%L4M0ZkB`k2SJ*$5YB6SK7
z+7XpbhwG#Rlek=w&z=VWPhFsG8f9WiDcOhr*uW8_Nr^PyB!_FKe*mPNMI(Ug(1To7
zGB5`p1G%+<GT&a-0JEz-M7IMDg$7o=Z_%<=Q%_y&!7uwBY5|S(M)4xZ<&V9`DqeHz
z1ytTI0%cT)_pY)#B7wz$elf#9tT6o&h+>98<gsZK25KxoDxxcZ2ZY;U8rcCwsy%M~
z;T65YjVij3{hPbmkg1#6Rx*^s#elM5ZzC~w_XpI+%ZdWPsJ24TUl#Y4K~Hit6mpBa
zbivK1E}NxX9S2nm3@YAQ7%LWPBcH+UBdK%z=ooZUf@Y-iXK7ULs;t$&A1QOEeEevE
z5kMvG#%^Lin^B_Q_ml^Qb5MO6vt4~yrJprWIV1Sca!3sG6)IrH0^@+#@rCP45L~GU
z5S<}J=GJ@SZI<60fMO@1W(PzK5TNa@1TgJ%$dabUF&L0v*=3r#?Pc5qtU--KBBy0b
zD((korm>mj2^zrk%O~BbB0Fo?0H}=pTl-cx!z|Q?TeS}lfb$pRkK&CPgqYA8mEXxQ
zxe(5a=;Qkf07~VoN^Mv9)Kv@ref04rKqeV-`kW<LSI7pce~wN7q*Qo%TC4z|-AnXU
z(4p&u3)i;Y>$#kZ7nj%|7{j;Q%fs8G($x?wVOmlCDa0|%&4fMIhf1)y9&!->_4w*`
zccA7HKmzn7g-B}W^n>z?CIlc8D0&T}hFl%&;H-%;iCcHFE{KWW+1UYrdD`<0?dGsa
zK%6O8LuA0rN=p0=C4pc<5%J27s81rN!Hv?Y@dgK}`#<z$7eIMAP1eIiGqrJalyUc~
zhyU=%6IAWiI@^Mi?-0z}EPAdmCUz$J(U&EG#%^E)U&QW}iZhYwvBf~7FwyDv57Qwk
znslk;B!r9a6B@C8co=JOs4G=PKDU@UUi?#X1R5b|5YB)>VG5Q(lz;!fUypeEPCbaa
zOz};qUQsL6FPem=#cP?1!g2P*SX$?Y+Hg-obeN<9O9kLl!21C7<U7B4i$r_6Gzwyw
z!Kaw%sU`}*bbqreB&C_aBcOc6E|>R4`dLsNiKor=n7$>++^NyWm}r`?`YBX79b#kG
zsa64~M)orRTkc;JpC6DmkdZ&en)se2T=OI02mX*W_ALOF%hMnCD<9toFCo<2@&qg_
z2>J+dxr+Lw63do$<%UVtRIX=JdEebx0ZSJB{AKOQeIW6k4#AB}^qt+_EmRMoiVtnd
zcfJQ^fO2eRkDMrx7%1tPMc!__kh)W|PJ~YWZf85!IeuE~v`2$L+<p9;q$^g%-w@B+
zp0aJY4vmwLz-Pr!A|JI_2xFpIf|4-~2|RsOA=keLu#?gGTLRRsNtK;)*r2C1kcb&v
z>_38p@QEXj1n>dR%c6&Sn;D?aYS#x=OAS;Xztq}HyDm2Ihvo*{EJfM5`%?BJ`)*z-
z{sS;S(pLZ%hW&68GV&{ld$w{L5XcZuJ3}ejEM|lKuwb=?R<Mp4qWf5*1wdAI2%}Hq
zN9|p&0@T)iWOc(Ot5&ogdqHJc(j#To>1-2F!h2E#q|j6$JVg-@9q4!F<YEm4MQPm3
zP1ORf{3fO%hnLg-;d`YY4%%Wnx=XbDV+Nau5fB%7GYPYqxmq?Gr=v5&?TPhq24t>`
zjgw1@`rGB~J@G@~Vg{yS72u-j)(%mo;touffK`$e!7|d&TFhxxoR<$b(TM`$={h5;
z3LIIg)74Qnq=I9>7BmwwRLU5o;u>*WIZqyF-2q5rhRbj12_Er(cn0`B8W5)?L&vVF
z2y{VQ*y_duMJ_d{EQtYNKxYA@(3o}Ga*9ahKGp@EGm^UaOU~@iE%x^{l2^zF;5~~V
z5UDn)7O*|dz(YN#W#^Kq4&*NXoSpr!4qnzNrsmXgJ&V}+TWHK?3pTqjM&&A5Rr!<q
za)t5&I*B>(j}T&ianp0TKDBO9i);>w&E3i!$ON3lfHlC1EqhdQ**kjGCY7~wyPAEO
z4Ce_*{Y0N@dR2T7F)AIdsP24vFUV09ujIkkWx!CpV%HY}Y16`ky=rQ>z4&w^uM9}-
z9c)+U3^2{sI<(#4DORml%;lP$QH+j~Heip2Tz>n=G<5Gv>PrO!BB|t^>DIS}y7*|o
zn6RwDh)4kf6htMqxlEtacwvs3uh07e*ur<pKkA)k%>)1zWZz*}r&_jRbCF<j_tOTX
zqzOP-B74-SSHuP{J@t(b0vClF->8n&&hC08qaExjruI2l?IjBA)bPA`3uoLa5Co9f
zA8!0%(8&cjam&wlzZXdx7&sP$9XU}Z#N)AYbVhV`0Q#_o9x&Ifw}*s8b5>mnA^bKZ
z9g{n0=GUVlRdtYPTl!WQEV~P#@q3SH>n>T3LcGsD0MEHLu?&Fld5}=^LPf!}zwg#P
z9fv7^^-mlIg-<#3qdK6rtVIh@VSU1Y1~7Bk)P*h-%&romJ9kYJ>;Pgj<GFp?saD$1
z4dxz*ZY(<5=aGXImud(6`FN}SCfa&PQ-LT(u+hpZt^zU59SSVG40d${DuZXi#*SHV
z2Rw7a)gt0SfKSapTY2izq_cbqr)P#>ZW>rd_tNmtOY=!k@fUG&*>F;&*HNIF)6DZp
z>ll;&usSl|Bnn($`v-XrnZQU-+OVH<h|9s@4<_FB7^oOm1DXf1l3oCsNl57gV2x=I
z_v(24utCu5b~$|bPvyA3hH1x7JR)rpDI)2mPWK%HcnhJ4y$2`wlB$L#2E$9MrN#ij
zeZhU2H4<hv!y4QsE%%Vqr{Qq>@BrmKadUIu)>Bd+;77LdC4_Z0nf7f7(9<btAWE5y
z4jgntK%jnMS(|g_Kh|Sg)e%D0R0SPQg~eHEgVfhfUs*!Ry6kOrb%tA2i`nC&mKtqg
zq-~b|x*5OitJ$83nSir*7nei<ggQik%t<wmZ!iavZ8sz#d7UsT(+BVziEBG~A@EFu
z%(E!xY>G0#*J?pp0v&$ZjSs&8R`r;yzEtmr&Wm3Bab&oBRI8w2aurzZvmmTQSj0&>
z(goz3kBUga%Vk~87GoQ<G(i0!Q#Q46<}Ras{j%xwfb!(_NUn12m`2rt8s^+J5*o;5
z1)nD5*_qX?KroG}(S?|!L)8wCmANy50R{9F#CsaB@fg3aFVA3-PYgi;3A-~XjfD>I
z*v+u>n51?hD^-rC;_QqdeGBt>Q%DPzlRf}?5}cdEcCkC3wK`~$m}@%RzU^zAMmXph
zk%~7-;vF;Bf3<V_1&>GKO6s1u@VoGewB&^e0AuJ{;PJvD<s~0zEKqZ!BiWz89u$ud
z3O*OE{B{ll%d-EN4<C)m-9x8JjjiJ^Z9N?^umk)FHRab~V!sKZ;y=%u(upue9h;K(
z^=LC`+4N)Ee7=2;YsBD2R+{qlcg5h$q#-f{qJ&vON;rrLZ+hh5w~qc4iuu>|QQA=H
zM4E0yU|5em_vP8#mzHINzz1+OM~>=FSQ4J~gLSjZ!X7l%h%y&Ah_PsvmY>qcIk1(z
zoiYy>iI4q(Q>1522PEm~+n;&dhmfaR`Guc}1IEPS#20Vjl&YN?e93$9C06v~9<d4X
zH<+Mk8WS@so{H^PlIXw|mbL<^6alj0E1N1i%*T@o0e#Wl2n*bM^tp0tPRPEz!5<Gk
z;t%nV)8DD1AQFp@IZKmH`=sW$fc36O#0TnUME$of%^^i+#HZxNPe*IKOpOP?{5a9U
zw+^pj+tY})uz0T?g~7i(iEle+<UDMVdOWm6Wc022C{2j8LB9Y}RK4IhN0$p<oJ=fU
zg~F{Y043G(n?d<jD$jj)%5m3=Q#MyTG&}6Jh}S|Wf(1too^@x4*ik!Soru(=IxJnc
z?VfommPn8jmA4mR@hN@zX5rRyd5<`FsKtw|S9>5XcdCUThl5G=TPrv>SUC9u*#{1%
zW_nEM-&tuZm=s3EdRcv>jhhmSS+%dzdag#*?nD%|eQ&Z*MRfNw&F15g=BLuxA(vKC
zdY5}Bn-|wdRra<@s<UiC1nX9;H3Lz<n{EN7AL&AR8`V6dMdaNQpM)i9Z*}&_J*61d
zH5l=zIE{P`DHK}$LZpA8cVisj^7L|0t#pBJ8u?sOO)QD5h*xG#la?{Mz}%k=Qxy6;
zNB3Asc5J<URFZfwGqaEO3MP}o!K2kseKX}#Wzq&m^-MuRA@ci)0@2?Y3ZhH7^|!Oa
zEuCSQX}N^0aC&-(Xr~7G%I)gZBb2>4oM#l|bE?3I;Q|~>0g3p%ki#EJ3B=NE*s05n
zBja1e7yLhdh+jr~0J2$ZxLK)DnZT6%NpBNvd^#6H47A}^(yT>R9I`N6|8QO>y@$mE
z3+$s^YOZab)KFfT;<ih&foY}KeX#kM673qq(y9^8m)KY9n<3H#3CKp=Z$w|$OtAZi
z7<i<uN&jyvrl$!2X+*FOP+7V!g;UeAit}kA&rvz_)D-&N6ph_%E=y89({gLM?1?0o
z(M#RYxK6FqbWbq5LlzA0-MF`yQ|+n8$F4SGpZKi3`EeA<F?po8e>#H$#15V00-SNo
zT`?YTpe;8BTXaG`$e;AB3Ljztt3re2N|=dlajURTBE)|YajDsKnL3@&*1_uXF0f#;
z43B<9r(Gg^eX_LnG<9JJ|J>qR0Fw0?jCET`!+5p9X7*+^5F24L2{<h9HMy_x@Q&0h
z(^9v+GB1V2ENk@gVyLw8=F2$P%Czw^gfZ{SNmV<zy8qU~1d(>^{XAj+MwGQwTXfot
z_ylwSvXh0C3F@>zY7|4#40J<GwFN*$afgmvy^996_EBq(a2T+CaV7Or_@r7I0laRX
zpGdyMwGjY596cih#j{zj1E|<~4j{Uf9spRMl%y!ImeSf%o^AF&i>{iaU5!k<bZ=%8
z1OlJR%umJv4t!u2VD#y2-+{GE)hBy7j8{DT%rH9{JvmGB7zgBzROs)Sa3n@UqDL+E
z#N_XrB^lu{4kfU6?^k8QnfGI&gAns~?JQp$=NIRqm;l#9w!JFoxWcvj7FbYL;lMFL
zJKh}uC`K(hm`CAGZ*P&O#pe_cTVkBofOKAVNd&)fKMUjRQha{A^`%U3CRb|@o!J%B
zw-J9X3Mc|d(Z>dFkI8Ez9zwK&#aY;vX~iAIT&De=G{NnnG9&w~S_`c5S8h~=HFX*H
z<q>08^tFtt6nj&IKSVDwi@bG$X-GX<_))tXYh?^v59=OE9B$YM450K(aPx;t8r>u`
z$Txu_BR`~MKo|v1=ptE~Q+Dl;nu>a9sYkIzXf@Z+>SI&{g8?==>}kPDlv#N7P=3xU
z_i!n$up9n3_ntlLxDXAX4fGcN$bs6f-OBW&6QmGi72WK(F*Xk1iqWB&yO)JyZHq~a
z+c8Y^*d{q`J1Dlpw7u2=$+m8U)F4nHTY$@N#bZYvY>AoZGzrBre8bMh8|;HD56#*(
zoA5<>liDRVg~8lbNiTs-VOC&@WT)pUwfkyGPE@Z$wIOP#5iD6#?y-xa-T58k_DC*&
z-I&q6vY>byK~&aTPkf}G*_K#bN3y8ldK=$OgwA~G65B=zqn82cfr6FiYKtdBBOO*u
z{DY7H8SeB*N8u`)Jx$F1&En1coaU$YH(qH~7mlh*44ezI5v<SOH=(Ztsfxm2IsVF_
z+YW@DK+>ux{lL@<a!I?!CNj6VQ-U(z16H6n!=ev_bq<Jx?MUL`-vF#n(_hwMU(<rG
z$w_H`#Dk*XAbHxwGU<&_dOnZ5LHyGOD$GH-Yn66!^WGba>e^c=d{n5u{H5vp2|4mE
zN5TqFz9VRD0SCWh@JVXa;C*hX%IS7Rtj}e(D;u&~hDMnHCmonYPyc}w+58qdSG37w
z&0^n592UfiJvJGhzNu*Ccj*4^P^;t4ds9HgwnD_uiZK3mnC1ll#tZYr<m{~>FrZzf
zxZ9Wk=|71AIXFmoYRdn$AJt}(x~B`BH5R!u27vHaY~Kj>e_nm>3WBUYDZlq7G)sI^
zl8pjV{L`OIJXIHa^V?z^yYC28hS6ti%9>-3=Zf6$G(-{)%ugn!#M0{mm-wlEVszB)
z754tUyb_TyH4VL@I9tytB(Xf!$1^va^xX6(<%TE_kqpxV8c0UW^_AZIR%{+So2<h}
zPGr~K+jI|Bqk+nSgCu*;ppI87wTM($RW{Ir#1vl&`BppxdAh23zPGX?#Z=kGhx{p{
z{V!SSnz>W<K2~+?$;p@Qa=W#ks!FnWuGc#rqu$<DWH74Fo<6lF=j)Y5-e;Yr@Xq0m
z1?<(Q@Ap!}otj0Hr<rguVwR_M_))FZg-QamSMkN{9yB2NS!nvLMXKLF(Mx|$*l!<a
z3i8X;PlB&bvM$}@{=RZ;xi%%@n*iw?a;I!fghE%C=S&AkGDvwyt7ZCS{K!IZFSK7@
z=1L5PaQD(z1k`GV?Jk!A1)_5V14SxqwUH2UtY*T+RIKUN%JZTaXB+*Caq9{RJhT15
zK*gzFu4h2R&K1Ne#V@Dt+~D97nM-(nnotUe$61cdY#Y@OyrF=}&g_|JbBlLizdu7Y
z%wa^@{f;>>dKR3cm_wFz#XPbrh8(PEJI;ka6$N~+pf>Cf-uz3zT!b<;80_JUOPhig
zRovMb|1)GWmFubClt{J3EJ92(YpPi&62BoHuR_-B;iZV&;!{nq1B3~B-Z9R}25LLq
zb7nE~5zBcJoe{YTOC%mEz2CzQ%&V0XdrFz9gg~e?<7syp1nC~jYB+nk>s-h%1{LiS
z@zK`#clImy<8=$}w{y*EKPO3VDvnsy-SQWi^#XZ;b35jM(2Fkj^9WIh6dy6dj_?|O
zJltkttnf1+Vc^_sbN(c8Gj?-f2%o5rgsA19lFHI{_=|`a@duxeTl3Sn2#~6r&Z=F1
ztz*F#I3^EC5|Xg)4G$oM614RZ!VptdCy=iI9C0uo7>76D)_0!_hqek2+`6_M>l(Yr
zIGo|e(iB>~3!7xIj<va}b~!@R_!X7yPK;~eDseXot4!FRe|L7!D0eY@qij#l%64}s
zh19ef<YP}qTU)uz90Kb?tCv>7+RUwALZL9M<8V)9k@cLGpZ)%T5~uQhiQ#2bt&+nv
z)Vc@vZhZbYOnqiJ&z>-Bn>3QE(bIKo+@3JX=E*f1(-ZLL%Vr;TU->;J^|RvkawyC9
zYRPjEXzllq>ez%YT`Z?w<^%tYy{BjEBKa;HbN2<p6!}y$tzF|~V=m3CnWl*HJla_T
zl?SPZc{sjo)ZGnRHp;C=Vx@!cINVjU2*v1eZSsWt4Xnse{WR|MS?*~f?J`9(cO-G}
zvhzU8brFlLH+CM6YQS-(uqR{=%g4?3(4L~bdqZjyE5pflb6Cf7Pr>{1&`S{KUzS$x
zzNRwB(i=faAmZ9LZttJ4b>;=^5TYn7C-vu$Dvf|cI{s)BpH$Q1AqV{WT5Uj!wP&l{
zBj~QQGF~yof-vA&?$_*%N1n=4*Hl(!8lz)W=N(U6lgZmV-)o&`hmzbl0CK7K;J_X<
z)60)zlI`(_Q0CYhcJl)>ddD{>9X1<h_}>89jyq;7x@A$^lv+_<U_{w{1em!lRB(oW
z134y_>6P4NF1Aemj=d|O<GZ(!1_$=L6AK`(xb97Qf{GE9IMdv^k+^q#B(3h7A!>G`
zOxXwVYEM4`>vnWC|I2x)K`z0MtEJbTh|T8}-dTI*8r|zs@3z3avXEAb9zB08G1ZUS
z-Zs@W&ncZ!FynmKSvYB=IkD_i-Gr(L^2LOO2)!Ec_Kx77kWV$%)K1w1w3G_Zal+z4
zeB5w7L)n}dV=kwxFL|5E9rIKzE$%SS4@L-G@7P0<V+w-d_Ic0t>^FN=u~%)tJLK<(
zS+uH_b_2?VR;N-#Uu?AGKXcpLX_Oil?GP%6=ju^Z3m<)4Sh)3C*<|NQ2vFfTy*ugx
zNHr$e`!pMdn{B6dV6$Nih7DFyDsh&|0AY^xzH^!Kp{?j}v*@=7PC1L|8`ure5Xiub
zR@|D1dcoEGMeLR|>Rvq8D$8#J;<Hg_7aY#Xz&yg%PrC<c6u<SL?-AA8sfO|hyDuwI
zbQ4#Pe|$C>G%^l!Xo^Ax=AF{NPJ3**6q0CKtZ>{G8v|>|Cm%#>#!}#>_<;E3$76&^
zK`fmS7N%aB>0Q%K2jiEg!WVCluwUicU2Cn_JJ-yA?eOK4o%8|u<xd}?Crl0We2t>y
z(gLC!Ca~@Tnw_khR>ZZB$D{00-dYG}^qUNq-M;L_(fvUP<85*HIA)JaUQ^8L?1pe+
z;TA)?CLqirwvO!qFL8c`(GOO3<HB5@#WWQImO*94x{7vWDe%L<-s3VcJ-Hs-y^&C2
z_?&FkL6Z*5m#uVYe7L$K?{0D@+N$8SWp<eSsjtdhwPV&9^CdvcNatk;CQtQztK;|4
zQBwg<REVaD0T_U7ip7|`Vki<43$;@C!1b+k&qmn^`F1JrHjcLSO=`40^M0_|5YYbd
z$ab|>N7$Jhc2-JW6|PlAZoB4E%k(l6u6MZL<61KFSCO7fBTBYUP`WGp%GpCL>n>YP
zurI>TL$M?AlI%8|jT1@d(9<trWh?20%EF9I?y<gJko57w<kVJzXH>r?T@R|vGkuAa
zb+KX?L39Fs)QAA8LcTr=0vME`Q8o~q*v_J79}$6ac+Qm^C=mIfa9t8o`IiabSC|Xx
ztYn9UQenR(KSUBc17e(pnG1)M!Ytk`3oza`lJ2HwO*g*X3MaL~`sD9tdt>Pjwkm{K
zJqG5v^+iM?c@5kRvsyWNn(t&Q+vJ~wL(*?%rN~EG?NH?iL(Dbp<f<fCvNzbBcCH_P
z86HL-Z(lM1E@v6L!Wp)nzyF1AM73aA4m&CWP7~O~GFOd?py-xYu{YdZc}ApHaJ~AN
z?0o^S7-Xs_zb-1-I^(mvI?J81?0mWH)!?3sp0;PJ7gA^ZfqpR^sdXFiS}cN;>MGmu
zhspU5t%%Wg(c9%c{FJ}A68~CR3({xs@fPS1FCde)ihsbp(k|lXw@b3YF>Cl?k|i@V
zuukzFb6D?&K|%7yFjOohZn;&=@+9JT+xa{Cv-_$>SIOe+4;aG&6p+|2G1XSZrZ19J
z?<oXuXHi*OoW|L2w$HRNJ8+wWdZw+ieT|vJN{%X5v)g+G`*nGS&sxbG6ydX@%4O1G
zo-o??;oB`=XUhydP1*U|tY08n1=OZU7A%WnoyW`6UU3OmTAyw<j+R#CUg#(rg);5Z
z(u~3?`=G+0VRtt6hG~?FxI<4b+Jrm6PU=wGmzql0r^B!516N|esIHJqu<lZvv@yFt
zQ3*I_+IM+qw>Prx4W->Mz;Pr#7iT@+Z4HH^86lJJ7;d&x2-Qo9%!Vr{X>E;8T(f0)
z{N7^P!`xO?N85dn6ZOiPy3Jxn*C353%BBn}A>S$3hxjmiW8@?}jm`<s1|zikL(+6Z
zSLN7Gje*p&K#XwPweITa)44zejfzdJ`}+rn_tU%gOfLp%Mubm1aS#y@KOX2uvN6`O
z6=72`l)wMp;XN4qVK%Ir%1<&1IStw!QdnZh0=Le<O-4N^Hue#qK$roN$7O`J65%$v
zf`;!^8U#M~jIOpRCFD>EwM$+f(G(~QKd@E5YL#snq-a?1@C9O~G|G0Xk>$zTRYjN$
zoBV+1qv@FX4pd}5%9^;~@!`9DUF}p1Eyxb8n)!}R+i9HmBoaTK$s5L?RUAvwt-O<~
zJJMO$TuEBly!{dHzQ7XT3sOJ1E4y{v1P4`TAgOh;!gA=~B`}&>QYqZss9VuB+V6h`
zo%~w=Mtw3%Q^cx!zg3|RLemzk_a^T!k8In=2<gLw9qPZ~y;VS>w!k?tf1E8W{kQ3=
zL^_}~WS(0HwzEAwB;wj`^it07A?tL(P)@R40da>D^|D$w$@tv>;Vd{^+rB$a&`Tp)
z?aW9-UR=|a1FDYwo2Kzjm7tm=%6%u;&iL&;Ekr@v!uw@<$`!He;j1nJof<RIt48T|
z;ZJO+^n=;(E3mRucAr9kAOI58PeUpcA(2>MEu1_(_+<aEV?3%x>!GT1#UoXw&xRLG
zU<9u}-ngfGte)vJf$e7o<9nSv3v5j!=kGBQbn#pm$USwDOh#Unpq9+}l+15;uLbzN
zyb|br;rPh_7`x}{Lwnoyta*gP>S)<!`$i1PE~R8)VIegmeeqdebvvjwMw>*jB{q$x
zH`+Y!@l1(*v0gdSa53z8pVTmi)_LvfZRYDv4EpW`_qVX{F4j5boTsaWFU)tRh3<d3
zV579~1n-qoq`1YG4fc$;{hI7^XGwlHY=*q2ez)-ABst-5b#C$zJ?+J!b0qQWMwR3g
zPbKNck_!BY@}cxCyo;%L5?_Q~t0E`_6R<M(7`e4zQt)~iz#*#i^o;o>#fx3ZgGlG`
zTtN|lEV_zi{!P^AU^BjmpZU;S<u_R;5101F4bkmbF@niKhLj7YLg5~MSudV^(K)-k
zQ3UCE^=l2RT^bOKCZqKz6A=0?1}efh(WmRJ{{9GSf>ZN*b0oJ($h-7b_JTIoeVZD^
zf}BJRESbI5-ln!e;`~>AB_(%mHS)|VvFl&2XQQTXPmLZc;8Nk^U@>7KOnfwkZQ>j3
zG2(9;#tZVxIRy{p9(dk8$M+X|tUvqX4{J!%+ulC8N;6a^-71WYo(PZE&22tnu+=1A
z<XfYbk&T|NZPHr1a57iK-%|+hnuzC?S%?ck3y^KKdbZ8!e(5oZBA%!wdGTWz17*dL
z8jJR<uXHYRvV)-=$@KUaZriv#kl7=>aRJA+=$_s|W$MoAi+~QBUq`fprNkrq;vQ1A
z#!4hTRgO02sVps`mE{d*lQzi}*)P=^CD-o?sJ^@O6)#_WmiKfyWP&SEdvA%Dj|{E~
z$u1i^+cnoX-ehvwJtKR)mvt`C^G<fySH_qKiA(u<?jA<RMB!Cs@uggM(Sf(MEoJEU
zZ3Cz;48An+(U-Wgua@JccC|q3N<=+rqYet!kMLAsm75-#u9UA9t1P(S{60@e#sERw
zo^dNtgf`$Zz?z_F_AfqV#vj=bB0c_a#*}?7Fp&SWcR54mUELlq9O<RL!Bhfzw0m~O
zw-L@0J1@tZdoo*7?y(KH+ngxL{?r*y^HF4lT3M@5h}u`u?^4k$j~`<Ik}o?dN>01L
zkhhz>F@==W35IJ2nvhJrm+4x#;fwc)Z;o<ONv%O@EhT|CQa)wVdYOhRnV|kv^(1fO
zm5RqlEJ)|+2#u^r)wMO)Ys%2qncNg&teYBQTB$E!AU)Ng+9?MQF%+kMdVjn3c5G~8
z+ZXr5?8WOL&OMMjg!trbt>$%;jy#)LfzttQu*q?dV_o%d)9NrOTdw-L>*wqmZGIt<
z{!#dMTJK06_fOBQ2;Qgay7<vPU3H?m#Q6HE%mrP>%vRt0t3Kh&1R>4uyM|LZo=?7C
z*Y1?l9|pU?;^p_t<|^%GG6o}`E(Q!=n0VAH6I77S*4d=LV_zj{OI@k!c=~YR+4;$w
zg=VUN@TX1zZUqplpTp@c*W4o#e<)27kOb?96sTM_NwSf5=k@Edv7sD}DeQ<a(`vIY
z)sr!}d(?{ZhkxGLm>S(PxVdXhH&TLZLA8{Fo@B?c9{=<G%(fmn`sxk;+U%WSzdHfP
zH<cdg`658UCAEi?YT(gUVsVHw!Gtku5x|6GII8x~i%oc+`~Yl7v&JP%qC=`YS~F~1
z&)_1tu6E36re+}@GShd=McN3MmROVS0>_0raDf;m%C?T+(z%<EM)b(X<q`aXQEZ)z
zDIzFDv^DapgN48bH9c}C_X~d2eNdwbe)p1KBrG0`Q2Mk*v)0CJh?@FP^7QS|yicJ_
zrfETNPAeJIC!`aic=6|2_Q%})WgYta3fIbg+>Zv5W4>^VapUaq8*sAInF901XkvbI
z+AgA(1Xs@HZIq!cI%mDVart{*;yi6w5KKW|$<r4uvCwz2h{~_M#yCjkkOw}m=$>k3
zq)3kLil8*E;5@bz>#}-=Np9thPhS8`^CbmGzrp%zjowsraltW@`g6k8ap+HMnBX?Y
z<hy7`P^{moq$>Uzh#$useEs9<dAK<G_yC={5Xxhd>GRUrO6!xyDaJ?ZT-pN=!+7wa
zT?0Qk7k~3>K|cL_S{zJg@Pv)fG7y3wq-}fyAXbbWK;Z{dtVMNs?(9=}6bvb3D%+`%
z>I&v${D#L>6%50(mB8J4a4DB$4<G@l9^5GP#V^3-eCFdv-_<r`BNB4%;`!%hZ{55T
zQ^deikZ}r*)%=7na~xhXWL$CgUJK^qszkdF#Xw74N0Xw>_6>t8Hxx3-*Zd3xzsJ2w
z;0YvQru%YDSD3o)#KOy`HbO5%%f^v)A@kh?Y~mk{8OvGqzH!;;%FuW0fpX!u_m9`I
zUu~arauHgvSir~Qxo_4Hp1|Y&m5JwDST6Fjdy_xq$7hxOra}RV`S|TLDxMU010Ehi
z1O6A#wHJuBT<)GlPFrN-!IhqQe0xE0o8;2X&B|y?Vy|A2N-J5;ZQy*6s#G&7_H^is
z6Wi*I%xq%H4%NUyS2j~(4{WR|lh#(qowzb}%8wG2Xrm&{5Aa?pCEYilX^lvxO^R)V
z7GH_l1g$0_X0Xg|ggNh>t~Z!g0bQ^85s6qp5#WzI1)uUMzVnrtuPZKd<keG;-f3(1
zxr~v|Z{tQAY$g@2$F)J?V9L!HT&c~PeEny|hTr&%yeLC;dY3N-_hDeW<#_NeAr`{^
ztra<!ST>?W$&5w977LdiU40V_vGZE>AsKp~=j_103|!7L(AIo%5F~Q1e`VG`XAd&B
z>;j_<!5>MF**QHA8bWM@xYr?ezVdK;J#U|+6d+wcSuG@~)R?Jl<0~n88X7GFsTiBh
zE}}#Y8?DYp|94AR&N?EJl3SGF9{0=_uV}^jA|QEPS0I@l7dljO0ExP`ei|xq7&=Yw
zI%1l}OcTr1#TeEBMu$h_n<x|$4Xv8TR80FEZ`LVz3`Id~*Q<aUJJHD`G1thDL-FW2
z?-fzI2kX!Dc(yh#kahU&@nulQ@ovhk5H#ikW9Ug8kBPu1j+dZvW}e8Cq5fvwvp810
zY<$+v=l^9AvmeJ3(lCHKU9jH=%9=~Bu9nKvamKzNs%JHXWTD}|`Gp)G%v>#B%#MYt
zXj#`8jz>Dad&Mkkt?q8yTo-;G&6i6Pa))wXSJy%FIV?0J*pk50_{sX`*_kzg3uwN?
zd4OSwOuG6!o61A)EIaPRxj}+;+319j+MWQt{8~=;IAolzWi92&0tk5mwt`eULgEKQ
zr(w9d3LT;joHYS%&H!0gfV-qOLU7^ZYX03$x$3J-(xopMz3V2Yz?qz+LZNyFl%b`*
zRv3{djsf7LgpXoOKmdN8v<>=H&+4VyBClV$EY3b8PMM|oq8@<G|F%xl#i8G6KYU@Y
zpYHf%rM9#zy-?lTVwrs{Z`HFdV7^nhig!2Uw-snaZ0C85^DY)#?QO>BlY1&f1nW}N
z)5$y_&w9f#mqZ_zk1`2>xz2V{1b?RCv0y|aXIsJitFAbIJqM_?WBK8WO79d6B*<5z
zT3M@maLT&Hd?-9Pg`VU6O&?;9ga{?^v!;nGbVcX6{26b&<_|QhD;B+6SNzIq92DCq
z-s<?hV!5%-$$oRH8Wh7LW;-g#k;b^|AuN+JH=5{oT|3s5a~7X}mSdtpYJUlu{i2<q
z;!!D8pPxXSsKL#1ft4q1F*3p1O+vh2W>P>azRk_Jd{IE_#G|CO2@Utvj4D1Awz)v7
zwogH3GW4sZF8;S;T&X%Pm)zf6zdvB6(>b8#5Af7QZ2;J!@va1D(7(SUuu%AnuN|Wa
zS030(z@;z7xJh9y(S_Ha&wtQw1vat8*K$>rhozIj9M#O0#PDzXBK}z;3U)5_)G11P
z_e&n;C)l}$^?49ya?EA?#X1-AC^z&WjkJ=XFF5|qHJG@!{!;8U4|y3H|KGBOL^-l6
zH(K1bHAf&TB)ge<A+Ue>4@ouc5Y)-l4d*B1D2_H{eyrDymbVA{<$3pSS96fVE3TqG
zm+2?xD6_cE;>s&``euX5A%XS^VOi<|r~U#1WeWO&LOyB^`fLE1+$}jBuUh$(Px8U0
z59I|qS_((*;0LF{^9ZE(&k7{5-553svX{X&ssET-n?-tmHS7JCMMmCmg@7Sb`Z9^{
z>Fm8AGw~!XphmB7Nd0_Ajt^&G{H~4CTOzixLOj>4O>n?^ZTNXCC>u#UB^$@)`=@eR
z!(<@2_`zsA$6gr}cJ%&=bGgi1eS;vW4E@RM0t(^m7Zb<V6{2U~8FlA_UKdhA53<g=
zLjXjRje+4%MFQOL7ka{0+h}u=x5V>pLh_ydsp)HT?>ne8ItN%6TGolRiEGaoEyyf<
zi%KC+?_@(hE-$l-Md-0|X2sipCFRbF)Ly+5nuYD=Gq_lU&4k2-aHIE$0Q|de;?OCL
zB?!SU;yh;TrJLcorT6?m#Q(q(H(f6TPU5F|;7Hldh_7OA=@w+VLv8X1aGn4TD}0E{
zIUKGK++YsBWu<3)vCipLe^b(v`tx_QKrPjz>cF3O7w7@blLikM8M_zb!GmaDIaBP9
z+F)+!3RHWQ;;)$K3SHQ?=JsFj6b`CM=C~Wg<<b?X-|JMmlyj<MNWen!ZQdDzs5(nV
z+a)_4CjLp<_se>#ZW>X?D#_7BUv&cgO8`>)Djhl1fbVu!Zn<&-TxMS=s5E+=-GzZ}
zVCF$*M9=FIV^$Sno-il(EJf?^$w&*T4qwS(x*+%Yd~=oH;o`;DJQ2IxiFz?ecV2PP
z86gxe=F4?ogl@<!?G&9Ep&dt~ir`RsWdK7THyUQQA>Aw~y7f7A?U>eCT>d>NrtA>~
z8=EgDiLNKukzhIJnB%7Jt26KCsdiFa)>ZiHQAzVM-i&r9zR(v-NJbTfUu5)@UzRBL
zj7M+nj#03oWwDK@3_ac8#*&xK0R3&S1yukTa``HHi7b0ny74qUmAx5WZTQE8kPzX3
zTV^9W_sFqtO6NTi>eI;!?>g&=Hb&lht<^JZO|rgm;XT!QB-OKC^vpF=FxV*|6N@%~
zq$quQ)KfdkBpL$%bBD_mB-nAcIHA6d{c*ni`<PJPkn@C=`GxVwl)&>m`e>LTU+xFV
zH)|gY0q%PX)8+WpzmZRTz>x3QE~w_MzCVt^?m_T*5g<;|l1-Tj2e^7cvp84NA%FZB
zHzsJ7f4o`AWkhiOygi{+(cBA}k$TF^wS{ZhZ`kK5fHaf>{bR1{H3IY}%dL2B5j>M$
zbc8TX6f)Y;2|PE+M1r+@$$Ta2l!VWVeK#T*ULzrcLWWVfv3Tm)AdWfaLjg_k5g;zt
z_1ku;Kqb?67-0r}nt#G-O0?~QYa@8u;>+hq(2T$Y0cd?woMNitbb0d}Q?s{pD~BBY
zB9gb=L`1$*j_ON=DJ3?8{QH+Y07gP6J-al*n6E|WD_ME-I~V_mF_=xyj=#MazWCMX
z%axeTvJQB5Z79bU0Bu#dqLb+L`ZwKYAcqrK2?L$2{1(1!6{;BtA9=q$7a&hX-eE%%
z&n06JF`$usUx%ENzf90~*l1|VK_D7;sUS~+9t5Hfz9v~xo~Xf%jk3;|iq}DsKx^Y?
zk`PQhy)P+h-5$)YwTNo?N_Ky#2*ZVJpYdWW&FDeG*ARNZZDc`~*+y<puWrzYIf+Yl
z<kjr?<+S<sHXGhLQ=b(_U&PZE7jpDE(N}lNh+FRVgYFBj*|vD_WKbX?D;>BWVY9k#
zt}|0=NnA5E2I6mH?L^Nn_e;V|ndX?Y%eXV%9{juH2v{TVMMb~QUB2<f5fuF1+ziz!
ziF|c-GN}_36~();^>`^K--eWEM*Aa%NYLvteo-t1jC;Oqx&5rc+OI)s!V)Zd+4aWM
zqPo!~Um(*c5rFkTE_pldyfJp9TfdKJGZoNp83gL<!xQ^p9G`q1B|cgISd#55ZZPD^
zIxsJ5ler|SI!{2|Ff5K{Z1@yP79D6Wp(J{a`Nx^=-^EQePw+s_;-0Yj*~{g|HSU^(
z#d0+_rA``WMSfZn>2OtE|E<T#bL3!dmeTatv#o~4q6_DAfyWk$Hje(nYZEI{^i%Qd
zW*kF@lG3vP^K-K1xxkYKBTmv!XQ_|?JEXqVxm|(U&vv<!!DfB)9*O?_2`Z*L?}7)u
zLNe^KsIMR(T3c$US<4k-Bx{6poM#}NY&;I+G~?W49YnK;hSbWc3TsHSlQbpdCGg;1
z_?jG;;#i@xwtbq6uVjl5ynCp;vN;y0dA32ISp9qA3{}7)iECu?3~sn`lef%*%Q`V<
zz2+qG(qhNW>#sC8Pr<9?1bP*+x9LDGL<sVh!jJQp592$}*iYA2aq;bJxkzAtegaSB
zLEo!px|B7Xf`lsd$A{vdj4`{GqI?%HiqW%~dqg)c`)PhgZtU~D<&I+|Ll^ZJj}kG(
z@v=6WZmfwj3Um;D(-XQWIUDP&A1|l%#iNqhoc?h})oXjs%KY{h2|TQP6Xl~CF3vZ5
z6=MBs1c!<IYKJRB_P;f)9hxb2Hj+7rnFFqlwYu)puNI{5*qjVWwkhKU^{8l4L4<>H
zoW8bV5xF&0C|to=bK^Yv&B$r`m$EvPtc`<q)%!T2JiW8ri$Zjh*^!in<=bvIbxXo-
zlcBkC@-NMYB>N&5zCHMr-TSfn)i5#UR^q{_=xLoi(3PO}`vG!EECXou&2RlqrjtV;
zA%Wx*ZGb`-#1n$hml*PX=kHb2t)`Up8?0C7k_4DV@7RkD-+YX9(f0zZrOs#>?k%2M
zGW5o!#b!ijzgP<7;Y-ZPoQqX@9z->AKKY%Ot8H0mE~sL%JTC5%5wfjpZF!m6EHqi)
z57{2_PNKjLI>g&Edy;O=o1`#HybsJ1$6Io@zH~HZ<4$VQWJK`3Y2UdXE`iR=;i4_j
z%APF$*SzJwtm~BV(sZALwj*Po4X4@Fz(>gP+0L`}Oz{vGH*B8P*BK@>(o!miY~8kT
z0k^5V;lQNn8f5dNh|b7CxIJ&AthD7#h`V#Sz|dP-YtZ-n0>A=!hzO(b<2FAM7tmC0
z8!W#eKF-M0z`0a|oWL#S#H%c3nXNN33En(630869x8(bBMg(U(aXs<jj`^8yUMWUp
zUd*_4Vd~^*gPg){1$ZiB{`K9Z0{7iWd6L9FFg;wzF<;Uk#o7=45=%x^i;T)&#83eJ
zH8RrJq6@<<9G^nZb~#hNJU<83v8}^dP6&P>9(qV-d}l*aeY?v9<c=J;*|>K7<#{S~
zh)B>E5L=$|k|<CxW=hZhMA&EHHH6gfC-ng*fL)+u*hS;mDFYuE1(<Sx&K^p!LsjpO
zRFe^_yRwlcexJ7~_c`o=xvYJ;t#>|L;(P-l#AAy2meR!$o4W3=bBQ4y3VKdpyf<V;
zILn}A%|cT2^i>N|Rhg)6bvC;AKxSFf!@VfTUBYd?iNl3-QgqN7hTGWnDPK+Ifm<LR
z+r_$FlzFP@QmFUY7eQ@Xnpg(*+V9U&IDv=j%vC4*0KNf0(R#q@x;nd!gfQ(d?cb~i
zXR|e>1ULIOYfq7Wlxlb~;d71J$%O7<UF4~oeOkQ!H5)(+*+Imn9Oo?!Xm(1dLD=}J
z%WtjEy6><F0$!A@9mT5n&4%827e+$aW5}B=LzF5-tBBnhJrnoQR~=>#J0nK;3sdXd
ztqnf*+#X~7*hY!@!T@#+6Y&y0GPIs*oIJUunWTWl^He$UjW*0G`%f)^1+e<9Af9@;
zOqks8H4iL-IVVZ{f1G`HTvOTBwo-$DG{qn_AVsB$(hQ+DK@k}hq=SG+2a$v-MT#^L
zq!SQOa8!CHAWedF1VM_S7b((v`}XnP_q!L~J9Foq|NLQcvd=ktuf6tKPr35lU7B9I
z<mKqDL|eYyZ2kRX%?0l#SM#z{b9hFJNe*S+KBPB{QLycY{*c5Toyt<)lki}%F_jg_
z`{rb4-YPIGtP+B=FO6~;4f!zHXx!?++k^;VKC8dUct%s@@jkJQEchGG$cOA3H0%;i
z<P7-u<5qyn_PgJDalG7zpAY05@ET|XW&Byz^rEE^QSU4phG!6((v$tstb~PoqGc`K
z#9|~}A-|Oj^TV35y-I6rLO~?0+U8N#a%V!@xS%z|7id`~c>>PUd<fAx2w3_<RmId6
zk#)>QwneG-Una_{ss{dO`}t?>E$JAs0o|jgkYBD+Jp<Ol)SLf@!e`$L^)x{ov?S(2
z_8Pb1NG+1uhaW3)ndlCSJxh%^UJcof1_wO)!v1ukrqvu@d5~|+MY%nFu>#1Xs`L57
zKq_WSH#^}>my7VIE@j*kfk!D_vO3q`#UuWn_vtQ>+P}Gty_RZzFikuz1!s8OaPHZv
zfJLMNh4AnY$E)5aU2cK2dKNWQMBLl1b^s)02=}KOyvU%z*<f1RXw2qAtHE(Bj#`f0
z(+n^&9TW{>Rux9q!fQq#)zl@T;JAWfHL!fe*2c3}yC4#;67EURX-!~g_*lotiB|H(
z89o)rYHA4`t0X(fkVd1Z1#NJY+x?0Tj(Kz6?LbQGS+-hX!Icw5qslwMgwP98gsH?g
zXfxvT97#Rv<z%e)c4@`GSZXk(AsvnSjBj}J);{?`wy()Mj5q*TJENA)*nO}1moOE&
zO#u*x+f$}|`J_Ufj6wv`!+EIiFopmQTKoI3gIZ;})yUqlzQy-%-P5u|H=UB3QHtG&
zhvv*eA4HC?DX=X+p9C`vRpH(<@hZYiC_?vD)mz?^Z&1T~4QTirIQtu~0334Z`-qFv
zb<U@JMc}3po^xi*$W!5F=(kq^4a{9DbjIZ+gFT=WE*l1^W-$;zgB@?7dr3=#bI>M~
zm?Ay<oswU-^dP$}g+Ho~_M;YiAw?(!er1wIi?^LWWJ>~DnQ`MTJFORsoKa$K2_tJU
zhDRD$0=lnVu!;*-G6-|9)^9cx3J%%DlXTfC>xJa;Aa*}U4^v<88;e=u*I~W`*07E3
zhUi=sa*1?7k;iT!KikbB&ds-6qT#%YL8kg;Njv7G^$X@~3aM($UOMTay{30&wEEQx
zqaIjKZBQ=Ut1qp?^>R_W=vkeT8(QZ7lWY)bl@XJQI+(jJN2TbaCdWN7$!#+Jm`=LW
zc!>y_5qR=NkJ9#r2shO3bxV(oF=8Dt6r9fc+c{7_bNB{`Zlkv<xlNT6L%MYJ`XwyS
zyG@T(0omtXu~<X5^Y5gYV@O#&9tD}Fi-w+LLp|Z@2%#CKvjCF7?3=GvAp^x9EYBA6
z1_2f+5u)@kgv3U)wPV_xN1ZAsK#A#=Malep)}2xr3-xB!4<v6%Js<ggWi|2lqZ8FT
zc}eP=nMFNIHsk8`a{Vdm3c)|m9ydvrWvxujo(n`#F9!<y*5t05aM(2aIazlqGPcpt
z5M3oPf*pq_qMS`9mJFn6k@cKwUZYl`WG^k3U?#Kzg%k*<^8D0@LXKrn>~nA8E?v0N
zqmvGEpEfh~Z6Dl_AJSeQbp?Fj4H7m-LE^a)(u=rpCHB+4i0#bUIF>8fJE`Q-Sr~9~
zJ{9=JAe?JjfyyW7At?3DRoqClEUY^Am259#HQuBE0nVrQ{A;UBZ0LPa+3FvS+ZH4R
z7694l>^9ecqN${Q<v!&2O63%>x5lvsPP0^L<-jFWg$~Eh^q^hwgGm>~cB5sG-t<TI
zo`)t)U}L6b;b56J?T-;d8b5Wg0X6J2BHV>yCZa11V*GbNFNsm{L0{I`V)5y<n=Eyr
zyr!i)092Cqj^Eic@Y_kmCAP0wQ&|@k<`{XyIH$*w(B!#oSIvMuAm7?4c@F<x1m~zk
zz>|dSBq`=*U#4O;_F7&4iCO*?zFzJ^gpHR<E;aGbFl!Dcxd#j`l6boLTl;w)-^qb<
zDK&PdpX&HMoaUv(dpA{6r1AK?azO(_-(P+sHSWE_F-j<Z6^<r&fJvAuGgiHhM__tP
z%GOtTnc6$3<Fj*L<2&zz4eu1!)(4(qq@K~<NTW%&c&ECRRB9C_<4a`zy87b%4(C|x
z@ffECeMSkx*cx!Cvdg@|J8i~)C;D#x+iu>$<|ypyxffUR1xV!B#c%hTr5Dx|h+EBU
zXz{+Xw|@7?=j4bWKGQu%h2_^`Vt3^Kv#eA&))(c)B06g-X2G&E0MZehb<_4$fa{rr
zz|2ElQv1{QYx|K^c_}B~^CCwm-akGo>dp#`<vj5PI?su1(<Cj_6<Mw{#(jMNQWGq_
zf5a}v;<{ANCOR=JqQawNB>iUWl9sP-;zJ=DfA%}|J*PKvcebi)_U`Lp*|z(aPq5d-
z{|IMFC1YZ@5CYc0RU8PM!e*bC`ziKov$|W`oI1sw>mFP~nimW$rUyapJ(F=N&#a|e
za$-22)c{ir4$KAItUg5|#JugPvu9fxzpi!uV}12nx#}HG!vC=!{{O7iXNh9G23J?#
zGb!zVd_*HbM{`pbAGkRW4&8Q|v^7icY3cW%L2`b?V_U)y`05l$eOl(20<)K6Ngpt{
zDr<(cZ&bOeB;F)`MecbO^Td5%Yn2)J=Z1T-WyS=i<}ikQwNi5Mm9UXAqm}Ut0~!bN
z6jFM=sO46`V(jH0`f<sh-sNB}1Dwi>>fE&wpQc72HGwFyQy(MNGw8_-YTE6&+g@m+
z5wuix(lUhT<P^R=0yT<GA<tS0-cuvDlX{EJC8G%$Ndc_^sS%~LrHeQS$q`iUq82ZQ
zP9QDpqTWrpQQb93dP%VA8>y#bN|%7O&yABt?kuf%)8xEUh{CqCE6aTHmhV60B<z)8
zEh<{!A(2BuWrR$&eSrEyO<YC_+X0#wF{x(|-4d;YLDjisprM$n36ECNuw4jN?YMPn
z)o~$o1JJKUi4<n(5lA*I1)Mqui7w=UoswD86BRpOUnfB1ysYx=(c1BmhNTYtR*=S$
z#yZgRk8br*CdhWQpx=V_6{%;&(@US~+q#O(oyrH0Ti5TB;m?ny!?qjkbyHQ#nVf+T
z@6`!CxSQuI*mi@J-_+1C&^B1|+QWPA;;ovAd@URFm3B<QaSWMJ{^9cPzskp|-u$Om
zNhxBF<VcJZK4qs?S%K{Lgqf}~9xsf-HH2Dj>AYW&dqxx0Cq_Vzk%_Q1sOWj+_STBk
zvS7v|li9k<aR&(1CQ-c`_d4(@VF0&{95U2ZVfkd^W1EKh$rO3}?o|7&)2mAw1E|!_
zB{#s$eSD&IGZ!*a)x?lJViX;_xc=-e&a`RHa$E%YRY}E+^|)x{5{p?3PNE9XmoM*{
zL<N<A)9D-*BKn+5xBuJ{EDSNdch(DzCzh|2Uq7a%gCu@oKikI$sErF^kgb5x6l|pK
z;jc!le(&qUDeln?ra=9erZI%djP~i+y0Gd!vJ@07RuPJ#RJ>a>#aqioa{p4pqTMOS
z&<oRJIR0mKi7lY`!P>bdu)?tn8t1YAA*zF0KlVw4j5Vi(;O1G&xgID2kKZg1uaanZ
zvH!b3@i)_JDZkyg${Ba2YVqk-q0elc+jL!57bw6Ca6B+s+!K~3@~2qMGj#cSTl>os
z4xg%T5R=w(L&3Wr2)1{eVT&XtVDb_0&2e6tzJaHFAF$K(a>uaMZGg~sev8(pL$G4*
z(nOLP#rC3cz!c}Qv%o5aoY5dv)YyxkwD;=0VuEPtaxEP1vE-1y?FGA0bM=~V6FcWA
z{~VK|)SPC@Q#N!w9Jy~70!tTVKa-?DQpbwl^{RmDmmn5yM$Yi5C+c^|dxD=KNEsd@
zBFM3zfb^XbP<&oA?dyQ{9+%X<dw&dbzuliDeJB|hHbTK4BKO1H;>C9x82YoHBnw`J
zZncPl766TNZ}lt}S>&^Ak^6L&#whXGI%<`43Wb*3nBh>=!Z?i;J41!dm#&KFkI#f*
zvsrpeZ_Z#jpx)Mt4GPiF)Ch;NY>TT@J{0m&JoXmyR?9>&q@FfWD4x>R+6-DS@fC1k
zTH>AJh69k^`Y4@f3cCp{Nu=hC>zR#~I8qxOIU@^gu-9uGBTP2r^>>YHAXmT2+s<I2
z%Dm@i`D)?<8(VX%lS%@o_tc{BYn*G?fUN2^<dzq-u!4DCm*|WU_m}0XrP5}HWW1pc
zD}Cl=AV&W^I`8x#%|f~`VJZjHYHeCj$joOQdSLbLE=5r4saLDQ3&V!>#CLqhY`Pg7
znVK&1^L@14lBpfhiYxbhY@7(vM*kEmCU&I`@)XDc-g6-q)a|tPF);=`x7(?-H9PE3
zz?AJ;w{4V#1Ba%C4>_uzv8XkJ8kL<IahW6ZP4<-TSLNgoRrXGF@ejf*UGaJx^7qsT
z^n<Z>7u~Q`{4F`7=MLu&&wb-a+BkL43CePCH)3Wr3tHdIYE$VJwlKH~%%k;WI(kRh
zy@Y#Nk@a=>_YgY3gX-By3a=Jm-P338El?<L+^CYj`^x?csEyXm$=Qn9bq1^9$ym)j
zRC)N5;Xzyw=}z{FdD(BIo==9;FC@nnMSh$P9WitF2&#mt&noR&Dy+B3g_4UfGPC3O
zeh75%WkhA_w_>o=t1`GzF%xF6>swi&*EtRAKX~wEhb=$(79Reb&-#(?iq^~cYd`eu
zMWwzsYWHnsVP!5&ndt}8Y@-{;(?<cFUF_42$;oy%gExKRpPjDnCj5C2h~lbFeYDL#
z9btBipLmN&KUCraqbSui(kOaEKssp!%#*Lei*YqmYxTZ<ZANT(h!Z*qD#d(1#=Y2=
zp9fMUQ*En`G^9>KMH8jV^m|eYDwi~gE~T=|krR*o@SfOm)C3{f8)<odd>f)@es)LO
zJwuI2s#jN@k)M_|oQ$Zt)<n#iC1y6NjugN3m=<-@&{%vZ)akmE?iZOxWB2XCsV~EB
zV5Yr%)UlwpC(9PHKdknKREuqmUj1jH<&{`W#Vmmb(fy;mM`GuEEOaP59|e;=&ro0<
zjToKUTC{<4J%RHwj|n!)z+zT5pR-sbxy(2;P~~Rq_Kc1@jv6*iaZXP3<!9h>fE}np
z?A10z`;y94Usrj7Z3C<g#`01Xk)wBEB5k8nL_x9&uYslmd!hM*u0=w+*SI0%OaXNH
z`R1k+nAo{1DX+{eKlEEcWeQwJz5Y;tj=0Z#a(PlR*1V|dE~)1gx)N@vWBIS94Zi~f
zDo{YKeTl7l^o+MsR#k;WR9Je^hlZb3zvV=?dCOcKzrTsc10{Wbhpc8uLuOekyf1X~
zX0HWg0HtPz60?$H@10qujqQ@#2CI2t${=}G9l#5KaV-E<C%4B%AX~oG2WHA4=}0+M
zDa{yBik1ce!Y{**Nm(_Tc_i70){5-tJ+inC(yA*TNZ&=WHT93NZIpf~tDk#dFv5vc
zCy7jovXuA^G;`bv+$VBHb?q5eujlKxp9{nOdC~iyS(Y)1A+GSIC2FX`D(VSI0GF|}
z2n(-*nsF7~%?|M)Y|xL$!|9kX540mk{-)>#w1if4-l6~jEdlEHU1N~}F1BY$lx?v;
z-;mh#<$>a~IG`h2XR^dof)4NoZq{Jo;SHI+)$_e6pTl4CPy5}CwJb5w1*S>tHjqcp
z`5GA@opU7!W%;m0W9t=-XBO<hi6tuNOy}%KAUjq_bX_W~3NjC>M910EM+6-#3mtqJ
z5dYo;$_YJCs;5++Q1=tSYd;u@@!rn<ukGA_zUf>zjIUdlO0(1bdG&n?9ZijxoIG{o
zTO_AeuRnXgbhL?g18U;oZ4E&OtrcihbpMdErF|y&>UNUfR3!vl|3rLmj@hCNxFTQ4
zasNJh&krhc3wreET3&&C3vF&x40^1zCgPEkBCS9%YVgeCx0KsY9KXBto?iM63_u8Y
ziw-iv4(%N~AhwcRa|4j+Cjq^79!YRH*6sm89}~ac(fe?v@!Y61V2iyO59<jTs$dX$
z4pn;UFt(tT&DjEc40~9B$yQ!pnFoNlf58D2Cu_*D@fE43qFAug_jpyBf1Buk{7;=!
zl2bGM9pskz+`@q*?|CO_i(_9@^V5U55h5IR?Qq^+?ZJ_Y1id(MB35J&!l4{Uevqf3
zfkKe3qAcj<c&>B=q8=H4lL!hd2HHVR_v}5vnb^8u=VyiMv-d$XIhX}B{wn88zM?8{
z^ue#qzY{OV(Rl$H<%2uGR$2;ezXQ2I(5XP8JU`~&-KE7M<Oe}iO@LZ-a#S(YTEnGS
zT4xV~Sr|FsiV1D762L(u0qU}N?ktn}wm}$O%-xkt@9oT1s3=h7i5UZ%tS4Pxq<#L9
z4cpE){G@^BETq<x7<EJHZBYM=v$HJZRF7>{2LfmTYd=CG64}AW5`agVd3kDU7k0hS
z0@UCh0zB*Nq~`v(CRsqrJQ~u>nFO^5TL`l|*>o{y4kxy|G30$P32NAe&M7;CnJ@%h
zY)=pOG#3lewWjX?j?oL;=&4)1_ouvLy6BQd=N+K72OBt2#Cn4CSH&5CxnkBfRj`N!
zs{B6wUcc<I|ANasR73q0*jAoC@vYigKBp0Qc;oTW<%GH(k-HEd9AFppwd-ZXzdzFU
zOQn$O1RhlVNKpQ_@6j;}8}<1au|FL#8XXx^?+Gdlm!XjE3^sr(H0WyL1c;EW3uyDq
z{=R<xE?fo=h<xZ25i^jbqx$v&Q29752+P<9%sE^c0BBzIc_-M$N4LH=OfK^v8-Qy#
zIVu4rCh_(i-PaJb;w*CNLqVHV>xql+g_m0y+QHTJmIc~pLFmMMHj6;lb6wl#He6oJ
z5*t|J3{va}><C|_m)W?^G#s%U(+nS>oAv~aH(TKUmJ;pm)}7Aw6$z{?`8Tf2>~)Vo
z$N}uIP<R-pyg}oH)X)Exksmpjoa68@f$>c<2fQ#j@|^iOh??~}N}k+||BXU**pLV^
z&0C^^z=j(leZZ|Uq~udo@>jB26%x-P(1NjQ1sxKjg~xw>20pYe{aUWoFL+W@?bT$w
z><tgCLY+_$xC9z9w?A&I${ei7v?`r3=!VK*2NlkYJrK<DG>vNk?Mp3wSt7_ds{p9!
z-B8;CEd7DquSR=cqxUf6*iAs9evf<i-Ux8d-I9~!Ke7I=+yCDq`nP?e`W7!3+}o*R
z3r2?r_XrwXxk|sQ0yx}YpryYtpBNSfIS%$cNDs5%x-bSp4q^3;_!Y<^H#1{jdI6&e
z`JmtEI{@Zeze@t8i?ho&Y$1qiBE%nFvpc$N|8X??9~?i1=)`u62P5sjjTrnh_ZxVj
zdN!Jc2VSRE*H#>?*I~d)&dBPd)O2bOjC~~ze>u^<3bQnd231CJNOOr9x%k~}_JejU
z)U|2_nwWjaQM!mYr<VWY<9b38BXykAQ(c-IeQ@YOSEq4-=h!LzrHZ*6JgdZa+2z06
z582ote>z53{{;A^cIDq1@?<F>Nf6upp9}lhTiuzcD?O~xK^Pcc+lby_!Ses<t2+D)
zenGbKKzJ|FNqN7)edwF~y3k{rOW$bV2C`bfgDO)J+WP^W1~Xo|*ysw5(PCgx%{NZD
z`5ulhhc5G<UXRafd0DxkTbM6|%tPPIy$`*TOA-CdFRSZe`ar*{6_Wm&6iwM`DeTp;
zdi^p1<o%g_pir~6EFmC3{GTrVC%>+q3<Bb7EHTde-}63I<4%zLluWa|6+h=Qp*##!
z6Kx>rmPw#dDd-J&-Fhn^VwFOHE)go!>H;i4B1Dddf?CDkm+7ZR63PEMD*kie(K3GN
zy7b}D>j(K52YCN5Rk9;V*x@|&4=>;B18#5P*mw0KNle^1Fx#RstOt+$+L1{12iG;8
z1dF&eDcSPqRb!#QH{(sy{@Qi^m1zI^A=&iQc8w!9W*iB<h(@-olE0M89=T$58Uk89
z$ZPhBhoaE)YIZQKM0x+<{*GMb$O7@%QUg<U*F*5l(d;Lbf^RxeL`pOI$T$9E{{7qY
zXW#~7O+$z29~tT(1IV>DS}M~HEsP`2=?{kt4*)a3gpW!2$W@6r!LJ>c=8wAY*GBhm
z1Eqgn{Q~;7nzWt$okN$^IRQoYr<rroM`p&K<IDaz_*`1>{Pcn}TQ930z8$zMc;Lq*
z*k1gt=h_?ztpV7O`jM-~kw9x8^<mPXTl=d&VeA1GY=h+ksw3Z?JPoe8GUFHrKNL_7
z-_RdFjt&s+a{_BmANg{KAGn84ODvU?NB-l;um9(D$H)%G6tWQSrgG@x)%3i=cYUi;
z_U0_rlL||O{?0@&<<0GX_^x$E;KFp`?IdNM!`Cq00&}-yH^t}tAK&pGPh?|9+M6A&
z3(mXCT`A=f*eqAXk*mj0w7g8UkBPE<bm+eR@+<6Q=uj9+^R6eP9$H$t-O!%%P;+qS
z<zKzjzr9}LS76j7H?h$_j)VbWDCR2oUWz;PV}E@6|8`CA&p0zM-zMRuvEYpU?{8Vy
z3jCet`nS#h>yGm)_EoolTTNsVZ$GlMgzceFRua+RuXcDCf9XBu)<X+<l&N~)Fq--g
z=HcNR`ojmpe-hdv@y@G)|9=1A<E!8vW?wld9STK%X`tZCEf=BZ`gKfL<j5RqfD%}b
zfQCOD&!27{JaQmpoy$6VIGf?{nNS6Pck*9&{r6tiF)&!N>ZRw0rT~Y}Q|PMl9uME>
z{*7ng^BM}Tx6n<5^h1};&48A{t%!ldzqJgGLmT%KwivY|-xj6<%b@P&rO-n^_E(>B
zoICWouw{3TE-o$bcb;dxjeqaC{=v>IECVfrl=e@*{hejN0d92iaLAG+3JC>~Txngl
zlJ##za%w1&Pinbw9ht5B;2ow&%KeSk50-!?^h~FmffeoHRb!z6CPru{=I>=5aeQE`
z44<sWk6d*U3!Xrk<<HZ9?^Owd6oXpfv+$9tGSY%yqntK@&fNd^kQH+Zd|Q3S(Y@f%
zWvg#-fFN)lX}+NKH=gVtF3alwyo`@dl|w`MSCG^RhmtInP2l-`=o8iFpjb|Fd+f;4
z_=`(x3`#Rm3TB6=z&YU@=&E-k>gxabSSORJp`*7`D=0$Rw&l!|La#AYzIyxNse1uO
z*7|?^X+}zD4o|4Lu^$@#6Ox}G0d9Z)&+q8;lbmYgpPyRzu5OH=%dNVns@YulSF*|i
zB`|4iA}_NYnFlNWVA^4pIWz+P5qOLx>27iqBvrfX0*=ysqMo~4Tq`6;zH?-NzxEwt
zI%t1~v89z9xjPQ<0AsC6{|Ob+af8NRY59{?-S`zf7=M;#+oK~H=#fYLkL!(r%Zs0s
zZa?xUg=wHr6iD;w+&@Fn<nt#)@sj@YuR$Ry52j-j_Ow0fFRUmZ(re)P5P3TA9H}?S
zo&m4L;Qru0(?FfG(D=iq<_xYGi_n7c|0<9FYZm^;u|v;G89K)NW*TsK>pv&lPQmXY
zHw1daT^_*6oqhJaG*nV#TMAMMJ&nI5xiz#uKa>DL{&j&7dtxSX;=KAqF|Sw3T|<od
z3gGajl>rQCuT62E1;a<et<qyIbx+S2ej7xOh5hP-y$7JlR;ZVI%I>?nzAqSj%br3B
z0qZ_#=;#Q>pD-}BX>_n=)Cwx>hJxjwSbGJ4nYRads^9&U*beHZDA6ND4Id?C<Ej(C
z*!C95E<IK~pdS}Ze~@ypm0|%*9^kWQfvbbkN<uHr;=M#n(`N{0l>n*aOgyFm1J$jV
z2TlVO!N&;~b8t>|L$~tP91X5ftyEq@?`@7nK>gOC@$aoRlYLb|XTCMu60#pd3i_8o
zO{9=b#>i=pyeyf!z#(AkMTSz!$=P|OgCJfvzh9+mP=xuQ@25t8=bqK;U_*P9lAY(3
z5>3C)5-3o!iVevII_j-ZcfN$;L0Lc0kYAbnc(C)aITEVga{E`D>$outs^c!s04NmW
zS6CNErF10gJTmoVi727M^3&xEjiU(<f}%wQ(A(xu0w?Lw&$a;cVhy@icxsGzI=Oa-
zxKx2-TU_*$5m00P)fswGs3JY1)e0Qcn;c)14Ywtq>Ln#NjiCo~#k&+e`7T|%V}qPi
zy%P6z%QSi(%%*}MTBz+d31|&06Wzf9w>d<$a33P|>`P1?r=F8Q5OXm|SV(;uLc3!C
zAYXVqo>C@m?du?_Zi{dFDnL&r0#E1kClae23W+;F6UUkI(>nnjA|SNcn+R#gXB-IF
zW7I+Q4{1?D2%$JKv5!0o;~!Aovx+TqJzRC;cOqKR>L7O_v#MH2Ua+w%9SN`iihK6h
z&3fJ&<7!}S!+P3OBS1loY{AldRmHno_?T<4&0EUQQvoe^Zfq+cq29<@9J>kBNGbTt
zlXw{x#=iK3d>2tofAF!pc|an<NNz}Cm9e+1XCP!2)3)}VZ`%5-^s~0>+2BEoCsj5L
zLZJox<Jq65UQtld<dS~a?IZh(6PK_Nspqk+uQ^_~6+L^6ygmJ3ra;a%+2sb{o7g8X
z+>kK2hT9l))&zIxk!gS62lU)~ba&YgHu2)^z({7hwp^ryB&NJt)ueH5vgG4V#YgKb
z;{DBtgHp-y=L#0$%SyBZzvj6-spnXSp8nK5d!g(p$zIp=R86@d{Rv4@PiLk6*%nd|
zs+{Hetp9wc7d{W=gi1Z=BVm!BmuN+|gPa~`Dl)w{^Df73zx`lKrd1TX6e@{W8#>q<
zQc{c)T0i5xbfy(DI>5biVMUArEhXJ>$mfRI8|D{q<>%J=fW1BBEHVk%edK1Y)go@y
z*C95li`Z9F$-^G)87i(APe9Ej-~_Xk(hB(>{7k>7CVCqF9f+WE|CHH^lKJt>ZP*LD
zw&1l<L6$egF(x(GT*tPaD_8S|r2>?^o1Dj)P@vnK<b-_O5+m$CJ5sa^x}p!90i;MM
zjomHyZpAMJQ7#ZhN1r-QGsKeOpeixso1pIsDM$STF(=%~jY7R_Yym8x&CsSB`obih
z!kdX7w-UceJ8Xf$;l?{1aho+ME94U@KO=C8*aroCqQWfP0_lq>Rv2sbg^Ud?<+tAC
zBFTc($k;&SIO0Q1mVH^ng?I12xZwzr401zRy;(VuE$dKXKyM?6G>q&``<a~m^L-cU
z8tOxPnC%STOjkDYxcwIuPw6PPpk4XzB3W%b0_oiYTG{xw0|9T1e6<V-F1-e?1sdE<
zeoMC*eko2E5w+@~a%*4S58JrtV%RIn8vRr4>s!BUNoITlfRkYZS#9gp;I=ohpEUaE
z>M|?^XD>FjY78hPsq1-^i|O}DYY##6cYfQ1Y~gWliHLi#^h3F}#aaN)d7eipPA!4c
zppg9SLit=HR|Nk@7m0BELVUM?furmE)J9{<{*RP4KBI`{jEmQ!zG-5)7b5MKvQW=j
zmEE6~2YYDlGZmji2(1jnyIGdIU6&i;)9lXuPwB$Ha?uuXD9s*V5RpFexrTGXL!_RA
zY<?#qP}|d!03hpC0DvfBV2`MabCurvk$z%(e{blZatlab_d#uL^F_NVB0tI$(95|3
z-nB~s!1_-qRJxGKLh;THez}=rK9^eTTWTM8q=^uJWD6h|PE{mAh6MlujqMr;=!80s
zN#6Tgt(u@eM%dTEj54HrvniX1Y1PIqM}NB=qdyDWYJLoqfZiPqi{ii;bbT1abD^%~
zB@rWMeROkS4uZG0f}iLeH~_SpbhPSy2MhSrF!d6*5Jt`Z$q&$lzVkD-CS})z3Ke!d
z6(*aqznZCX${f?Fbmqz#eFr%@FT{&#TGR*7`uo(y*=z$q43Fz1zd?pyWU`Pw0)IQj
z(u5gkxc(wj2CslVmcYoj$#{>ocSZu7SIET7GQy5G<$yL#NcF2ZjRW}ernSYBYHTeH
zWkE=2ZHO05Tbwdc4UbxbXms2Q_j~3n9aj6Ta}1VsJ*9E?0v=32&J2kWz}bO2!l@n5
zQrUnGa?GWHD;)`zzce1omKvBtR0Z11)ykk2RRU0RLM;gT)_~<yXU@JApYPDXWEt(D
zBza<N8$cqQ%L2cNKfnN*XUvpRoCiAu;%c@U1p1Q`M(b(A@m`>LVYNRdv!5=vNROd7
zPFqWldyE48oY|)lFQGozAQ00458|0lyhoj^<ClZ9GnHAtS`8i%n(qBw36OaP?CvOA
zSzF3pU&<AbM7RehtZMIL!Xp_=A+X=;HXG()<b{Ly=#tU}HNm$l`&%goW%{F}LTEX<
zA&{!*PV{k{sD|3Pw+>J{eeHs?=9b&8-C8OyrF^!1E%7@6`Etn`EWF8&ZV6p(pMj(^
zgAc$n6WIBj(@omlmz1J32;=yi4&Nf>Lbfxx&uFJeai>JMh0%Ay4|oKgBaFO8D2@2G
zq-SPS1<j9>8TjsLt(;%Fg`e7v+m7CbN6L2Nk$!*+U)l%sZPaf;V-WkrFvElwE(s!R
z>3G%w^l{=BMkYUa!+6GGl%|u6ypb3tC%GXKfze3n*Sm4oUOW7)*#o1Scz=XbD#{Je
zp!Alt<}4*O_T=4D#R$?Ff!4R~V1eLvtW=;I)(pNzK@`CYFe<O5weWNUxR1sW&vV4S
zRDm|xWtX!7p6&GPnBet}2(Q(2g1byMu`@vr61HP-5y#Uywkai_{Si0*qCBveXP$<T
z4A=kc2cI8@kk+qR<)nikO9Jkp75%J%O+Pw@+bqB;fV;&a(YEKNOfl1Z{R6-Kp@SW7
zy@8-^XP7ykLMJvAH7?!gLMz`IbG^JOL9HUlJ<*E!q~qB%IUWV{*LQ}uo7mP23yIIW
z2Q^@ZR^D%2m-T!aC)Asnq{(brJYZ7?+g25_N(&FWtmXY>@-?-ByC5^6cdYVyRk|S0
zM+g!uxD~iTv?a#8FiHaeHs*Kru7*kNUCF^*cLvysm%`oTI}0DZGldg(Ml1b0v1zvZ
z5Qn)8Fdmz#Ag!Y4(=$8z<k}?;M!=Ly+iKxuwZMV0`4{?6N%Q5v@aNjr>oEGY6KAFj
z>K3jOoXZTN7{t2Po0jnGM*Vkpi_;0CtlL0?DQ8z{cQ(7i&Z?Q=jTn~VV<pC0F`3o`
zjqhGacrdkO7*dQde~JrOV$QK%4hZ_MEqEOQboO1~9bP-Kr?*`4dCD;zqK5)TDk0Ur
zHA@h=&U8aD2B4DRRL{Oyz|9<{p{8L~Di#0zMJt%}_DR73<0lvV(wK}{8it!_&xpX{
zR+va6lO5Si3l>Wkfsw+8EeI%5)dk^ZZj6{1RWsQ<qJTpi@+mn+EzYd1o|3AsOjiwk
z;g5nNd53$=VIsfGO6sTzCk>g52h6DwEQHqp1nn#3G8AOYWreQN1|?%A<11^L<^Uwh
zZw`PF<sYApWFr1q5h|AlS&i%<vk;d?y!sVsCE31g0(IkJ`|8JMdOZvWq=;jMz=SMT
zsPxp&VXHo9bFC`ibyMJrA(z^?fpdN8laDUGPwOT|rs@qAWK+exUb+m#Lu#`QJF00f
z0Didb{}N;D>L0?L_7p5Jf0N*#=#PSCsk@M*yc1mOLT$X<#`S6lG9X0M!e>9X6X?Ae
z$hnQ6;Xo-gCWh5)>R<dx=Xk>Jox*QXb@?~;75t_6=3&Rzfv{PXwq>n%^m31D9)&&d
z@R4~ySix>Yzq@&t^A6Mfpqq5MmUYDZjJ`5h`(oNvtp>Wnlb7oC#$PTzXn3*$JPPtw
zq5**HcXVplG0`&Rq^n)#Q~^0pIYh&RQdC&4v$C6Zd+_II-#DMsXYoQ)Ql=LlT_?Ao
zQg6+|zG^4SBHKqb#`+Y_q(B>9`tUg-OGW>=?aj)tn^WUant}xO6-i8T`Ab)D$GIjm
zvG+9aWrY|5(29<IKufuTIx%8apBxK;i|IZtdNr<p*7S1Xn1xk<C2-nD)rMvV%PoW;
zqNur<WF%ywAW1_LvQ;VaR;uVmT}Ck-4=3U0B|-^o+G*z}kRWkC@28ukpsSFfphq$m
z-+{6b^}ko>DUAFU0FSrDlGbYPms5$b6YNyhZVjFuIyd_+zBr`8s_yAs<7EL(2}MWS
z21lMEWBSnSWwilcT#B@R_<%*?jSdw9iLCE!l^l;OYoF(Wvts^MtoK%Iz8XrAHeul7
zptZoX0uB;~S3ad3)2)J@ZQf|MZ*Vh>kcf1ZG`$=;WC0yRtp*Twb*jW{@)cA)(Oms+
zi(38FSn>L{DL}WZ6*8sqiA^`CThRgP0o%1<U4|HN`qcN@L2PCupIzv#q8zNCTf4_H
zR@U(|MD4Vxcjb7Alg1N;GFYA?Yq%}J2N7y_^(X4Tluv}Au4^KVX3gazi94`Epi~2x
zI`3%(_6x-EOi4_m3A**2ArFBX9d>~BLU#Z<*k^p?I0rkg2OI4c(BXbZb-&f*jUG_)
zP}C~o3z`*eWFOJ8b*N*Yj3M_>DV-3Q4tX>@5$dx6i-&MUD3Qiy!}a{pwWr6&(HBNA
z-vM4=;y1F@fad`o^tG&H?<Rdxg4Su}MH~|3Y7t^1yZmv@^29i&`%7>?%n~jB$p}A3
zusa8`mR!*ut=mNGCl0V$qmi7oULa}WV+%A*+`*f{h)0$(;-rNFf`ox?RDfcfn;M}J
z6FV!+7muUR&OU=Bs8SM})GZtT_=X>LjB{$wI3srr?i)K7Nhfp%rdeAXLfOt-MK#y<
z9x}ZvJRv&jfn}QFWHjawBgEA%@;wcV`4Ng_^b3Yaz=n(P!Di=rF2=1$xsSTWfyW{&
zl@iKOTOw3Bm(jPWC~K%WW%id-JtUr&Fe$S6R!KzQ8l+#aA-|2ghYBP8pf^6j<RGIv
z_iTe<nMF){KRvocwM(DBAXO+^#p{jD3tp3_4p(9i%CR6$5saO5)7d8aL>F0vf=0Fi
zXRFV43QrJ=z_C~BN>LO3me=}d@`Tvl*UArGN5UL(;|;k+lYTh8AYxi}s@K$gjg^xK
z;$XBl)N+k_Og0oKUJ*a3J!<tX;%y+Ns@pM0tzQuxJ&_d%i<))!=geSl_N~e{>yrt0
zG(wlA@29hcZk#Y#6uX9{n*F`xt?OGe+F7$`&o9@pX@oc76l2(|A($Vv_Q&T#55X5b
zLS4_hFk986-^q@p`)#cA!W`fp+ofcMxKUvjQw7ruX%L12JxuG;iSadwt4lca!cCz7
ziNMQ<YN{n5>Y?OjS#TpvV#+jFHcsl969Ze!5+mc<RTqkOiIN3A?QR8=acu%c^e4bP
zEu+5Xspbx=VYj{5q}E+AZq(FagRXwIeZQErajfvy!cG;!(x}6i@jD?n1Llt+4=9)S
zU$w@NenyL$5k1Rw9I){Nlp=*s1^0BK)taOjUXDv~QDO>GbL{lG#L@`W1BPN)8W&P!
zR_1x$ey=2j{V{{^%#U#-Pgm>DaYzdGJ3u9YCc!o(hZEZMyJg|r#}*uC<VJn3Yzg(_
zSuATwNUTgm&FQDwC+Hm=&_WaCu;HcOKUa{-NweLp+nkT{c8?e^1e}ce_7_7X-9M1l
zGv%ACtqc7Ss!Ez>4ox;C9iepqasw2{$0fmM<&k3$C(dY@f#?>&1=pO08aMoGjng#T
z{t~;shnmvAez;)pzlw!_astZj0I<=YDyZeiU#Lll)`N=djN{wBVOc$db^qB{5+U~m
z4QqVdB7$F~etf@rIT2vQE(Ik!Q1R6#Y7`UGAal`!X6)DucxqgjTsl>u#UiI4dZ`}B
ztE`G}Tp{D37j)N^qjdyH?i%I*Vx$hv6||$cn~=j-v$ZG<Ob=}MSB!Tdo5(ZVwU@~3
z1Bx)B+Yn)iP~CP<^~p^R+_@fnP*hFXh+~M{F4d;_F-v$FCGvsNS-1w|CsWSSChrGp
zA4cOcgwYc38J`Q&CQ8UZkSj$tmNJ%29PdS^zE?Vfk-bv;0q(DfGtigpG4FGS`57<O
z#)chlhCMLraAGVNFLXMY_8G{`AyN#RNDVz}7d;r$UL(K7`+S4+wm5ZkFRAW;^XeQm
zNxNy`y}08lU)oaFg-J;I22>eIl;h-@;ri_yc4Qb!h3*DplhEVUR0-{D$JDM8hZW$j
zlZeA8^JoX7!^{uhs{4?D6*@Z@cWa2N-5nGx`H0_-;mTXOi8m<%H~IZvYGRtS&mtpD
z#ule`e>5?hJWLOzUJkt#*ggQ0HU4g1luEOtZPA9IB4nPFfLI`xqfM5TbYR=2i4wF7
zH^W7~Xfg#!5w)j`H(UR<r~E%!0B8$;IgR?O2%Paq89CiuF~UWh(c-zr3XTCA11`Me
zoD}9{qOA{=<E1#1zkc*uzxrLqo7Lt3VLMuww`^s=&ZXakMGg3u!R+aV0}zvS#5Xfo
z7d}7-)8-LCd@_)e_8;a7U7-^V;5goSVxc!T+l@KrBSJZqRTdX?v6io^yA>hU;xa3|
zrd}Tv3$*_TjzL%R5d`^hI%Pdhl6$;DD}3;@v^iGs+FSe3fxO4HNqC`;e^C+b7Zgw~
z!t7#S80eHN@k8oLCZY%9+E5VTStsa>nvo)}nTB*=o#HO};d^at=daG3L%xnQ67i4e
zjA^vRv(U^Mux)3paB6D1atvm&vMBexqGp!2=%*Ne=V*It?W;{uh_Y>+QX;V+PW!>t
zhKfK>Ndp^gp}EQpLS^vDZyS)x0jyScipc?KGbpBK=WU_+y9?`Q_sIMdfy|4&KDr%e
zM(94(*v0K?GK<s?mw8^(4=T?xszdiqN=5ILoou(=-$ik<EL@y1o=Fm3W;b-lSeoI^
zpk<85gwN9z?xwQ^7o(d;+6;nvXQeKY^Xx?rWo7pF^Y427l1c){hooI`=J}U^Sb5`1
zt>Q(K!pyhYoSub!@{5`S$Qd71?Iiq*V7ppMiCX8GO5vIG#nrg6(6Rp_M)JJc1H!SQ
zWLUe7qDRO`7=hT89GdDkLxL}g?Cj{8kI$mXTB;nG=-50kyxaR~bOqLv3q(AM(c#a=
zsA~7T4;BvAsF8a>K(}|QH8dA7ZT5tB$$Y>xM4-udm^Oifzlm*^mhq`@o^AEUIdbvG
z6!3hBU>}?<;qqd@CeYq^J9xp+J3lt)2Z^%oxB669#jv6!7B%*Es>svC>P)MKl+=Le
z@Xi95cV=rYtiVX!{gXu=polExh)hm`8m1FW^EP7cz4Vz+WmH({(IBoS3$|0O;ey$Y
zb^|YtZ;xR7<hp~n9@Zt25>ca{M=&{rOw^3n`r?8z)vM#kH7td$IkKrYVbIr_eP7;-
z%LBDMalm#?y~E6won~_h?tbdkEvoFPEps{ci7+#fTsI(2^5f;i_BK)zd$HlzePo15
z(eN7q+coJ+!ZIP}g2-!VYS}}5CNrX@q6H{7@;h8u_UmN~FwIs$E{dvXv&3M8a*N^h
zSqY#V%lZD^<p#-U+nEyD39SZ&(0D*3-Qr8s3Um^bm#rU%<LSP1<0O?m)!AxjcR3fE
z*Qz-{pUog$-=p4JMA$3Z>CzlGI#Vg$ilKZFY3d+t&Rt(KHr@p$%K`^W);v^CbHnD8
z+Bccn>ShFR#NdL=MPLKEyPqG!YvpFt`zhgusFd}>%$IA~ZdYg6iN0b}nu446wZzFp
zXC{DZdk)LlH(tRK%Rm})L@k#wU3pW`QJN4?_l{$NpVD|#zsUlv+`^e?LMT9&QAKDJ
z#r6lWM`*!<a{0f+KT1B^+pbRNL$tK)1nOq8_`80~UIbOOMy`=O;Zc^m9f4WF6k@sA
z#at|L)jpQ8hTIX8<yXF3YH|Y05>yn&ibtH{Gd<Z23;opW4)SPxWDGy&o6`-XQgB&(
zv6!V_HP2&<x_R?siLRX=#5)kel3L5LN7;0LjO`zN<_UIre+ahh#&&h7Cv2+GMx&S>
z8}14~^UH=)Xs?CmnpS#6*ZH2$8n-FHL(q0CJoQ}`yZf*9F8$tVS5IsZJ90eYvqmnm
zPo0(Rxa7QAD6Ztko5Z$jiF}mX=!4~?hi3F)P!Z2uoPnr*haqP~M2+NQ*i11cKCQLE
z{i>HML!@t2LjHe|ANdmqDzrOOT|feQYjf&H{^_%$cT`T)4zHUaSE-+!iXehbnmS<o
zTYamLagk%T1iEh#X#bi46U4Oj)dm#V_9(ZXPZByDhPac&u8vgmk4^a&nEL*D7Cj!X
z3oXD@8X*xX#t#BC4;q-6{Uqz&Fu+GX(VCF@IW3KMn(Hbu{%Dl<yEpuD>@yKPi3O~+
zpP#ZdKem=O1>u|Zl$J_^n-%Is1Urd=WU9xc+8N%?N=NSJrpta`w__tew(`AmwJ?S<
zQY8o@Px7f?n5NnAWFHSd+afJ-0PKx=M=y6b0=vsk*Y)%+P55<7k<uzJg6^qX?C-`D
z`I*>GID57F0+=@k$?pP2yeWxw2$r2Gz|qKmXZ>U!IP!KTf#W>42IFd2T}tDdi}n+~
zUq3pMc#b!hYm84907+Cr5gyMXMl;T1bTifz5WlK-`$d`F14c?tOs;KhoYOGD8)@eM
z4NmxxFWw>NTh&+IQ27~dC34<}D!8%sMDh8M?xpe1a$+}7t!pCf`K=e>@~-|DbYh=S
zYl+q}yjSlCRAq~JZlKJR!Ef4EaeH9r`D0>V$@l7mh9?DgVu!fVBJ%a38hi9pQ|3VP
zZ9mht*8NSZ(^fmtBSL4;4Rm|Yl+zr@gzRW1o&xFc@|imX@=im<{=NHtn@Lkob1A^=
zzMgg?%)LqdOX7k!d!ij{#ET;1cPPtKAZCmY5`sz_POmBroEo9sDr?T?3&dSfz2ALW
zg&^~G`(e`KP6^x62hI)=w(Y@tPytq#$_5Xz6yK`&!aktdhvo>gT$(XABt7qJ8^b%!
zA;g&q)N^eKdIMnD0Idn9-7@T`#|&M69;oy448uTz50To@qhV%imogE^gz&nIaG7Gu
z3)2eYV$;%z11z@l*isbd93)J6ArXE5^?Y)BvQ;AA0a*aEi?=P#ev{Q_gj9s*x0)4R
z>_rhQBFb-A0LQrg^4QmpgD2<cIqznc9^s?@tAhW}Q|lxI!daQrilaSp-YrltMgX?6
z#Reotiq@T8CK9>&Tf%6cHBd@{I5jxUy!*0?*TuhgubjM-c^<4Oha71lk!nvMc#)-f
ze)Jp}M*vW&N{p?9ooSZYDUi8nNR8cUv1Fv7&-2cXwIw3^@|@Ju56OyXsmoX4EH0?#
zpp2D>dHTeL#v10g_$C@O@C|)}L@8BA2aB!EX_Rb<OXwbPkd*@|FJ{G?Xs@@@&MiQH
zpwOI#Tk=DF#F=`?4T*cGoA3yzyeYbE?c%V~U%wKjONGb`KtuSz{k%98{3MO0WVj#@
zD|{a2T(I$B5f(MZQpn~vYi**ozJGbTzdpHAgj3n9LpX+1+<-s<G$q%*-d(_S^V1FP
z2)PG-u6@bEL-Qb~nRLXWHf4qErFeLI5!k@m&x+W^n$x(AzeC5~mt!|Hc&H`1wFxyp
zf@56j?-)D=MfFp17w~j!qK!zK=c)F^*>t^U&6(0dO~~pZ=!Hz2Q7u!oapf$>FB5&t
zwr)R{GPw<JN20slQ%BN`n(+6ENa|-FRFHW35h&|leiH7humNjB?4;?WmH;8Ku3N~~
z5h%(Fc)hJ#LDK$iies+rvhg2FR?;ooo_W`Nbq)wg^DT@>ur&(;kvXJf$_CfOKA_e<
z@Y-rNV$AVIutaP+S4!M<d8qF$2vr%&XFfse-fvFp1LKq>8RD@Sdz520+qBa+lOA4O
zqt-%$Q$Mh3%!2(aWglplK3zg4_soBkXmf(g4Qce-l$~r&8z02B{(u*<*=Ft<#$~rz
zg$RZGh|5Z6+;lpL9oVFPVy=lt*C`o?ifnxb)iofV2SV}e!*r!#sCV&tujw)RI7^^f
z{nfk8$gN`pf4WQ>>w->jz8m6@6UHVhczjkoza(hs%izT@S^l)J>&<XUGXr+f|B?tX
zQGNfZ)vt<!G5_nx_V0xnfWrZ;?dRAStE1Qy5~%oE7ggJ-!frqov;dSfASi}#eR9|t
z|Kq-(R1Gf;fm6$U$z=p-T;&3Q8F*&ywGQoGuQOinUzCcH-UAcDgigo<9Ltg}<>rM4
z&im1zO3@S#uZsQf1oJ){WFtTiPP0qw7=!(FNP~L#02no~Z1pU1`0+;ZPaD&^p;v$B
zVo{c@mBi%QGp_CdC8h@u*EGO95#dEqBQKp=5n`=}T&n^K>y)h!6<g^te1z5pIjh?~
z!~BHZx9SIyjh@`kpHANx<8T>!MX+jfv~4zq;pblLC$T!1sDv%h%=xnW!QS&-#tFJx
zv1c0GqrNliw?9VC@|V6P4R<66Z%AaP+P~RV8}ndXt%>$oGYA+?kfI#`O(Qq#SNV>>
zyYCmtqQ+9~PSR1VsU@O0YX5)rEq|-&ZdHDCiUbh#71!$xvJT4+b)TjG8b*ItPp32m
zu*53MQw6BzmX@QR%89mS)^+0xxToKdBGxJ=^2<TdUJTZkTkMv&y$O1DZ`0!*U&Sea
znn<A%P$KBUkd5!sp2=f#VP1GZA~?7p3|NQEtJqInh|op&mE`l&JNGa&wG<1M8kcVl
zNC0u%3+Morua5yE^^yoaYieRKDm%yGi{RNzYH@%Uk1vKIcjV7qn>TtiONNZAS{n;_
zc($(Oepe4TSjaDyVnWR|KcM9%Hkcn*A17bMylw;6Bn5oFI~2X^LLU_tks-Z9j&X$s
zW-oT+Z{n}?^IU>In(up%Q@<AfD8ThvBG`yv&GwXD;z|{`c@Lg^xsIgDgY(nYToeDL
z#K?F8gl5VKwfGxHLbC#W+ya<;wxZLlnnG0XJsL!@BI96s+CU=)wp8{9#Za4XM=-$k
z7}?|OyoR%_kPt?a$MWY^(C;HqGWen%`<HT^torGaW4%8rw9hH#u8s^Cm)kE&jd+s5
zlp}%uFd{^T+~6#8CC#e`5&~b}=hmKBdlhV7LIE$G2BKEFS=B|0f|o+20SD2Bb`4B<
zww->mPYgrCMD(>q6x&{)r+Ior4^59?VY>()Z<3vY>LJwF{>5khS@uA6MW0f^zb17a
z^i#5Hv<-vE4;%`u<b`V=!?ty3M961tV^X#N`|3`UB4dN&$qfa}2-}tdKmN&p*n;l(
zEYt*=xMe_?m5=}-f%J@i$FCl0ukz+^z?q30%<>h3b<>xm;^_H0E=}9IK{G}Q^O0-e
z7ir?!*|r--G<N}LA`mZ4l4=lODKukiv(hTQfQdT+DzpX=UGL@C7Q5*vtsR$eCU)Tx
z*%nuB!$ZXN?k%(_waMLjQ81@Bm313-OmF#CsZuUHBB=)z$cSfmZ;<Tt4VMpj?YuMq
zrv;OlG5&_ew2>sN{(&oInyY;dx6-y%SmmPl;2#RYI;@>iF|!R>Dnh_Pc~aeI`<7(*
zF(0R^p`AP&KS6oFc+<)Av~OauL!J=Q2XPrcb|z+M%!YwxK^RA$pMjNv$b*Xab%3G6
ztn|(p`#~x1%`uO9H5@z8plcod9=UOja7<Hr?YEP$vk2NxJO-IDAuUTEwUDbW*T;q5
zr$%gtF8d;Md?n{!qRGExXM_odj!1^=3WvAe_Pk<K@nBF)Kd5@LR>7$IiovdP<tf`@
zK{@u(jm6(v?w>2N94NN#=&7@Qcy}jbKB?Ff#6`5=APwWZ1&?-31q&mAsJQ{i_oYUt
z-4_GpWsg$(<j;3XX{a=D8WN%>qpBLGjQxsfZ8<}?-J^$|g3u0Rzt03cP~q534^!T`
z-2c!qX~?$lQ|+n$Afs~xGS9CFun3H!+M=yTun1(daiFG7X3EJ@4lo5Fi?#sY!|K3W
z;2u;n--Vert2Rc5hP)fgvj!*NpUmCcz21Ah$x!0|LZV-h4O8u+52Sb(pD{ET->X-E
zV<{owlq-PkJe80F`ml5v{1wRK>!b<Q%4~c|<fknW)xaELOw2U*5#t?dcX!>6CEqi+
zR18#|bDIIn;)V^wDuP~tKR{#}-pMQfv|nD4jyP%zeWq6eHS+@L0^EJCOr>~x0e0&~
zbQhgD4!Af7&4RQafbQN<{j{$jOOsU$q9acu`qEYf8H7$mL&!pE`S^ZF5&H_nLG4Ta
zM|iga;k`Z__9%}WIeeVBl0i#m_``{2qImw&&*rHlxQ!tY-osPvS@zF4^7x`=e)I1W
z6;UkhtGLEWPGr$^T6u2=l1@AMWk6Y2$<q%ZOA(QSn63+HXCB&YFx=FU{E-or0LEEj
z!&$Fb_}#p|-9>l??XPSCPKO09VWmBM&1LkB-Bz`JGeFw&E3YhGn_+E8V(U@dHX10W
z+onDDNPz$iA>y~z5z8v){EEH~l+D!{|CVzbBr_AEUqLpYz~|bC2wW{Pb6T8RV?>-G
z(&>k5Wy2Clhqv_Wb72j6AdTyFXB6SUvgJW^#URI7Hy^1<fUnS@h<e%<ZyihPkt9{5
zpSz8}Dp-&D`pQcDGcQh@tdGyPDqHGei4eq@9{v3Ld_wL=porW2igq=R&X>i%8ghvo
zc+1dNf}D9LH=pJ>;Lxd@O!+Ib0oFK!k25`0uR7|0>>cVmP@C>*L2-9!;xgyrYg^?3
z#%9Cx=5TW`^=S}rqA@|U{FjTrSAsLYIOHfhv<BMFF3AS~CI)Z1Xyu6_!pzhNHcy<T
z!5GNvc?GKMkRQML(a$2il75Z6H#s=4Dto6KaGIOWT`VN7T~?$zvF-az{m6cBWZE-w
zLi@(j+jxs3pCg8~+yy(t9r$q5uq~M<&GK{M;dKj-2Ow+fty(Wa*^8imqtv8zG$C_i
z1Cp_Mx-oqU>;zWS!B+bQt0f1Lh_~L7s`yt|dZUnq52XCd{@QS&V>pHTeiMLlo#^^<
z9>PJO)lMLENsMpKWZ*6rf&Idc&EYDj%(HD)04c50BlidG4M(>GNQi$ST{cYxNjxF9
z?z3c5W}pP5TetE-AcPNiT)2Yr_^DUtIXm_?hCV_-+FA9O<G86L=No+~IUt9E_TVL~
z<ixo7Rl&;DFNXWd=bV7q4kOJqBn|)i%di<EHa=-1Fv{t6PD7lNNNHKC6SgfhfNoIZ
zmWRBj@Ja8VE@1ML>5R#l^8`#gKc9*N1s<=N7IM%P{}Ag2*!JHU`tC1az}XEBXCrFu
zA5gFAXcZqCoEnMn2;CI@S0XP74!CK{n%>)8+~?7gwPRYgFz>h8wF>7aSWPCtv=;=B
zD9v8?5HsES<k^kl4FtdZH`(tVAfYJiY0Vw=i0Mv;6~klz)*}J*eNf@jAdkDi#y{&|
zGj^P2iJ3924noj=diRh3{P?_K6>nt}hNXW`rK!;wr$g2i6=?(2E1^{!laRd}yHzXp
zCBrkTyS)HRbw|q^<p`A$E)H1dR9-7IAgC=>2Rsl(UI)j8b+*0YlFAR-NX73=zOu_R
ze6=|q?RhA0)P3`7)&sSe=K08BrsiTXW;}GhW&1>YzQO-oVa@=R{kx@IOo|K;F`ixN
z5LW~HX0sI@hy%G^#*|t6M(~4g61J;;f(`H1P{V#!RD%h<<5zVaSdg0VjG15Ymcg^k
zCBd=pE#pk^k9&`Syn0V+S<(V14pX-~i1l9-1q!@?kNi4W#dNE3x5X0We61W4dyDre
zxjaF9TuVc4=#>7?KSXW+1KR$_oE;wkmEBp(@&1lm`(SSi=MV0HEoeB~f-S6SDa?Kq
z=c2R?H*P<6EwpA3(!OPaG2?YPS^6Hn`_Lqan|6$_Z7^Pg7wWUq04(qaYZ{cKSDV7k
z8jb@S8224r^OtWJx#`J-9!8lGc#MY=ba%VCLmN#m;BL$T?S!OzRIfT9Bv}unc}%7m
z+y`amWI}3i;zx8}-vb?PN!2B?GsJpy%EPZ<uj=(-+xb%nq=#^>*Z_bg7?9<R7SY#T
zq%&l!D=)33q6wq2`Ly^u9ietP)x16#It_4@6E4yG^a9Q2e&)>-Qcn^Khjs0g?QR0*
zJ%{o{S->%3EkRT4+~@QDeEbC>=q)Jiu<r%r`Ys4AXYLNqV?JI1YGJ1IXF_Y%u2_a!
z+8JOdBKg_eYx7l-Oi0A*2FEli)xw%n+!L~KpFF3yF&n^*ps6CjFjHf!EUOumU(?R}
zcNJ|2Vw(d|tFP#(U10JTu?Vr`)QAnbQ5yoa4kK|4E2xIDL-6l~uUjAv+ZKRCuwf*h
z3gnUdw8i%mKUj}T1R6xRUQbkg_)Vg}c4_c!3Si5i-`g_iS#&ZRMlbFv@A<T5U?D%2
z1qPf*<WC#e`dbwd!o{Ey^22?%?Ja9q+WoePANQe$p31HxXfUdDr<^=)?$wI!Lr|BH
zfDMg|*Nm%<(%E`bfpfP*g5<k3^f+;bE*SonjZbBTo@@a6nkUumF6M62;_@)qxKHM1
z#4_tb1*m=$3=-pcbu$~*24c`I6MNFX*HnH;8b^beuW2R(WO`Rds@4r+15r1E$|GEw
z5bZ)#F_uY=%9?TQ`yj;7d||83>f#$JvXmp-9x;e>+sH}}pIP14;XFG!lri&e*NgqN
z!DTz>c<ns$iUyZn*asWpo(@q{f%z>}DTF43ZJhlmnWjJb>J&|MnV=QJYBJCb_|zrg
zh-gz%j0y8N_5V2g&ZwxGtZM}n5Cs~LAUWrZ&}5J%CjkRU6eI~qmY@w1BsLiV$$|t$
zl4K-G&N&DO0&0_yEKU0AKJ(4Y`!LVU`>t85e{}clzIAWis#E8j+GlSjJ)AuEuPH4-
zlt~is;pWX$(Tj%t*sXqpue|xF7bMcXJki;7;d3sF0$mfGtMPAM0|3?p!xqmEj<bYs
zRR`I0d->0obv=Fx92Z$)MT20b_s(FV?-3MA{78|X-th(S|7nJ$)}-Ex14a5k7p=(R
z<Hi4N$CQT*^3%HGgwoC}SV1fT^p}t<X@03!e;Zl9j<)cc6ac!20BJ3)EfJ%1nM??e
zvY7p<8{w8RjrAXR5hUH@FAi70A&X|v8xIxhxb825C6xcFx$+CK!xo54DSIh9sE#|E
zJnezF%eEQv@|sWyjTN;+xsIdLV{&Z<;%00z?EA22COmVe=bnV!0pVy6(JpO#?BNSn
z9%WduAQlXGwZ+%GQ;55B>bu{MW0de-Vmhat5l?o3I8}}TPoC^17em0z3moD{#MiIu
z_+#rd3Em(L@K+!vW#v*p=-ieQ#3#&8!m+%KAZwlBP(mnNVD!8z!lgR8<U1E(`pG-P
zc<iq6S0pBNrw}ckS8F#}v07;SEZ=0#1paF=y4Stk?pdFmiLeS95Wvd2FNCraHbAKe
z&TSIzEd8j>5PISt^S#;z*)vWi;<9De*qwMNc1t6zuW#Tf7T5etCaX~N`kDSsMBPM?
z={BeFzH&*h=RBCEYFKv=)<d1t3#18KCUXF)T>yc++Kh2CR&kJ;8g3=2tLnhV5l{Qr
zDy;o_-lx!{UcxwdctQL*AOkNfj_{{mAN_fw<NB_1q|C9n%+KNtFN;Q??hSYw<J;~Q
z++$_DK)4@|<{<fzUSiCqL$LQ5Kvjea=^l_+E`h`>-3Ml_t4!FfMx*q2{tp9m#+?}w
z%~WycDT|SwL7Zpt7NPm<&V*?NJ-S}qmyC>u^QXna_B?YvO#s1TIGKBlHC+tV!c!DF
zR3%+M!$ks2{X<0>wZk`jnd>=5N-*wq9WvvwvFTAkoxrH24Oo;XLh%vraJD0CwUeXY
zmSlIR7_(+At9~zzQSFsyxJ1VGjR<S%H`rLTQ%e2nU-`0Un7@4vNeU!ZQkuq0cF2g*
zdH^-QxBC%3IwF<KLdra|xoJ7P*{AIs$`F`xuL)O1p!7EYG8=4BTzm2>moLQqp7E!l
zJgo?H+L-mq&9S?9KR9qlB@y3mwUQorX!^@!4Fiz<UC!xUqIVPdQ7Rci*$%Zu97`6)
zYXL%UF74gG;z5W>VIv$zkqVSKK7qs3*5ZYj$1^@Ub26zInlh5z<~sF^LI9(Z&@t-u
zNJp-CAWX|Yp<aoUVy_b4w8+LmB6c2OT}C|(#q%7>3W;@5K8v=N$si7}kl8Pdx(g~%
z?k~*S3Xw6PekK5!5iQK<%Tcl2u4S~X|Fuflbz5!qiC$}aBep7f>FKjw+a-_Abj)a8
zuH75HH(D&Qb%AjZ^=I2tgSpf0YJZu2`fT2j0X9b-TWaD5CV<c6dK}zrMs~`ODn>6<
zB4kL$9?**{!?XC+zZW6vMJ|#G{O6aS7qglCT-{d~x&ud}=}#|jlBE0crUN}INO=D`
zLN}}ufE%uc1{1T?uRAyzGUSGL#LFlPrdKMXZxxP^;MxT00Pg9cd#u4Wim0Z;Gf;!1
z?H}7rc$3}<(_U%_qt0rnA~XntVZH_I6eG-Q6F6=u-~oLrYZNL0JDZ|@Nl{h*mV`s#
z<+TZrh4aywVcMeY-+(8o-R|_MnP2KNC40I7@<d&*3N1<@wYtkhsd36wS#KdCFybv9
zpzrlf<VVUn{5P{(%#Z&52lR=EVf`|zKC>|(_%v|^JtzA4Dq(U0$X<tf9}Y$NlyB7n
z(9zYQb|>@1oA^)4wNvcYm}Qn*GJwSHtQ&8czLGpWAx3BIj)bhcfY!3$iUBoX;?peW
zQas|x>62I?Zq+4OZaWEKU}JYJWH&G9mko90_{!H~i~p#z#RC-StN`Mn0e?(Qd?Sa}
zs-D*LXot@CEr6?S$ch-?9X|s^P6rHXWK4)q2Bk5-x1kI#^J7NmKh}DutV1PhVPOeX
zwi5Sxfbw^(#IGjoy$T(HnR|o5(T<~DFzMS?tay)nS1LJNs2*`WmuuqMVk>}Mhv|s9
z%mU<h>>xi)f(%pVeELnQfmtZK`{sBg?+vfi-l*Qln;ykM<xlE&1VSBJFU3e&itleG
z;~>jf!!q+k7DyfrKp7w;K&7b^xXRPphvvQ};Ig>!T!}}aF?oAzBG<q)VuXKXIyfZ<
zROun_u9V%x$1KI^HgZ2O{#ws!Jf+xrSKky+?h9;Rk8o8Xrkg-B*|J4PQ;KeMpaI)e
zTiD4WJYaLOTH}eVcjei;vUk)lPEo2`?kmHSL3``PyRcox<h<GL)yCnQ0x?)jdQ^+@
zE3Gr;WR1^L8dF~2C;jJFQyp<hrlZM5;LbxVHnTsyNSVy!O{pG%hp4=O7dp5BfqDG{
zoxXMGoB&#=$1c4Sga~3d;3L{~FZrH|_*et-7jX&_FfFI29f#Mgmw}AS^Adinca+Px
z)^}5@x|GvgnX_Q6BqTdA`fk7PEx=nAs>QDrkP3X$;KE(cj+A&mWfpLaJT+Z~He_F*
zvdmP3X@+s(_m=@5QT;T@so2Xy9A>HEgHA#aXz>B#6vJ@rhls%!$cGT}Key#6NKKHr
zYuUEu;2={sm$IYi>@mQHdd2ko6%K4hxEn~wg{r`#Rag)^1&hf9n2<o_*+(G)A;h72
z<xKfD)ap3e2t#?P8)=@bON%!`lKngJ(Ni<Tj#4|X2Wa3<9qS?PA^Y(J${|G%N`Rf+
zwsZlGxCDiVaU<1CcBA;hld@MZ5GH|aD{e14se}l|<9vsWAC8rHb*K|K(ytJ~Id++A
z&a^cZu-$n|*_9hdfWUdA`q*9CAzf@;a@~?_mr>}JT$gKsgd&=~wjYXC16d#B<>0t$
z!Z79h{{2t^1gCezcqu}obqld7FK}BZ%LMrin_eaB_R4cGrQI|#Tsb26)Jv0<0%f-4
z)6|V!dw8qJ$`s4R5DKHsL+!kI4l9d*A(#%Z8s-Td0Ob{Dd6v5?nk!g{w>4l@nu@9o
z^X9qotvo_BtTr{jU?_Pd(P&WFO}g45UQux=dzMX&`XC&ya)`KkP|OQT>W;FN5V<q0
z+bz>U=^pK&P#9LjBJCz6iz<uN3c7Ng2X^^T?>iqWqJcwlHyqMmAd*fQorjW<xh8pB
z{AET=S<L?T$VS|+@5Y!$`htkIUhZlHUeHmC^xbmPsk6zb@19fYS)+{i`Y(-*hU3yW
z$yI<DotHzHMlw>ftl>tL>1ZBNo(%4uqN)uJvm(!rFUtCNh46*F`a{BaR;R7Q7bF8$
zK>bW~EFI5sijcVKsr^UnAw{YxmEMlpBsPmku=SBNh8+&WgftwaqNS`$B8XDayCd4#
zI_hJ%1*E&BzNlX{j~(1hroP51f~jhXk(imDF+=@MvYB!pf2C?HF=9d_+a85_vyDz{
zH~=K{Z$)xL7hfdhIUjKsgDkTi!VY5<zZnaoO4tt9xty>3===3{P>$nw?;3AtU0x?r
zfqLAB(S?J_a*WkJ?}R5pcE{r6>z~$^b4$(NK9a}10F~XUb7;|XYnC4;G_8eh!$#+S
zp+?|Cy&)v5%7?ka-*Bjy-#V{BrKoOrk;4e(xZeR0A=+yF`D@{nco#2teTKz7CtgS4
zSy#>x%l%gQ&QZ;DbuoSZYTaZ|0M`&RaZgO1k(ok+bD$Dsvg2x;M>jug3mCx4J5NBG
z{@Go)P9l#)uzGx_GhpL%;PUuhxG|YTx6niFSSN@t??l%AE=7oX-Olvr0m%s?PspvF
z>H%}#pRZduDKD~My*j=@F6rxzHsw2ZmCsmFN@qpr#8Y=KzFn1DG2D<e*GJqw)||3(
zCfw%9+gBauWF^OIg;O|?`Cm=aIUC1kP_>}zu|*9AXA+fO337_Tq)2=NVk^IHyY+*L
z+A}jH(~G5C3&E=4gf8;*I*$bdr+`kLF^Rpiv_rr8TtF9owPQ#P@r8DqBq{!oPFo8b
z?%K*vA=)aYx0HuFj||J#XK-7UcMI&-@dgtZC35@gKwu~v(mlPgG<}f8FIO{v%KUAC
zDo<8eHRUdHEXVZis^JHj)PB<kTJHz$sY#kD8~a~~^u939?!X_%%u`7Vr~uy;=JGIY
z7n0_IQAMgN@b1#Oz=>h^%|%X(mwQKIk*mq;_-!?6O=*Gt9m#@IwRFdAwR*+Z*{1So
z+F#4=U=|Mdet-JkEc^?65m-rPacE@K>vKy%6A26OWEN&cvR*?(u{XL~fr!WFH+Z@T
z7<|2aBxqm!^!H-iq@7JpV^R|eFFv)!+U;>hI}SN<KIty+<Pi5`{9ZK%y%ayz$~Rhr
zRjqs$(O)Ix=ge?@1^mRBzG2=^_vtSqMVWF)HEEe2WHN0ha3tU$b582||M!s%#y5EY
zG`QuzqR;zeew-v=aECb=vS^C<fpz0?b2@^V`Teya`H%$)2T(#?1eK*%cV}pt<>K6{
zb^WdJ@aPdUzW$&X$xtUPPc&7ijSVgVCdlb`IDT*Zi`9ZB)%RU8B0?P4Q=M)peisKN
z16d6c02LgGn)u&W&=>f<*cS;fKj-=2MCVpeHUW7oAZSATGfCCh>&L4xAmTf@vIIFO
z-k@CLej%s|FWf)SViUNA#r;M6`3<QwD_QV|y~U0p@=iAk8+lVPli6t;Hr1xRyJE@Q
zr33p3RduXDwDtfLhzTJZ(9)X*w19Rn;up`7UF%}R`R4z;p8xG=xJDD8Lut4uQ-6No
zXe5LFj$@NB@rLAq_8^Hdc@deMr&XW67D@m?_4gpRQ(xOcyOB9&lH8O<tRLDxAC5J}
z?w|<HcrWRO$49wb*noco`7W>A!gV$QVXUI!+BYVlNKxZ5m9JZo9v>p%3n7mI^kVS+
z!mxh>6M6CZCwtC{{qZr06m&^nSA>)3c@qc-gL#%x$hx}T?m&f4wV-UW04l@<NFSfO
zg@EYxI*b*<HLqxS)Q&vTTDYT;$$qgl4=Q^r07SXPQ-G@0JNIXcW+uHKR1iD{m<rNK
zpaQu92=ij<&;84W;8E?@U5JLf_Is@Quso<+x2OP6u&3%~{rjh2_0&NEDrJqDDIlKn
z9)O4+_PZ>tB{ezXRX1R+g^Om~GtXG>6(0ET1PBmaVVtW6M)ENzLZ|2a8NiARh4Q7g
z+hlCIz&miCV%V4YzdS%Tfj>F~r%c=YPwpD{Zy6Mstmz+V{GZPASEGY%@Ff@o#x`E2
z=ffa<EYR78SYllU|CfjMmob!n40MW^s;?9df4=gPR4_TKQa6i=`~FXs;ru`VQJ5Jg
zDX<BgbJFO4SvIeLR=MT2xX$0vUtZ&1ewl;~4Z&tjC-U<xo5eyykVP}k^3Qklmwo$h
zzKnyO+IM{k*7FybL;yA0=HSG3zS%#Ao&V{m*+pQR$Qib-oIg3BdjSt)>R_No?5}3s
zU-rYfFZCOsQ#2fm-?#en-2*ij@H}rh*6~mO*UxhUveySyomA(Co8J@A>S}w>v;T`}
z{_E(c=>r}(Irp$k)Oj1&8v=&EFxL9WU;pvk0{WL9<&?owbI1AlRrB0aGMj~NY15<g
zQ24*@@pn*<zw?-#I6s5zp&qw)EadO~uX{Wcnn9s+a@KR_PvHXY=m+BXUuM>S8hC%c
z`Jd2QzT-@vN^pK0CQ*U`$`&+?f8IO!%g+7FFPjih$LgeN{mZ?RXMkqIeTh=B&o_H+
zDF26#9FX^fXj?shvu5~Ef8ozbS^wAlVrBrH!eGwCH1*E|u}KZ`%+%^u8?64@3kM`~
zU|G&_N=#Kge~KmO2w_QG!Jixb%f9_LU-m{r^CWOx@BCA1;(?|XLAOlse}8f}c93}R
zu#)+FejNUUJQP>tvsCf(xAY&L-=7ZoU5DJC>(6(P%jed}9w?OR*=fMMKmVZrJd*#z
z(F7SVloL*+cdwtf9!<pH$RGaZ`6K_PfB*Goz1P4ge*U}%XFhk~YywnZdTIt`>k0n1
zW<|hH6&i@ch_v+cPmM<iG)sPX`S<QWn;nd4f;jVYnIW!|r&7b8{%MKjUcWe^u;eqb
z!(tQAXmW28AoBgU#?H?G%q+)yZ;^E8pJoyaG-`!2mG{p(^23Lfkgf;HNl^X$$9@+t
z9`Il0IBv$#$n<-$8g>~z`{>Wh(EnI(^`QAgHZNiO_kaJrtMspe&f~r8o-y*z!Pd@h
z#y++6<xu7CKlT#>O?4a9jRQX;2{iSP?)2P>_dl8{18K9l4=P7e|Lm~89Ti{(P4(fG
zYg_#DGC_`weG1>tP3Rv##>Uz(y?Xn10Dh7j_Qi?o*PQCQ{&{8R24AiU7|fiX-G~42
z@Bja%gc4dOuCB=B{quREw41cb7t3~H{_$f=&{TYlXCZ!4lK^O{Me6w9yI)q2lW!2S
z?{)4HG|ZGhQz<YT5EuUA5E}wCMT)ij=pWx)#+hlP%>0E*D&Th+tBCu?H_y0#UfH?9
zt`P%fqF&(Y+kaU=1%YkxklX_#N4R59!Z@bN4CtE{Yyg#myB_tc#>yM3v>F%w;E^f-
zyx`Bq04oyuF(gnaR|9m;-$T4CPyiD>em3rSGH#Y4yUYHIA84A40Z^6iU8p8~FJD>u
zy{muY?>7GTJ$#>dXsZB7@jj9FV1PuP;5zpB2LrvBfxQV+Sz~nhz=YTd1lm>~_zajP
z&WbVXfKsi$>jb2%i%t)G&#G#n01b+K)}B9KegS#=V`o5#;SR*l07VfAPYrh6jdq}{
zp4-<^X5%%Wn{ik1Na^I3_%a9+=FW_P7KN-@N2KdOZ94zPSy{%(2T0<h0AiY{?J?dU
z5v#!TZaFRL7p2u!hBs?PzHi)-Y}EU@tmW|-`5S_Igf#+16qB?_JEVVwB$p!qhao@F
zj6a_9J*x68$Tu~s1iUv-ttM7jW^LO@zJ2)(Snbn_GWIhykJRYXYBeg`$PKIc`6%@s
z_<*cU0R+V+)IF=8JE|W|r2Rm-H9rR~vjCDx52&vmnAvQr{3DR!t(qB5x`v_j_%!`h
zcUoS0w&KJ_<GDZe;&1ol0h|Hu0f#t66XU6#T#>xe2xlrJ`q|7P-Ix-!gGhv4uZU)Q
z?5_I~+5w2`k%}LP98o}UHZIKlI_)n&$Avw1&P(l4DJBwmbmTxORFq3+Ce^ev(gtOr
z!klRq3nA%aW5U8*Y4T;)fhnWT>KROAfcdP=j?Do3NxxaL`^&jkZIS?Q)ATHKtl)lL
zvEh>%%QnidPbDv|s7)Gy%_G}h1bylO$F2obbiM+f<bfWoRB;`qf!p5IcyYYLVR)mA
z=mfrEvoTQ@DULD9h`RZAd-9LfCW#Q(u-ulhcgg2Wd{Y(->=MlZE<JsRk6moSr25Bz
zn#N)eq^e5xkWJ3tpuoEVzzsAt^oKFuXPs4Ho<LAW$jKiGZMJqw{#Xb4<f?N-@NX|x
z9uhs9G@CZ}X|koWzJe%#`|#W?Se)JWc9M_+z{}wccr&C)<3N<w<pW=<1Na*aGzPS`
z_!x9nN|)EC^uUjZ6K|j@atD6U<Fuk|bJ)r*BPd>EgKTKkIn1U%v3dE65SNB(;S<EW
z!Wki>#ScmB=r$1J>stM6b85l>Jvbv{z{Za~PwthyZcy>~ZUq?LqG1M*UVGTnT~MHT
zkA%mCr+**bu-<rf`VMJn+yy_#18Be%FrL?lT`A}BDghjIOd{Z9wei%hpmG;<g_Ao#
zN%6s@7utY42_!^S@IK{9t!+u#Dj<uYPwSUfoAH3s{>O8Gk1$~I@E0J9%Z0Xm3@7a1
zClEWDmhJJL>6yDr5Ve}Xk-t;Qvmi>oD^Ya6n{e+z;x14M*;0A;NJ5i+3X&QxB(678
zrgguwC_J^v#pb)~V%x%|ZRMl;mN#>w4U(E!fuv?`2nUy-tZxGjBSn61o%bW<{XJcz
zo)E{KjjYNEC=<4qmj4X#`?*CC_d)~+iG7Nu2T&>Adq_p8d%TnN<T_@#yne^Z`4|In
zjg&fU#vK4^u*P>u9NC4<E2`Swx@`#fjuk{n7yoe=^lzPam%53N<CoRc3{S_|quGO}
zG7WA`U|zgGg1Og2)o*D&s>LbpbEVq8joeFMM)TEw`W+zq8vq*t3tVT*yR&zWkaz}B
z1UqGvKqMrIV#oe=S(u#ab*X)D(E*R-^Y0UUGi-u{-NLPtYXER>_uvEwD-}-xnWt^z
z_EVX@T62@RR}t5GaJ2$2AYrrD9@b63mofv`VmEZP14BEtr1wT+5}PGM>-VR)L!T*c
zNCnV>>9UsG#<wT9>F?Dj+zrtJRVAmgX$K#q)Jd8feb^#dXO8QFC>d>0sq~AR|5%s)
z`Gy!c_7t|i%g(PAJlI$}97_uM{(4Lc*J##5@B|&g31f`=6-97pOg{DN&i<g8^{^$)
zCr#)ph>NA+eT_W34)|$eUz&E*uV3o>l)33oz?TmxuDFTsa4of5y!?<j>myr%>vqS*
zU68M_mk<%J9n?Zm&%4lFLgIba68{m;h@POqTBwDpgbFsm;zH!`8MX`H0!4f^-(%nG
z9a12?@Qz$s1b2XR6fOMvkLkn6Y#3JR+{xED;}WX8w-9v~!8n>bWYi5{2Co+8rb1YE
z1@r1RwBtTCAeWrcTEV0K6eu6D$(z}xj6+JYV}Q>nx44aUNCFb2`hyl~N=9uk`1cm1
z(IUuOh`4$PHZ1bSy$p^jjI~BWpK<~ZRGoUfl7^@4q!tg8#Vj!qhKDP7#PM)b_q&jz
zS3k(^;M@-RICr{VMP#gNY$#hka<#$j`YxwBRL~fAL2(ZKkgB72hh-_?8uBG1*+hWh
z4DSQAhC?1^$tMIALp?f!Vm%EObEl_Ex&n1j!M^WR%(?^E)lXEg57RH|r|h{+w;h5+
zb}+f4xeGVF2h?mu3RgcnkSu0O@z-fv&Sa}P1MnUG;gK$s`uIhwKzsm5_BcZGf~#gu
zXK|d%(|sKp!2)i00%)`jv$Sgm4~*Xw<*k2IM`}>_NRErPe^O~zF}|v^0^#-B6;6K<
zot1b4BDUG;D6#YQ_67b2d*FB#t{rU_oJ)1+2jTn#6p>b~B97i8DO^m2b-OwI>&Mu(
z6NcAmnLg2d0*KiA!Sp_!6D@UTR~T=z^C<k@2Fb;nI!&rcA;LRmF6Blm{xUROzQ88k
zMK~UXH~tRqw?W?>3?AtRB0wT6ZYjXj1B#}MtXCH^@Wc=zUqNX~tQkSDx8enEaU}EO
zMNg`sF+j1JLiQ@sMw7zyd8ufQK|6rL<q<Y`9Ljkq8wv5lFc8TcTWPA*vXY20wL-i~
zYP1Nm^hBp90tqskX_pgGNs+aRLCk|dEyZL$eiziO`*fCyddQg5Bc8f63ud<boeSVQ
z8y8)NV@dPnWf>Dd_k|rSK+|mI0mQuw^3>E7!{&Wt0Rn?|io7wJO}Mp(czY+6>m#jy
zBiO4DG!{F#fb+pcX|Yy)VD}K6kwG&R&o)DL7{6FEKsli}8+z-3&?Ixh3E=Y-o19@;
z0{Pd~SUg=xQZJG`zV1Cxf+YFfOc|k;MXveTEu)$uP87ubnxo5#SPp>NBPr_X3l6-+
zE7cZs$6g|SMuQzNM1hTPC_YjEfgA!)DOliTCEFCg?_m$i<Gd(;{glf4`|V;4g7`gq
z$kIgTV+n3~`k0y!I+BQ|J3yGV@LsWrF>+bm^oIQk6BaFv$PXe=F?w05@AMLrw*AT8
zgtm^9GTO>^O>nirBIDHOeio{0cE7Uf8ru;i>ypI>?_pK;Yf~56aUk4Z^4ry5+2e=F
zzZH;}W9ccDb|!~nP|jeNQndfaux3aT<C_C+-c>gChbOuuyM3#lK6XxISZlw?x6(ib
z3HI}A-WEAB;_at`v5fFr*Rd>qy%vWDt^NTXv~{FEzEW+f<sX!QI>mC#i0sKovy6mK
zLXXVml}|s*z>uHXIw4%tp1nT)1<2wUNq@UYls{c(hD2TIQQDYb2C(FnNRK*;+C;E<
zLvgu-I40de^zV_&|8PtGLmmV8BlIdTQHGUI^Fh?QWOaZD^U344*$Qv9wt+7z6RQ0?
z(t-qw9!DPz``&)F3My>b89RM~bvF$_2Znek<V{T*fWc^a6ZoYQCW!<rf;8}jr5k9S
zL1iD9$3wh)C(5u$0mPH0i}>gpJpFPJt|WwwBxa1`5a4&%2a`x3O-%Fw@aRp=c#Avw
zH$Xh41u{U|F!y+IrlmdtC}|6^5BsN<X2lU{eZ>1_DG&vFGOvd72|{YvCPGl?(6aqT
zP)1nLh$(1PGlhjR63WH19f65p!e|v;%}9asaSl)#FR;`>p62~7TaykkURl5^b~Vru
z@4_~okrB1mC*YCm3e^aFZ>M(DFq%|J4;50Wi5m(U(NagK1vZ0in+GWc5E~R!s8Imv
zQmLRaX;%QZr=n5*bH;Ina*Bu$YKS!Ca)DD=dreKT=E(f4e?x;uM~4{9hI#1-RGh`)
z;HX&#(>~3Iwx}2{mQ{L*(_*oLq03}boWkQ6UhZh(Xm|1<QSjDWe*Uu%^`#EGwF7`H
z7S_}fSe2Sc<Bv482DM~2f96QDelA3n9)k+W{NnJ2J9Cfd+JY9Fh7eb1fQQiFk;+0&
zBiwh|<{!s9e?5H6qjz<(=;TDF6XwfcWyOP1x&K4HBZ6cX3Ahp8L7Vb<UttjfhblBZ
zKxCH}u;J}-u_k84+SLh$mUc4Ce-3uuG+=ypsQ}}&DoOYt`gH|&Qqp&zI!!%Q9*=`m
zpgN!cq1`2%B0$&5l0JU9T{Wa<1?52tpvg1eq=)lQR)=4=V!()=PHogiYG5=1_W-99
zciKvyqmQdQB+l1oYq;Km^T0c!a<k#+9@yk>8gwpo95~XORS;B7`>Vk8-(@8e;C^1@
zyM75@-aEf#>Aptj31DzNcS1VoHQljNcSCXFr~hz1Q(gP19`r?(98*emL$L$PQw9kg
zW4BBwm{INZ1mMwU_8pcqw?pG!H2r`=%zS&^NiDigZdoIb!!62Y+<wy#O`tKE=5|Uc
z{kS>w^A#rZrKwGq#7`{E3T~%YKJ~XL7wtjiwQb19VdTP`^7LpRt~y`HjT5Q-{FYCO
zJ?j+=fZ>|nu^a!OI>f|LH7SLU$=dN+o&`t<9YQA;MhT@vbc$=0G&wFM&BGF~)5+B*
zmDUjGJM@NBUPNnwGBO+PYYwqp#!<eH9{`v5bu1^*BzgZ?8@Gv$^mj%g)eQBsc*hm;
zuNM2OOiS4&iZ)tbHE-ldz3T6i=#RqY>lhn5S!0){$yjA#RK)EA%blZu@Cgtz7IOL-
ziKRLm%V_@$=&^~11O_{)AIhtQWD;lBE5;`pN_p@FW}f~uu`pI7f^0Ib*L&Af<>Pnx
z5nk~!S#aV)9B%w;22rlt&t`Y%WZ0}q;uM2D3&VRP<ms{0yXk^jt`nN(i1I7xa*97`
zbFDbx!Cf^_TFN$+6VG)Vx+|wn9hK>FC&i%F2O})yi+E2n#!p#wE&F&om^hwld-{eu
z)*k1&utqISf$Qx^hy{FVzuv)DVcAx-UO{?#FWK$5JORKAAw4_ptnN-?qT6}pmg*^5
z2DHj>0=w=T2NpEhQchCtC-9S-71wFh-+mWu2N)UV6}s)OIS!B2Oo0^rt?4dtn>g50
zu`duo-+hQEb?W5_tLGEurPY?G{Rk<SIrY7gZJMJyUXr=U1)<79#Il@|Om}B+b;MLv
zdHwnpSd{pI%yzJ7oK3gBaZ0LakniXYDoVZGRrQMvDeXtM=SCBXgA)gJ4n2>0Gi0ie
z<mP9V;#V;F1Gdbtms;<ikS;d#bT-6~mt!JJwQl4!ns(b6Q81IX?O~+U20K{DHLT=m
zJ8^h(JcyiAJF6cR6jrKr9ngkzM7j5X>1yoljHG{p?2s3ve;#RJ&&J8DD_&`I38_KW
z+Nz9#c{UF1KH>B0Jl*peXrBxcG@EecB0%<Q-YCCbesgix>s4@Nk1C4WW_5W^R~E5I
z7uLp7{PJKJHG<LL5!j#<G?{xV>pE~)e`^yveJ2%t2ar~dbdDGSuj2ZqufU4&%A8KS
zqY&z5`=Rk0zV*q=Zpi!kiB?8)c`|IYKAZC%^xexqj8&@iRQb6FLL6P?U$^m1Je*W%
z>xo4+03Riy2mToc{3E><4KhGz|9^b^`<otAQyl@OsW0jXQQj;Vy(H!k=xXe!=N2+r
z>h3S&BjpR2N(;UYyMR<sF~p0`k|ha>!!A!deQ1k;F@A=U80=^^QJEi{+V4PtHv856
z#VMy3ZCzCCT2!$3kP<{|hEC+Q<0S3xtTg$%Ei%i$n0F!kxtk1)-70GFTN6nzCCcLP
z%^AcM{c`7CS4`wlz-_FD5loNUT!}CGz4q@?=t|;DTYQ}(CRXG-_Wo{{xO^7rezOjB
zu#wMNLbGbW*#og<RzgXr$cTi#$S^4GFaTit9|MtXgKww@>`RM2uIrPK_UYp|O;Od1
zMOhj?V#RqxmR;M$pJ4CWG>rJO(2b*o@4+!_UafxNN1>#b+ofA;=+U0ymQLve{B;5j
zQ4=q*#o8-iuX4iYKfL&MMd7)^Ykq_8P2zA0@?uMXf-P{$;-kW%-5`i0MtqI6mDc)7
zG6R5dN!uC=zk+t^QFcC%vh0?I-lkcI3fRoYTNm149a<u*&}7d`kB)l|pn^Yw(WuvT
zp&|jUqyq;Kv-X{iU#M|>%XaaEFO3lB+l`Q^VpM!WU7*Rm3w8*YFuh5XN;jwmJe4W{
ztZv8IuMLlw^&-_Fs5lAdW|Bq%En3Q6C9Ol{4?zLUSb{6OG+UncT(f1qQ!!mrY#Wh@
z?~>l1%;q?@qMy35!dax;YoKkcwObE}>Rmf)u6AreqR(rw-3&tqYUJ5NF_8#2dXQAp
zJ82cvj68v}LN213s<1+JdJTt5zU%MrfRsAlqS}XQ;y1+kxXc|RRtfZ09UdF$W~?4E
z#dg@4>?BW}WPgi!ew*GU!)yN?nz%}5U#<P*1#SCDq`%0NRw$7h&p_hTqD0;rC1<Q?
zeNlRNWJPw74w}B=j-;%E{9KGOoq;i^C*o8iu)NbQteMgicsZxx;?cYGoXE%lqCFnM
z+1=%83Rb;2xdXYQ!PzHuTkV``Nk#Pz)a28w7#7GSeT>^_S|>WCw84tz`~oTqUrjxr
zLAWj+<~Abc*dN%Bcs~6m|J7ekcU{<BXgz8L)Hq4<j~Pob3#}|e6@dZBaufA*T%5O3
zQCslswlb}9z5cSKah9cU_p4rr4;PASg>qLjY4g&)i|5X^=dZ%NNBVW{k5dh>BiWO-
zf^{n43EP^Fj5g?J%w+>k^6zsHK}6HIFl&RT=2Ri2uh9grI=Me9=BUyq`G~lOrC;Bc
zGvL^3<P<K+qJ3K!(T>dB)VQOtx45uhnK334r@Rq>7|~oOu!UG!v`$EiGD7t^I`mZs
z#rPWIzfJj$5@i6^{$c3}q&4+Hy3e(>(TOp^(n34trSKspFQ1SbsfKHgt)(=JNq}0I
z=c}^J#Fy%hjxD?8hF?ahKA2_O;Vb=`$Q~r^p45*+6jPE83<KX+pvT;HKRra}J%Z?F
zN9Ps@mIwKyK>VY*NSS_r#k*V9p&H}2*5le$ztF%Hi*M;1&ReMhdrCz;y2mTt(65A#
z!K8bn3aOfOAjuc%LyuRA8v8J|`}`r5y%kJFgY^vBApI;k@{DW>j`2qD90B^+xb~0T
z!UIBk)X3mUEJs?&h#%&Av`6~~%B9~7?mS{zS2xZ&kQyz3xckd(ZB%{3Li{KfdD72f
z8E<822W|PNSq$pSMhTO1rg0a6o@f(Uk%{Ul+=O!s5Dic)8e)VI%xF5B;Z%uSuNw}H
z+GR!GzLK3hTLW4RACivQZw$42MzwlIh>F~BXam($aqUyi$m5wk>6FMHMk^k-=T>Jl
zzRS&Jc&MOs477dI&<s^Y=tv$)VP-ca`_^jm((CI?$R9JFRd2SgZNA@W$luT7S2d-0
z*2+vL*c!3ABvIr!cQl*0LEtXB$(fWI<s-VzefUt9R=+u_0c)DGY6wp5a~S9ozY1t>
zeN{e<Z{-yE6It*W%i=mk5i9AWyTHcYwufxp$US`R9J*qQdFmc_<%d@$ANuQ?Kax_V
zMuV~zMz`ou-n5HSbrU&69iA0YdtUb^f7XGltePRI-Y@EaDG`U&vcPw<F+0>mC$D4M
zPLU}~OF1Hc0XhR>d0+$TaL&T?&7TuYO;?yp$Z(Lgtcca7Wg_&r1e#%e+MUW*U<_JL
zDwTCE&N-8}>)ezOETvn-b?x(ic9cAahjIGG$TSSX{<oTKtXiCq9DM~KwdI|RjJuTe
zO1K|x(OmibQa?dRf7U#)e_0`o`(ov9s;w)+XG#xg2`}PNI?K^nvI3)V_uaB*>K$Sw
zt_NKZ4oPmeIcolxLx#6y>js)ue0?+x9j!q4j@Z5v#2(OI2(H#79HkfZXeLg0K&tGs
z&bxl6M^Gt@w1^?UmJ<{e4l}}IdIa<0FKd;=8zi}GI_npeIpa9MJ=45b-f^l12E(IY
zp;jSD+ceSnkk<P4g^Es2Nr%qUdCD+mQu3E_?lWJacIG*fB?(J4gC>JHkdyF<Vz=Ci
z)<Y_G6Z`ngsMV$}iXr*5`8zKIl4!Up=Bhgue&0e)lJ()@a^p`rHQe95nL2aFn(1+Y
zD`nP<Zm^j+b~nxh&wO$bJD>9vE7|SP0A0SNaQ|jcqf>FEgh5fO4A~#?dY2c73kMQI
za#)^P-SZArOm2NBsB7%@zRgSP^Gp>`UQzI`@HBzPT_%6|xYWcl$YbSbhPa#QYFkLt
z)(c!Ivl0R{8xiID&Amxb5u2L27qYJ=ohk9$X>f{2Clk(viDBY$sY6_XW0*BsU7p~@
z!}z}ckXV;mFePI_Xl<3~E2(Aat`mgo<1TdGj3ky{E=|oP<sHaKJqU5rAt~bIci&?^
z!Ak9Cy*F~|+*L2n5TIZsLQ`bxnt~FI#st)6|6#BZqVe?{<^uyBgYvk%#JRfY^5q?d
zrxD1?$=sn?v7FL_*=Kp{n#c<)Ym}C;b$Y=Jv&}jU{Z7$2MR}$*`uEs@O_uetqwb5X
zB|zIsljf|x4}Tkx%{5Ska62q5Ck@f2@@+zS8!P!n`$S@lA7t=zW{0F^Gee=rDN8$Z
zA1FexXNd2Q`XPz6%kg}^ZufBcNQ{a}DXS~OZ~`GuZ!W0VZ%Gq0+3-k^>GI+=n%viT
z*gcn!$loz^T3<CNUnqE{H7D*fDp`g4$B6pNwA&`%@e{q_hY(p&o@xghC>>)2GR!aJ
zZC-~jV`%6*ifb~X1ldUSRDP-YqK-RUsu|*La%b6SI5_qd+1@<i*AyVR#KP5qN7EUF
zxvHk_FOtrQW1<;UjqQi!aGOGpE?<GlP-*qqtaHz!3~R<&>aFkzk$&s39Fo4MIHj+G
zVOi>ih&w`5L$Al^m}^7)-PW^NJ9LYPuMMR1Wo=GYHOdkSW3HnNO5ppQBj#EDK+T-H
zuz&S=yHTC;V-kyf+VJm(wmbEG(sprLwi%+(-q7~WaIn%Z#8HQS#8&0hranEoBVq@c
z?5;TSNodM&k2iT62tlOqFBPMX(j#`7Vk_}WZ>I^WQXm;or!ptnU#ihReI74RvC)VF
zrz3dQVhj{`Fmo2yIhsA!`S^R5r=f?&!^ltN>K6Dh#vg^7E$H)bn5;cT$BE)#$eIn$
zT#9XxHfgx0s+0(;Irl(zt>qDgk@fQ^!B9aa+ktM%gC&WSS^4<w7UXr0S--X}S&nR!
z``9Kg?LaLEeQs(x!F;Lnr>n{v20ne(^>!Gqco+35vRw{^rsBRW<5{d&-<{-N`cTg>
z*KM11FQMCU<e3&5bGB*V&Ctc2-M@gDn^{5|^Oj#7O~Kai|J=Q5y24NbqWu{{t2dVS
z^sHXCg*xhz(=xrXO3N%}nTm6@vrQQeSEmsqru)|V0^8M)TRfKOvOV+95piN$hVkfm
zx0gG2saw2Af*HiK+v(rBOZB@S5Aq7W7%%&tE@*c%Nbic)=n$MR4@;3QeA>xhu_;K=
zntxqb69kO{f!0Da@vuAwU3m?Ey+9QGz9}HVC1mnP;J)U7+qvj2_;B60v@8@Sg0@Jo
zrJ4WnqPZr7oHMwGgYveI(_m969$JM)Gdo*0hGIS_ySCLA1l>b#mNCEmTQt+K2hGrH
z3T|Pq*+IclR2;0ojM9=S?<!N?w_kTiJV){GeN<0ho~Q(|9H}+2xSQTRVigz78D=Z~
z#Av_QYcTE}@tlEDsWUt_nC|>9T_GIa83G=q<d*rkkC8g`@iZ?xry)uTd6ry2ocWX^
z`JS5-8!Zm3m!rQdAOJ;^IT`c=uxiu6<-47Z+c}9l_`C6M@r~)#=)or<T0X7FDcatm
z^k*j4A*qPib$;@b3JotT{`6#@{UghQIgf-B<CSP^kpqK0x2dg(Gh!L1D8UuO)aG(}
z4ch}ZWBBxZYakxclTrRu>P0WFlU36Unbiy&uIG6u)!%vRJfV~?^-SD8NA};HVcx!N
z4fpL4jG;Q?6vMj<6EKIPUDpvk=9RvRq>POY*<V*?9B7^|x=ymln2(=Pa+1X;KFV}`
zYOsB5{{HL}jpXAwj-`<XydxU|r`>?UDaK0D(kl%@Su3f4ObL_RP5FcVA*sXFbwe_Y
zqM+Eo1$5#T&#yHdqG-N@_>W;R?A@-@qCA!AY8jZw)vWZ-7w^L?CL>Ey_KkvmNo)DM
z$bYeq{qPEtN#IA4Ftqtg-6M@py*L+)sAHtw9l0<gw}S`$kSv~Nvgxn%O-a-ILdgYE
zZB(g!1b$3CN=;n7aepbtt=$;l&k=s3EEnDyCcnsXlUWf-y)_m&FQu8LQQziees&M(
zn1f9^MM#A-id6j7S_*AEUta!5<ZeZ_JLHoNnA?|~Dq&-RAFsG#-odYPo5pWZ4vCB>
zO30pAY&VrP*I39#WA4Fbi9azVC(QHo)&GP~c{7c7E{PRCnD(XX_}=?|l?{vgM5RFw
zoe($-atiz3j?0J2D_m*^+Gvr#w|LYun<|<wZ^;f2TwNLolinj-|9~L@3Gz|rc=gb?
z)lV?wYQa0(b<C7+H#sacR2ow%Bc8Edl%6IsmK0Y<iH;onhWkW-Es5o2Q=Le`?(6QX
zd&6+@qdOoQVkCB?Vnfw2LNcOb@=9zQ+c0U1XGA${Xk(dz0h8OH{9U6aX6wUH2k$Gz
z^*H`l!F8MC2X)Bqzggi|W5>y#d@`)gvLZ6I4}sZn*E>rp<r&bYan$SR@Oc$!u=a!J
zk3IQCfiUDA8>naadAq|rB-ExOU-@0)&35_cZ<lPnWZ!P;KT5T;a~)_`O|+PD3O_;#
z6NY(Zn~VvwZJB&XDd(dbLGOsHhShplKAr=qE8i)vy;rub54p0X!-gi?kpoKJqg9Wm
ze-{Z}@XA8!(x*D;^DR*y%z#uglClF!N<rhh;vBPBy7V6Y@{!#_W;Ojpi~Ga3eK=&4
z0}IuQSMV{O9ZjY*JI;-iVnX-sA3o$!ID0L~yD8*(KRdLdlK|Ge)L{l{V2fVaB`iyG
zSUj0^T%yID7H5}Q@-_R;8mXW^VTtNIEi2EClJOoq_>L4ld84X-I(;FI7gkC`K10ET
z-A!4AI!d{^ei3QbtuW#7Qo?t_@`GK<lwkWV%0OU>t37<H##-7kEL1{7kYaU;a#NXd
z@ga)dQ&v}KHc`NqGU4F9`q8trKbu})hMT$}5L0*EX2at@r}zCVHO$`OW2&zs<jux?
zX~bAHmY#)YDZCvHBMkY_0=uYu&q^_>|AKv73y-YEB1+#He)ZQPzqyQ$TVj&p(IRmp
z2q>?mhTd$csYGkNJLZUB_D{X{C2AKG`5@Y&HsGSI^qF5GM=tkYud#W#_}YP@Lv^^7
zk%^+Ai6+ldqrZN+XUZ4Oy;zQ#pZx3v59SGb>LSRXb;F1tk^<f$uv1R+8BX^U9yDJr
z1Ztrpdq^nfzeUBBrOI|vPk4XEXTOTLESO~kwjc<x!}Oag7CgiB<$?kPBQLmwwRkNL
zRuF#H=cAN+-defD6HJV^g?-N@D4tZ`)=(n<N8=Kiur?U&*SsPHs~Mak{3+92Y0P1f
zogg-I@{H|N=_ogT8o`t0mL1IzrIm-&Z?|F2;Yfk2Gx80)b_?e(2_qvTyf~2U1_a71
z&Z{eCA*4x&${E>K4=!}6$)b20qq3G+H^IP#y-*Y;$@(sNb}-4Mt1vH-?Wm*|wXa+e
z4{6rpQr~w`YwR3pq-BMKBKX^M3M1wz!*`;tDnCudpQijlCK$G|E44yudpnUaSC{O<
z_>fhrXyCRyZ4v=?H7=Jj&3)@%S;&a81$FpmK{ydpLO*Zg(^s!MQ_!z%)C3y^X^UQ>
zEkSB8tROsN8vdJUuvLdFJ3aUGjyNi`eY@K`i)A?dKu}!19aEs~y)|hxNHHz4PLVA`
zyiVx+j;Hc9Jle(7{0>Jq6{$$4fpFMdmN45yE=Rgr%IZ)3Zeemohv$xuTg*STHk7-*
zjk=9c5)Y4KK`4I%nny@t@kAdIWQ|R{<jRQd?DjrY#~9_0t8Z8=Gji-f6@zwUASM!<
zU&og9#@kI!<tEA)f0usuW|JUdtS2s5R+YXumF`NW*&E{TC6!idCg48f25#^?nKwSH
zk1fkN?nSQd49paS`_TT7py(^*KT*WIUt!mKa@V76iTu~M-@lK4cIbL2%A2lX(}Pke
z1tfky?R3`UnWZ7$61Spl2u#O<rDn<AP=aut-4dVr`ku)8ogMAHk42K>6@s-N2=jWQ
z>Ghc>%}EHdZjXXc{a4HK1FdNz=5={cqmSEe!POE&I>3aKM^%~*L>C44Fd$DMAkgh6
z#8-!(wmuC%ffc!C2n^ri;GSn)%_r~_wM`Vz7uwI>XvWY1FPCJOasT0Gve2u}I6qC4
z7|kWTf_+iwB@gq_P_GO%Rh<r|l{+}Y$KI^DBd77CQ@GAKS9S92Pztm?t;}0B-x~(o
zZFu@wE<W3+$&00_xkHx~u*=dvou{$y1#$^GbFM6bFiDLv-#4NXl}C?)Pd`ktH|!Mo
zg_2|qIm!U;_sa!J6&6yyH>|G`K0ajJtg4dwkR05=JR$wQac<wc+|-Z}@tiP(SY6W4
zTIy^qE?gXgQe5dDD{lafuSA2bq2ZIqj;aN#a1}KAgvtVuZaNz`yz%gP;sp~6cG?jS
z@xIzDEl^3P5_j|VQy`vX)8fUC`wS%-sQRHqX7dZcp!1@oxnd6D`D{nT@-hnd!`*cT
zC(w`?uBBWT@XAap6y8Q0WjW&`#V}k$#PN3K#YuBAko|R|^AvpD49Gs=HUqDB<riP;
zN;e(dPI`#C?%5atxy}be2lJ;EyWEuQih}HmL>LYUN0hyaK_p<rvS@W1UYjd#*PYSM
zde-A9HY2hz?310aRe$*?0fTYA#kb$(EBx>)%ZE+i3DZrDcjScPsxOxosZ#1outiE!
zjiA>vBIhwtpblyG)xkZI1KUrDf01Rr0H9k&g#Q39{{J^-uVMOO{xYd%&XF0FnM7#3
zR!d4kE0@dt;TIxD;BU6FebX|D*aH;^#UH77!mP(w`#5Yy#6zwoD$B=uMkBM95FOuW
zV3*8loL-(d<uWtfabtLNLJAUF-*-P-Q}z47x4Jvo`%`O}FHVOZQ(Srk7hde;R=66>
z3?ru{KGaHAy&pM#{4G)sl`1re+wl_G$ON;4&g<m~`$M?T?Vpd1lT2Jv86Oi3Ju6u<
z8ofKHeCLWH1H;M{t#36t<f;_bWHgVGRH>DJa!@uMzN%BgwH|I()w-p-?*(WOrQ_>p
zcsU1<O!SDnBri}?9-rCUvz@Q|U_eMBA(CWw{sVK&9gXZh>-4->PutN_;F5&dD7|j$
zQXROjL65fSYo|*)JEEZ;8IUEtNH7$h+V_x{+4tuWdM+DVa(X*o7Qx5FML5oKB{XZn
zXfPPApmPnDjIiF0{{_|#qlKAZ#uuLr(!FCioJnBc^k0Wxnv|8$U|hTP?PfQeLTl}D
zxZ0k(@eR0_ZX!1m&ksK#x|bR>>Qm+1qpa>qRL$Z-SGw}^oVyd9wPAV8yIChJ53q(#
z;ysvd-=6C^<dL|rs1bs#M@BzL_brv`Qov&Zz``4KA1<MkZJaB;@FhpalFjXFT&^`^
z?ofMkqC}GSJ;FuAO+?`JYeN^;tBcYcyd-H49FM$`waRq{@83VuN$?~WMC2sDk&UwH
z!Ne}bAIZzgU6#Zlhp(!ysXgNleze{}68wPa;ovDOC*g?K?hgxl+WVogYbyVC&6fSD
z4<ky?04;uZ_gc538_C@pa5U#Iow-TqiusYo7e39?7b;Xf`l`L17!EELmm(%?hWxv}
z_*Nx7y!+N>O2&7srNlI@2u8-_0V`>XkT-4M^(iipt<YX`)pO)X_OG-H-4Q!cMLmn$
zv9v$PKdV9Kc%LqW^K12rC)6tV%XDAF;^(;%FdSxjmhF@I+<Zmy%zGpnj9zO@e~pgg
zdk48{mtW1edB)~$xUH%oWv<x`%9DymBrmfMeKhA-1qR)fl_!U9c5DT$pe5u@>3z0l
zW5lM~>WX{Z6-$lWsw4axa#y3K%}_R|(+@n3n>!65`xdH472bO*<!V^v4IOd%YebAo
zi#54~x<@Z|q%7awsnoN2a2jqQiW6SL@ZxDYTF2o{Y+p}atDC)YZwn>MPu^MKO5uCC
z=>xoBmMPtGJ~VT$g7t%r(yO#zukG+SNpH<UI0PDH*N7n{O%2imW7O6c^8;E=Z3oOz
zj9$t>>1ouSt&cqowJr64(5A3wRa2lig^A}I(HwkHmp7lQEn?)eXl2W5tZCzD)@Bn7
zGn}Kf*h*#HnHKf;ux*JF;xhTK)QdbN#^&|%P&OqE{|qDvzbqiw>VmCk(y2F-&9sjq
z75$7dT+#kA7?y$uN~V#EL9w?b!Zsr%R`Q#1c#!Sa1vTNVrDSu0?d=PQvjh+9p(VW~
zS{|egejOmT*)rxq-fB1S9v~j+%Pw5~`eaVfTq;;b+m1)kH2lC#f8-g<Y>bija!$9F
zbC)FT?7A816`5ZYJ}{dmLapheDf(;bul8xJjpsaGx})t4>CuZgv+Kk;(H=~@2gJ?q
zwFhdFF=t*T<{1%a7A;S*g9v2H^Q%+`JJx^pCIMu`@0BJ%d%r@S0Z0DtTWo)O(-YmK
zOS-$oEG(-=+NaRJ7+1|Z#Q4$9`@03$TGI|zM4RUX=siWzwRzL@I@yPiXDhevZ7G|M
zE3?0!vAxnkm}-+f&*8TtDSM?y1v~GK_zgSh{*fWU^gVcAT(C_TUwk;p7u-B7rU_yP
zVn+4!^kJ|E&Ud5vV(7C^fMAe~{U{di#iDa;LQCCni&x$E!Gd)_URch6h<3s08)@+i
z?v$aF?d|zeQ2=P?sBGt7d`5M+lheKmK~;lH>@|E0G3U2xlaCzt82r`B*#vP2pOT2T
zSu9vvaj|a>aax&x=x{owjN9Bebkh8Ek7}rd#%u+LcZvdbX=troFiT{s*R2Aiz*<w_
zRu6M*?6*>-MnZ5)ASk+O;LVLvl3N|SdsOqxL-s5Q@}1#JIsLB{J4z9$FUrixv==S}
zWt)iL!we|01>2CnHE`&%md!w}Od{wgjM<BC>LIm%Go|}2|Ar*CNL$STf%Qx1%-Jl|
z7}a_kB%vrD&%D3rm3P{|b5AY68($|2Cw{cdNUUsFxy9=Wd+Sp4z|Gr4#Rg?o#9v}9
zRIueq-I;>m1bHkgmMcDi@m>IPQBGBRAo!7GSvlAQ2Yb{m#2D5(A5Ff~CjPpQ9=E)X
z_aV7!B~&%LmVi3eAty^K)4bDKN!SqIp-=AgfNh%!MNG>O-CNk@$30iI@d<~TBxP(}
ztRKU%q<0jXr%>~@&J0iCx}bRFYlL#htpTZPSuB5GsVh=Voa3_z!4AeOsH}|>_Ee4j
za8hm$S;RZ+@0rsk8%U%{l_gMiKM(UFdRE0N=ikyz`%B1Y6y0RKamwhE^`xWvr|u{x
z#+8IbeXyfU{;*xC;(qCga(Pq$vI_%$6MK+b`wp-T)yu~d?+M#bFDH)wfX%a;^L^l4
z(QpfU>+`uwEbN%2gv&XkV_O59qG84rwN0$3`hoc)7GXvh+bc;6-F6v3^HE~={zy$n
zfzxg!PY<mwNBPo~rV(sP=85|Gn5Gj|_Jo9}N*h^AkviXs1^_kZp)V9M$vD{d;fvQR
z>cPgqK7FF;2iKIq^<UWz%{D}%YzSrJjdz$;+YVr6>9wfSs0T#i%FRmat?LvF3d=a;
zEP6X9OE<Stxg$+&4?=9-R>oF_EOD{Q)2xlL=l2NzQ0-`cW?1Fhkp4_zmm#NlA0EUd
zE~g&-xdAWInXFiNN=FfEjVbm%T^6bL+|E94+@cIYN$7!+MXH3A!-O{EJSOi0W%ljs
z?7FtePM5Yg0K>5S3L@ie4dr54+nQwM`esXRLB;i3-#3Gn6qv4rI*=ZI_py^Ap7YzI
z3U{k#Tc6ucw>{(Qshq)}0y?!($vImw+g&?#seu1x;P_$r{7^mxu|4DQkHe0v%;JG(
zl^k+$cs%Kb8H<!m`v*s<l2<U4rmH?Hsha8-`f_K=IMiTNOj~F}lBedTA|{gCCPS6F
zfSN#P+@hB~J43@XvFk<h2dzPh5yOi8G*8=)1UWq{_jgAGBX%&6uvbrwwCs4Uvj;6m
zzi40FntA^7Bx=_B=k#tGzG9L-PPWmzll!tJM{TU@B|hPy7THCMsH9w(k&GJvJUX0j
zSXmhLPL`5gH(20J<}5ac@d_2&`-KylahCU^f)75wa;9lds(WHCHl^JDa5CQ`%$EJ^
zUn`gXQw1{*Kym3L{@x<(xU2XsX%q+u#W;*0@`JK@BX4|+&D7T6DkoC6Vjp-(=PsMP
zNE0>kiAUnnqFiylg1zz!y|ATRi$#(c1YH{M6Gse6SVU_mBt3*<<Pn}QvGE#U#>TdO
zu+OhZ-#g+b7&%Wvoo)X8NOR{I84dAPXq{Ur9|(daR%wKi`|;-h1lvw!FwZ?_(H!Ru
zpML{)QMLKE>f36G?qQx}Y34r_2Pv4<#uR0T-4(ayIQ#-5U;p?R0i>mLQ7TE2^=-%9
z?_%BKqac;zHH**>G1SZR-*P(*?ejJelRWH4$4S3W00Uh?QjF|R;VYlZ`3Is~hw(_A
zwx0-vp1H_pM9TL%10M4nOqfKR3*&sz{zFcAQ@1Hc<{RO*@iKbaBoU#mJxPKmU`tYi
ziKLL5_n}`B!k=Wnc^*%BKzCG*W}ZdKpO$9P2Yyx-g}6}4%aztx&ZpGebx5ojWmhP?
z0rPiE{UA`G;Pv)FYM{Eg(?;6Lr`jXhfv@Au6aZ-Y6Kued5z%8b3_33A#W1gC$q>?t
zEQHf+<O0>k=wM;1C-9N?K5q)|l~++(7HP%jmRF-mZnize@_VG}85k1zEc|z<D4#?K
zw`S-iH61I9PorcPIZRq0<i(|h`>Aq``xV8kf=vC%Z~?XGH_uBxknXO7-E~`+plbdM
z3%Agv7c>XqTJ?a&O>Ep1c0jvVvoRYfQM9=ahQOW|-?1W1L1+J=SB(`7m!-oLT>7Xa
zVX~5iM>m;gnOhH6Id80*HV>HWBK7sabnW%Ti9`@mr5HXI%g&72*7_PTL^xdej&EFC
z|4CV&=<@%g?5pFV+S;}iPzLE9q(M3dkS<AS5EKv*0TED8x<f*mK^p1qE(s-tp`--_
zBt*KTVF<r9=Y2w+_k8Dl&OiKyJ$v?Ed9ORK>qOdtxwWP?KJ(pkN|jX8%;g?spxjBo
z2chbNqYF@LaVi+r*fB`PH5D6+rNyg;7*w!gC~-VI3-xV69V(;)S}8-<(|{4W>S|64
zoqrvA3_Gm&{*)u9nxJw00>&)^9XxXB9})0kDjOYQqIVE?W@|1EwYDS)ABtcNwj$iQ
zC$)$8#H~eDXn$7~;Y?MVH4)($!_|`@DWvAqQdQ;_vBMh~IGM0Z00K$_Q}0#TXARS?
z!9$Op)kBbpa&C|8NlXWIQxv3v_QFg<hnzPiXK9xkCU4N^+vgkcZmCEX#P+MrMmQ0K
zYCSiHk!jIXTe%NZd&ip@yv!Ea5)N|q5$ESLJuUJWrj5K(Y`KfRIcrwX)Gv^F2V*U9
z?y9Ye$FefbDkH4*+9Iwol=#;ttUd?oHI2PwR47L<wU(BWDL*t?CenO&W_kL<La3)V
z^ABUI;f~BFiA_PsWLu&*VZl|g(jeca2G)>iv4h3$HG@3qzuL0VAZNRk_d#rKTSzhd
z)rxmMe#v_3D9DQHt(MeI-ERuDAN{C1ck#TfG=aRc)Gdt(=%)77g!!<U2PCRqdGnr+
z$88n*w1KV$D`^-oTw7=ms7`fn*%nDW^TT>Z&;hSxT?swkI1@(tB|cJ%9{O?mr${d#
z8;b9;h6eOBkPUT@U>}CgiI*#UEU~JjQHWxU4W4}yC3_2waVFnhVY3h}qz)kJ7<yA~
zsQ|z~=wekvr+#&;S9e9EHAEj&bLeae+OWd=GbZ&!<7*dtSs^{|!;v&8J~m+;Y@#=;
z*WfG+H;0a%PHR&wFXCh$<d`{7aa*U2HL!BhO@2b<B&+AGF|5e0e7$8+AO@4#CI0-{
zo&F9ySz8$)7f6ovsM;tRYEpoWw5eOX`A6cj=#RIu4qvh3V$4Nea^1f}YjzdUReA_2
zWCSIm7idD!9|%0F>$Kz_BP>UVa62!-5jzyisi>ejK~0EbkgH7>_h*Z6sDHk;9_2E*
zNUHLWNW4d<_<6Z|15#2Vp$Lx`?oBc2_(QB|Fot_{O_T^5B4n4MB;T#{j{1=Gr3M8G
z1`Gzajt0^|g-_TJLc36PS*s#@-kuIY$J52tb$1d^I^kd~v<XZY4qkFdL%XP)0OX71
zDcgumD@i0v2*0vrZi@Fgj{+Y3@Pp_p`9*#_g!ElWR6m57j&LHcdXgto+NzciU+eIw
zJt=li5)-WzaT=9`$<X#c&4DBW-C+$^{?QY(>v9Q>0mR&dWH73$Opxzl1GH#&Q#+#Q
zmjH9sx9fyaH5%HKWGT0^UTY$r>w}!I#O^XGXpQ$by9d6KdW1viMX^-bjk&P?i~4jP
z^3e{rn0?bU{?v%t45ho2N8xFNA3vGecHKhWNhwI_N;6DeUi~$;=mYa-{$@e6-%The
z<?T=kf(M6L;nIDdj&n-G+ai-|A6T2|Do#bEM_7=`*~~X&T1djPzMoqW?T7c`U%frV
zTcLHty%}>JYTe0thc=LbApLP<IgPo?7Up?(w7Mu;4!j+Cak`zBVtHKTQW?iLapW=_
zJbeQ1u+t?^c~@%@Q%+yIttBvKR_y8y_u4ZuX%IEvfl>$T5Df9Ghb_aGp^ObIn|CO+
zX@G{Y?Rpy5ZJTzjRA?q?k#1!nbvc@5@rV@R%J9g<(IoOLLEkeCltI1l_I`wC{f3}I
zT<}Yo?R0BG@)4me^+}kcW}owH#a-3V0mE&5FOQL<+Ub~1;RU*x`vv%rDvwMm<d2{(
z81rM3O589D>#WXsOGJS82vw*6FdWzkhw5L35ma&BF-kkK0yr=WbJT`3i`=Q910Rm+
z=6tb-p93dja+0$Ls8z7gj%NF@TzaZ*bl>#Fq~w}fnDYjNz5?o4zFo&~0|>tHn9m@X
zWUiT(7xwhm3J;y=NWw?;L4^C&SWM>ctZK+iz;96oU`;(eyJTsiW+L{f8QC}Wxo&@j
zEEg4VtnU(?Y7p&r<PQZYFrk&I_&Hw-8%MG737#tqJ0@pTpMK!bxoI6|xW1re*}2kh
zecA~d7BLO=*d<NjZyE_Zzi)js{O@wze>uA7p4>_6_4&`h11CS)9`}lDbpYKOV5C=*
zx&G{DYCtR%1HzVw3k#_%RZhGUOxu+8B^diV&eBF%7v9j2?y4Ni!;&I{CA=>ih=>kr
z92QT1V!q^H!evyc;HpNy#AGQG94hQ4e+R3-6~sn`hH<8-G-9GfDVI}6bG|r9;N=L<
zKe2;6cjr8HB;0S+V&H>^e^+c8r#Qlm9H_qLDRtb6AK-sK=WOm)z+g>@cHe4V+G9iS
z^8>a5C|1E{%})<{WJjiO#U}|-Jwn@%uZ`<Cn2cDHafRs-?*{It+QfajS{y!K-6=tF
zq=%If{73<V)IWyrlLX|OG8#QdDKLWs+G8=;+(TF<b@4dj1qsg(uVOJaOv4aV(URlm
z(ZtRmTLn*)s+T{YojlUqhEtaA&Q}2NuA)6mv@f3_fF^*!Zy4@&#oTNw4d>1Tjr)cc
zVTM%<;kC#GvuStv{?P^xiYY%Wby-LsVWO$wB-LSF)3|*+zj<R=CBWcJyp=ttAFD0Z
z=A3zEEKGlo*`?GL-lib0<QtkP$1rReawb~p$CJs^!+jJ|$)GLwjn#Lk);8|B`Lfy|
z`MSee6mz6}SYl6=<MDhTkMy0dKk75p+g4*L3AxB~)U*f5-@t$PG{7Y(T+O@v@vA8z
zx)u6ZNCriQKr2v-w0f3xF;6g^%S{ujKE7GeR6smGe8liDJgk<EXI+a^;Z`$LO9!#2
zlG=*`(_OZp(x}inNMNW3_@CCFL9vq*fpcYeEtWCeCc2q#6M7^ED5d0D8FAFg<I7E_
z{Y*QcEU-q0R9K5Im<c*v;qM=Sp}{dM0J<nS!iIs$MD@z+AR}uwM?UzpT{irP)o!aj
zB8YoU_h(qg&fT8TAa6r}!yL%}u?O1`Z&M67K{@&u`<L{ChICjFG0RN|d$`A$@+X;|
z$6+lr^&CbJbEtt3MKp`W2Qqn4>y$VZOBGec$skO-XqUKy=L>tD`w|LYW4i85CC2s0
zZmjQ1yh`Kyx~xAZy6{mS4+H|);LFFU22>054|CJRcm#W!7KG2wZAF09^vSD%VS6k^
zdmwW%VL~Wqz<l*s(lk>1Bgnl@RbO>CczIG`*m*~eQ#U4GJSBJAF&ayNI{KA%8pC$l
z;%WFE*zn|8J(1k!qMZ-IR*$nZ+D^5ZAGYpaln%HHB9Hg2%R!oAf|U=%IRV4dv;VVL
z6YC1232pQZ2UsW5^ea`uKDMOM96ibF>nH_95{V=C)0%JQgg+QN8uurj`u3p%j;D&e
zZyE7^hqEv2M#gZE3Ugv+Q>%+mN27I5CG!xa`EY!~1*atc%&6Q;Od6LbH3O2T)soi5
zpq3|edvNkY(G#5P74PX+ozc8HDVja1lKm^ErR5IrTt|>@=V;k^cgHb$;0K~}WmwH~
zTv!2m;Kd;o_$Q#F#Z;&Qykz%Hj{m5W)}zN*0t|ilrDwd}0K`~FIQAt-L)LEOsF%IP
za#K<K0nz@Bsk6fx%O>~GfnDA+svoF`e3U9t(I*ROJ3ckacB$m^mw=qR^`^hJaNtiy
zI{}9G-$4Bfyy&JB<D<9q<&o$-{AYaF0{vz6euV1HeaKKcnCMdP5PBfFKPM_@HM**3
z*!=4keR>+Ptz}_9ek}^Zv6!%3WgS{86t+O>PMCT2z|bnMDC?LA+sAp_S9deX^%Mf9
zV<v*Ut>tYN6jms0kr|(@Q%t`1%`HHXw`S&kM7|0Z^CkGi4DH_(ULDtWczTm*C5uyW
z=e}WL&Vg`-@)h05$M3msAl9guO$7v!zoPO_8-9=#Z`&6Wh2n7tzu=Q~pMu2qDXR%w
z4L%`YjQS<M_PU=mp2(;i55Y;btxQH8pIL`9^ftM(O9&n%jJZTcU2tn9E49NGx0jK9
z5uq*h{DH@_S(daIrb!%zcw>l;4S~$)Sf@$-6Yn)us$V)n<2gQW6B{twE5hCHo6E6g
zmA2j@X|cWc{mPcM;X8$F7M2U)Q?c)8lf=+bYV0Hc8O$f2)m9PV6z&u#sM)60L+%?O
zn$`3ceHLKa>A7>mBl=$k3Fc$kW#NuUe7bY*d!UCD5>}W-Kx3<Mnn<;jJ1<B1=7k~S
ze#hN>NH4+sn7)+}p$pxiHD3?JSKYrF@~(Uu>%O%w)dN#E>#3Z3{*p*Oo5)2OnUV<W
z$ztFfZCH7wJX$OsUX{(0Yu>QFqpF~(yB+nwVG=CwA&sT_2ed@y&r>rJxCt$TRQ;$y
z(Yumnd{gytpx)by+he2epE95yklqrVgC^G$$nwg0jEaq_)GLt)aosWUlB}bz>(tj*
z_;>=qAC!#@cw&!SzEP>VYX89Vw+b5|ImZk%ap?icmY&Ck9y=XhR-3na>6gAbjvG2Q
zWH4EapA+CHn#%W4D-J5Uhs#PR+>*?k`gB-aN|lE`cEPw7h1`FhhosS%v^~2??Pu!6
z77rO)n|!cXVM!rx8cncIZ<qe-Ev0X@3U~dj5*kOEHB;mo)KQ-D4$rBw9LY6VU^o2{
z7OYEkc@-C$9uZ8thWxnS81yMyvIW#>6;5_@lbL!57Bwx@oqP8536DL1w4PFa#5Wdi
zIj(xTW0=7FWK|4g4Cj4r5IibY6V36Qv|H!LduZv~1Pu;v`aA>5!LN8y8QmR3sC&*-
z4Gj!6svzGuTC;Cs666?z9auk553ElwvdHzuhg502>e9T~N_M+<^E?rxT7!8zuj0@6
z-rY1|+ofk52(tylB&T|DanzjVl&30i;LeCWv$+cwdc2p7;W<L3(g7nlcbK_98mk0*
zs`O2zvA`K<k`-J{vhGKXvxcV^LCdctS;hnDf0QO!acpYtm4TeL#9NfQ?+YSMsQm=5
zn(~#UI87D^ex7jtaZa|)r|?ytzCmEX(9q$Pe=n*=N4b=3ax;I=^?&1Hw(K&4{E6MB
z<|fh09&aDgyH6eil>)?sqDRpC354bL7*7SbMCI1RM@RsO{IQ?NtZmTel=7_n08d_j
zT8KdTF(cnBVa{ANh5R+S4QVI^ZXii=)}HvBDo>#9<^cX|kVzd2w0{)ToO474$)FL8
zEj;ldV7iI;@%={AD`2^!Yh@k|Tq72~VS{hu8_R}v9qk5#zg+trvgJM{N`6|E556m!
zCMjp)Lqp6raFpBl;3a@R!KMuheN0$>T%Kk=wM!q;iqTf?3L<m2e52VO5IrbQt$foS
zk(`y8j*poI>OJ0myH}kN#rgw<)$9nW%5y!B$*BXBrwRRjdUV~b;*QdC*r#AxVcfl;
zetTg8R1P;u|D0k@S?N;xs6`~MXiiDh5L`#Temw;<P&cz<?h*Wn`alhpV*^d7tOg0M
zt@r@$fV_chHhu1>QkK6Y?Ayw8n6^y{qq;+WFz}Ff#i-z9T=DgpeiF(b^NATKtE}Qw
zvX94%ywA^m(o5Qs?L#9;hi<pE4NDljotq>LBA*MpaxJUWnj<|7Uh^Z6Pjfk;Ls0(S
zm#_^$H+9i+QlKPGlrr-U*Qm`r^ExlTlmdx_+)#bT=`H}Qu%#<qdOWBx*I;Y0@YM)l
zDekswj|yNx?0kBBQ^rdz7Un`_MPwMLCM~p?o_CLg;=Lfcvvlfk&dq{wCEf@00S#@D
zWQ>&z;|2}1qlkTqd~aTc6PWSUqqIZ6-J@Y3M_f34@l%b$x~RlMD=mRF@OsC?#4%H(
z2i-c%|2jat-@(b>Xbo>N3>G;c9>7%D!ab1xW)IiqJV)5#uuSH<6@{lq>?tgJH6cX(
zwuNR~Oda=!6!%~vTlbD)!rX3R+Jv#XjjqD1w^+`95QRdoagI^$yo1XWx_J4o?#EXs
zfwZN^b+5#wstFyx&p}~U(-x>q^wK~(_)jWkxo2?@?EY%{%0EJJFr<z4N6I%$6&s+c
zf5&6U``#_08rx}Xweq%udq-B)LkbQ18BAy@Uud5<BWVV(z8~KoBGT8G44QUP;6>gE
zfhRq1I3qv3A0ySXC(H~pOiAaNNy=3s%!EdH44OYJ9_ow<WL*IXx{_v<163^to$ghS
zFg(Q-8#GQNJ_26E>x80~2p3&V^H7*%lKdZ-6!0(k2iE=s*>~JSTX2fp`@brr84<P%
zMBbgPfULqzt+&1kuiB{^y!7n6Q{6$d9J3oNEzh9x$$FofcTHL?|GsGH=LSo-yz+LB
z%locIzT(xBJhgQE01*8Krr3ufRK4=O#A*#U+N1|(NTWX5B9t?_e!;!d_L}F;x0@&5
z4ROuT;i|#}r!7EcfmFD#)m2)Ppj>6*+(=C1(&$3MmT5}*HmV{t%)sR+yZha6>%39&
zT<Dju+pgQZxOZj+5chu0pQkYUFoaMpB)QPGo8e1#?p)=_O6NIbNo4K|GLd1vZTUP9
zlcisB-%~e%X?*QP5~wlxoHP?mC?D0Wwm%c@jEHH}B14phNx7Dhu@uc8f54y3o0VXf
zntAzeK&XGj+sdm^cz)H}Y4)%F^E+%vUsgK4ISrM=yMjd8iQxS|urEZ%fl;JYmtp_i
zCZwuOV#_w-R)T4*fk;m|KD_G8X26M4R6dXNMUVv4`kz<;zv!>XLgfWQib@-vw{CQ5
z)%iYI1GpAxcxozg>T>OEt;;B!8LdOTo*xdRSq7Gr{mu~}>0tl1{Ax6>0*CE0`d`Ak
zy&`eeV%s0=4*+F_J1;RW9>FyF=d)f(yF@tNm12YqMyrz3aA(AU@NUnYW|ACyMdDJ$
z)4K`MkAYAdi%*Wx`vpK$D*)VqWC!AwPD_R;Ifi^IhB~G2{9G8n(j6o2sJj$v#~N)n
z@UpDr6wQ-0S!erNxI%p;pW*5X#6)Eg;@y`+1TUI`5`;a?nKBT*a!^k8o=;NrYy>uW
zAqtAclJ)`=I#=NL--(GW4S=#pc_#@>0xlVr5JG}aC=RiL;!pN+ic8R|)gX&^IYv7S
z`9WQnVlQJr%Yu-)w8nFEcde+bG)N|RUeB*ejoVFzwU3_zvrCVs(}GP{5k-Ub(544^
z9uPkCPF&%BPSdAg$%wZ{puYKqW+%x|P%Zuru~}jRO#tWALy|AHX&^~_pf)h~d@lPi
z!0ELcr;r?_9W;=uUj^zYA`a67reXzl&uu)7(fe-CY>#8LElD27U?fkoOiC`|FXur&
zGAZC@<I1x_sV8m1OmbZ@CiPaSEM<?!$6AF%=i@%PRfxBT3Bq{r=c`MkLgXLU8n6tO
zo2EfLZ>i`5;>^hAGm7*J#ktIegp2F^Q&nM(0=}d78fmDJdr19F^#WW!3KN&&7yH+G
zSR=2(7$`<)yTf1H!?;`9trRn0J!!r+je@-2khFQEX4KZaGD^v)`N=*$AzIGCDJSCY
zeurrLDonwtbgHb{E4qeIEh+hv>8zv9)2a`ppWpQC50KEm=T@d=&_cE+$j!vu`D8Ah
z20dUpcqV|P7q;omkI(=}EBmqeSaKRI%9(B$&ih&xMKET=AkP@qpP6`U2k4n#JcP8!
z3yu1h?@<%f(<2%AEOcLhV-TuN%l*(R+zQ~eAIxJKOcG(Osi-5$v(zKAtc#OO)68{7
zv4`oERg>l9WDd`e1oa;Q#@Ay4dL|x4TeRpd^W=qv<J*9t-XQF8uXFMgBIWp&sOym}
zXWZQ)^_BJsTEb!^*3F|xZwhPdfsE7=*0HCwYvl4}EDv!9)CCsl4|)ETi(t4jlkZJd
z=A1k*vSE}TzBEH88BaeQw7`nAgp_B&h6vAzvBZ^4ck86m%q9Mik*~bd+`^S^SYX01
zZbPsWi^B<I5d2%w33dB8(PZZ?u~m9fM--tp7g__&yU=*+?XLp$2h^Ay>)lfA6r~e&
z=RV@V0F&T6kgXo5>m3GF3!ZCN!j5FC6*;pA>tk!RI`vYI!t{W!47*yV0X0=+(Cj>-
z3|nxel|G3>Nl|-5lQ;KD)_KN2-txN@&zzu|4aKy`b0}j$Q%Pv<jQJy&@*pPTOr%Lz
z@TyoWvh~A&4@baHE8s;g4IfnW)`+b7n&!;AkDqD#xF=#uE7?E1QXG^Br!smiiI3b=
zu}%GuhWNNl!Shin^IP_SQkvjHlKqZuth;`;*b9pms`=9HB^Nc!^+BiAC)3M@^s5?G
zl1mZ+UU@D6_vG;o-tG6o6zJPkx1O3yA=%fmc2i)`fn-kgLcN=TpJpxU@MG4MK8OvU
z{0VuBo+JtgPx5G3_kV#U2e*iW*@jA)e20c;|K#KZ>Ym34**;rTXmYUQOs*Msf~irm
zVdjYP$c|O0R6{f-RbeJNGJk>eTf})LaR$8I$6pG=Rk%VSG%`O^w&d{6#%p^TJj{^(
z9Ba-+IWZMb?*IAPV`1;N8dOvS+4FOZc~Gz7Wkt8o)l>He5hvea!ff+6{#CemTtTrm
zQPD1M^&@Rr+a@8`CwN2=a+vnD^K)=NjLtno*cXU-P|NU_?~Zf+<p7T@`$a1uMjo@s
z2<tb1fmBXTy&5ctfl1|R(=Rue;jwn?!nHGPE5RSRFwG<DAl)F>(|&^11Y^H-y68(t
z-I&k|>KKwU31hoN+(1@op^a76h8=B?U4B07k%q%4&CvwwVF%VV#RM${5p4mHOn1vM
zCyaoYB(2W9ptq`I@-#oLww5Jt<=fl#lM};%s>fi|8Du2a^Qj=g9Bxf+=rAN*oWD0;
z9>R}5e&?IwQ*MeyY(*K2lD=t;7*Q<0E=ossv<z0c>~n*f=dn@whD0fM$8&;|HdJrv
zCg&(pcr4P9ya}TUEKl;#zHk);L<N<1NSJFC39$zl`ZW@nQX9@*qvPf0r`b?Fm4IuQ
zz)iKoI7u1Z-U{|TQ8$qBJnGi)vitcc4ntbb_;nUWqMQ#?0|Q=9cKA>55bI}Juh!=%
z%?ntzc{?)j=0(lISu-29qDIINVWuMvnVTFy9hM<!p-Wxzy7$=D%3r_u7pdb_Jgs7c
z^CT^Nk-~@ZC=#gwFS2UiF&Y~UIIta)^MJBQ&(*i<$GG`OVhuebsh4#%7BllP&sRI0
zRvZkZdm|ZzvlQX-iSgb?bo<Orr7InhOcnm{l*3A3$~C+}K0*^*Q$?2Wb~K%DzQWHW
z-c30Suf=F(IKuI{<w)6CvCp9+q(IL%RUmZiN9{=F=qUh?2T3DTk-VL5R3-)Q+0gf~
zZA-`a3q1+ghwuiaHb70RiNxANxw2}h{b&nQ7s>B?ifIwcc}0MC#>HzdC`h#@^1Xpp
zIhj1A=gHzGsUzfQOY*2*V{OXjad??UAOU}VN$3Lp7R7mH6d`W19YI>sBTkJJf8w%9
z8YIy^#Y$0I&~BcRs4&9obW;}v>E<_bAR!4i0_-;|8C05OPhv=sSnR(n&3e6UKaigW
zbK%o&FvF;=#TPT4R*wfwKdoMrJAZ0Nj6~ljJh0l?(2^QFijw#7rgDDXh>8<h{ZH}N
zEGYg;TD%3E{JePbE6Uy>g0EqronDaV`PyvgdrzxY#11#NM~s4ey(C9;Ei3hw$JLxN
z7fj`>$LIs-JTb)f<AVs8V0UT0v3bKA!sLa=^Ow^+?OwPK<aVP9W}9=5Q;>Eb3R%b9
z>3$PkQ%IAB&EP|G#zu!=(m919ucaTeZ}4_d!v}Llu1U)9)qFpfG#U4S)2ov#!Qo%Y
zvr0_GIYHflns3viu1?`<0XBTsC&L-S@5t6yn(c7w4=8lQi1uwuk0qnLV4lu1Cj|3J
z?%nQ~GgHRXEb=~C5k*IGs%PXZ3v>E$1fdHn#C&8XeJw6NNO^0>)H{gWLa?N%-0(xr
zP_$ROEsgk{ogb#;EPBO=wnT_+e{2hXO*?G{@RUDc`pFn?NIz)%@*lnSa`Z@(Jyrci
z#G=(85xexubCxs;DX_lq0XK`<WLO4y{8MhCDA=S+g(fB0ddJiLVkQ_Xcjlzu+0KG*
z4T8+(U6Q~g{@Jz0{B_ug=RdJGTQD97f~?pFhv!FM{tfr~M{yXZEE>R2OS-+bLQr6@
zVK9hrwYiu>ObhyoCYKSS|By=9^E(FKdC4I0?6df|dd7m?*M60m6fy9T264U{QrV2(
zTYSu7=I@3OvziFwu((j+zoTyomp8A{2jRh+Jz3pf7Db#xg5hFXdJhsEA6b6)@CF=M
zIC8$A)X}&{VjkgL9RB3>c_zAAAMuop<l8-*Ss0a}jmzqDSk#km6%cV9ugvs_bNjqx
zaokVQ!gf88qK{Rmorhbx8Ty~5=Jh%!C<};e9mEf(j5Yw-*dHh>$}bkGZAhbx$GAWh
zT7~Z$%1uTsx)AowdPG_}a8M=SO1NA2**JxnqPnZrAYxDbQ}#-JbY5$PaO%7jbGuTS
zC|R%k<4G2yPSM-WT|Gx!p0m!gp3zx!{gwlsTlOlA0;RHbZx)hqSPVYdXK_XZwO{Z`
z0UjlVIDR?upp)Hq$5XtrE?29f(1i{lc=sFfa$d{{!^?<mKABF69^3VyxP<&?caG2O
z4JlK%#}6AZa7ThdhI9uJw>9Ae%i(;f1{l0U7QZI<l?ZI<*rQ&j3M5XJOZF#@VhiVk
zRQ0|p$%RK$Q<@l9awk?Jl1GI5wq8{C=eZTl7uk&B5|&R@_uR<=Yh8<LR1n~tvZa%k
zWAY(u6YJ??<&Y2koWi~*KtMk@{zPgGAEaylVn~l-rI&(l@4w1l&a{VVE#!M6A;_Rg
z-yoKd7?ovn>LKMl2$SRK(s~kx=LRL?=iL-91#U8wp0=n+fhcg%aqo~|WPqxF12ONB
z20;~!t6}XrTMjsFT0QdI41-Qwx+SEmBLm^q+*5xPn1V!`?`X)*=RBaUERQ7Tp6m0*
zu*?ceY~;#-&8sYe${3AKY1Q1<S+3PuzNx?_p522RFrP|d|06}zufx4)z$&jfT<_V8
z)ubmvY5$1L9_8rK2y)P%9D1rc*?Rg$PpZm0t)eS`xj81S_Q+}yRVS)E-YL5F3y7r9
zq1^RS{qJ7ZdEf41l-SNzMUtIst?A2$Ya?yDh2x!{{7+3%G|@hxPx!J5S-pkfBe+;8
zfPj*H+Ijj#%am-Fn1voi^|x#+u1QqdvHsl?3G{xLfU$lW%tX~4CTVX((HP-2t~j$I
zwlkx;tp<c^v_9RL-00ZTunrWIV#ICZyC8eu$BW4WnCY&th%$AkaBJ)#@aULId7+zF
zeSJF0Bv&@tc#-S*@qeQyJuyR3sF=+-8`WG6_q6~d;ybgOLP9_ta$;0%8OjfxpHOkA
zwSdedu)XiM=`)EN*YNmG+(A2kQb85?D7h^uj23QtH)`bBG<gM1)%uux0|S1LQjof)
z9Y#wAh2o9j?r_?XZYixMWkjVE|D@fAPaewXtt3^~h$AcdUDIeQ-+h$`aa0DB=FbEA
zVcksMsi_cqU*C&hjt8*PNBG`j;RM=!Bn{q_Tt3~+6n`^dRLamZd<9<16spg!j0|#4
z)4Po1#!NbQqm%&Upm3dH*4f+lE<C4@$)Jb+?T%i!!UXeS!Tb@s9)PMl8zKag(wZX}
zUkLNn=602bCK&-)kfYV0ZrzR*7W6Vvq<-Ll<FgVEu<RAS6^r2yu&KkR@q-yaqLX8^
zsnxitYp(6!nh6N<E+}()TMu_%^Q3CG4`J*F<RMMc4nBU=L;mWbtJ$~m;eaE#Lo1@|
zx!iJJI7Kz#ie`XFnG0W|AetCO|G9YkbkVf1lp~)nzCJg65Sa^T%f!)Pg-=FcwxGhr
zlrr)|os;~hkW=E%D{LCB$|ZDoFRSwsMWkp#f8yN1*t|OMqMP7}-GGg1r~;+3!RrLH
zEyvD{Q^?ibg#sRdP^kfX%22!};XY#O{C*2r+fPi}_4$5Y7=&1s2xiOY>v{AmVu0KZ
z(-SMXDcC8F(waoegclGK+H8i}gjVVhwGv*l2GxmdaZJf#tI<C<VkQkRZp(FLaT1x`
zepv8Qmm1Ev1>k#<omdJL!iXqFg!lURwQpgLbvOmDn2V2yzpAHy%5aly{0*I6C%^2?
z{piuLFEs-dA49dUW663~MpbBEcikUa35@#1%0BAkZ<(C5LLF*Zfv-3VcXNnYgm0C@
z2S_Z3I~NBeSBdm}Se@k7*(neVyA2?@4RbN|^zdz<_^q1opgtv`h{K%9xbdB!t~pgt
z`NaMDr=ugI@>&kkuHD!AH)!8bPFJx`=9cDct9lwd=Lw!r^e4-72zwD`&Z^;hn!is#
z!XCvJhA(wyV||m;RNzi9MNI|1RL-5J25W1>P+#+)Q}uJ&aPvs>>~JkSL4C0chDqjy
z3eg3x!9zNe{%uU_D+6n$G<yjR?b>OkC>yc{s4V{V3j%HMhMgyd*9(MPucKsB3}$k-
zrS_iEtGcxP3<AZt)TCpp*3=!3YLMxANOAN*)GHYkcGP<bB6)#97%ntHE-93muFP~9
zj}sM>oMvr_&d(n~2($9WU}nWYjpk5W(-Ot2KwG%l<1B#OR}E>?OY&0c?xYS4bGuMk
ze(pZ-tQF`v`3{~x<J0j89kdN}K;2Ku)}}xwgW7fv!2wMDbH@|apLb5Ox3i86f24U)
zcoKXS2085&qZ<3vmZxT_d~GiHJGiFMViK-b$-qREGR=c(MSI;OX76(OrAS-d4{eOZ
z3|n#B`6j=bVB1LwAV8BtkCMyT!;2>RCz6^tw<XN>5lqUX6OX5=pMG%cW{T&8$__UT
z22gqYY{i~@+hi)2;$9I4*~#xE(2frijNYvSv|%|Fg$z5e+AhQ!aoscS7Gc?ey5a1L
z++=OMJuL3(fqXL{_OhiykqM#!<`QXiVMzZSj`P8@Z&I!RwIBye<4NDg@2QIm*N4q<
zx~r^Uv~t$}k;D%Ebp&Q%5#z#trnfVf|3M{TBm}3;eOLdE2@U4OJCO24aS<BG@r<eS
zT@vjUA<H5mf8ub9amvCgfExEBdmv(zDX?1{mny>6f9}JNG}=ttwz)GkG@Km25gNp<
zpWt|yw(68UN4P$hHvr{e!qnEY4-c|yn91XIeS_5|)E@}(v8x#Mnr8VW?5{~CIP0-z
z;$I@lEKEH=s8XNF$#9W4da$7CBTHuH1%R;%X@0)O95pljTFer4^9XHTY23Vcpdu7Y
zbAVug!|{ay<}Ip&{Bf5hHuq8UiVdx8dWQDa<mJj+I$2>Q`c=%aMm<t}Q=`G5-J$lx
z=cFu>&^eo_TJB}N(s!no8c$?C-e0cny{I3$Xh|`reOyn+I4kuFu1#qc>mcl`^fr!`
zKY06*kXiU)n4nLkV2L{yW}~I>kh-tDs_X&hk-vLz7lWjt?d%!9daAh8v<i|mXpgKh
zHJ2;%Tl(WUHxwCDhUhgy1_qwp$;F3zFSbup_<RQ~Qyj;X&SUf?Fg^FaExRz|r+?7&
zw)r^i$2~yzUAu>QlTm>>V!~(n=2bL8NqbYu<>)M%{&$tVp6T3ekuFgR7cQmwd&jCx
zBAA{lIVRC_iUxOFm)||@lWiQTO-U1ADGhX&5y|v|6Ut2`7j5rY_gBOmW<0WgRiMi@
z73~$UrnfJ#Eo^2-G$8Zsz(KzmdLAnDaaC>gB!}k9Q~uGEpiE19OBM3Q&F!}G-lnPz
zF{keFTZfC27PPm!ue2R=Z|8v8I_Ah<9m;-{v*uI+MIHcm2P9P4dR8CFnVT1PCxjl!
zh7o7NcCMHa5W0U8j&XCI299{*eG!W7Rq87;4)Q{>)i*mVV6Qy3?6<^A^gWinG)y10
zHeMgjO-<j-PRSdXUZQpwaU6LyN6l~m*@nfnCDgCFO<FBHEHj`KNv!Nh2&n89>$la9
zqKPK8HJZ!Y(MtNj8SnX|<rI?YJj0~!CUX@j{6UQ2+;+yDi@d)H?{WkTRr9e0w#0gW
zpW-FN2UOYdBNR$b>8l8S^MAXdu_3AeY`C@iLmNgM#nohfl-thXK14pK{9z@O!X`rE
z@a_;)`PB7~DrQb{w9uhuRxD&3wuVt47N;)SNHGTr$2zy0|A``Pcvo?#C17ovp>sP2
zCS^Qxe$QTeHW7-|o@lAOguBPpEBQ~73jWbfX<d+8;K;Oe{+|}hlYSH{%(76|*MH}6
z`-HU!q*zpGpA=zns-XRWWc~w>d+UCg+x;3)UxH`IqIGu$u!iQcl3Tf;uo4)k@iu?(
zpyWq+AB0okSDSB>*@GYt5f0y$>a`o0c>JqqTF2$G_!0v0?T=OS0iL;e5ejR~%*Vo<
z2jGGSWX2M<ZUCH2mB<eh!H^jq*KQRu{$9!VziC}9zs$EKTe1tcenrr_%MlV{2VE<M
zICadl-rOK0ac|!pc`~kNDxgro;$%u5%q6&SqZ7{L1UtNm%}OT6c=K1&z_#;}x3kkT
zXLBc~ckevYmW|Dq#_Wxj`+4~X&iiz)UL(<zM*T^LU&Gn|vNPvqaf3DeHQGP^48p)w
z$3EMmeA0j1=5%dxjbU<<v|{;240FqyuewH_jxTdaAL!BVZ%zqm8ZUxmQd|x39l4h?
z06?4AY1Z>-+H@QB$OCp>t{EffO3`>#_d=n$PpwM{N-t=CKkJob3gFvU+d%phac-Iy
za=O~mb{QalWABF_1qmQWvJ@i*OAOxs1h~QoI@*k;Q52i&V59T){-D7>+y6%={`rTU
z{YiRRX7Wqbe{~P_O~N7Cb!6^2bOQaKKT0R@sjUmDjX7^DMmwOhIr`=th+={0mZcE3
zc`^auwQuT}SlV4PX9vx{&H0b!&l7i+I_sgD4~3b?9fR~m`&@uj>!b(r%j_fa7E>Z5
zk*tpZqu+d9niu;duCEvRbsR`hS(Dq4S#rj|hW6isoG$$WjB?F`!GnJc^8a+h2XK-N
z>YAaOIR9gcq<sME$#;gyXHKhD*VgXMh4M5(7COVRLBX$jg0QWLxI$s6!}g5C{`xo%
zXJr?Iqw6$H446sHyH6f|XogC-IUMiy^2jdtr?kfzywuM(P}i0QFR=(64}qC>`bS7z
z2=|FBCmWwE#^X!49~Kv8<=&v(rZ94uf1{^lvo=~_9H;**xXcis#l4yUNYyfV>M4bV
z15OH;M)@D><N^aMr`ZHIxl7Bb6}6mx61TML{&TAS*|$G=#&5Uy6q16msF5{AT)N+9
z7rZ5QdFkQrFZpd?|7@`TxF&6Y4PANh!uWmJrCX}2Ui&mTU3Yf=+t0&B%^*Qf$K~cd
zBm}MIU^J7uG_il(Rn)iXSy<4ObqmI=9n{MEXJe$lfkqdd(jQ*=yN^=<t^2al^40pE
z3;&OyO-}&5HQ`hLn{#pXT5bvWq2>Da%%yhtgrJ%yTtKPw+r<6feFZnOGl1>N5cX2;
z(%tzqSJ6Gc@RJ>0zWhJy>n~r?bpyK6``pFo^5f9K;~aT|Xng-Qp#L7)zxp(q1A5y)
z^#EV__veXLf~#<GBu(Sre4GwyjXkHKKFz)K(inO$1~>w`T7TKIKk5ABofc^$F}g<V
z%X2G%T9A3Qr}zHu^B!`8hAckdF1XaZQGd|tL@2Y^<;%Yv6~8Ui-+xK137UsV<$$mM
z`{=D1fUBMnWr}&f-}}#n{ukG3I6&*15dEnfm)>^`^p^CgZRX#c3s&CG;D>5^^yTQ6
z-YsngF5e#;DPg<xmP_~ic_U+qz>a5AdY^UaYn@;)dh8UC#y=JRmu~%!5&Z!KK`;7F
z#-)yFu7iW@xhKYE^QFuGcUwn`Ks(&1^&?b%pA6Ara1~E%pQi3_K2E0|)h$}O#>=x9
zgF1b6VDW!ju1qR|RdFiZNPc-Ke*;3khO$QNZ!cwK)MOCFV_#k?UwA<Ckccjt-(~Ax
z&A~sWL>fWDJ2(zQe9L<9+n(|fB?B)(_^oIB&9x$}h7y;n^HqD7mX)+A5L4l5$(jE}
zi!R-w;|C_=IrQ4;<z=9Q+JDzz-%yIc<!}Cve>LxbhLC=8(7yCL>Dv&{>Hw{B{7aWF
zwdBtq{O27Ttp=@5q>&W6d=UTnBYuC2BPW<eIOe#l_V1GcX9HJF;f;zVzu)^`H1+?p
zD44+Z;|T73bLpV_eeYa4@BjUUj}y?{#$S$Z-Tn79@X3M}jOpgW#^e9b&o15kyR7%Q
z2Lw_q`_k^Eu1LqCmeB6iO7}~b|L?938iU6<PGlUl{NAzj$KdLh^)9zZfAeu~F;H|N
zEzKy!*I{$=_Wss4vErJshsL?@o+*dwESC5*6;hFj<y>l53~C#Ch>ibEZNX=a>=*#i
zw*$ry%2Ys+qd=YFeR0wb9KO^5KjMqU;~G`4AGsKfr7n%J&kZn>MFiPNfAf6xDI_`u
z*^g~Ogp6%~>jnA@xIK9-2Gh%bwqXY<MKixaY0g5k9{lQ3CqJTP<GBqB3%3jSuOp-U
z-(MO<=@it)iT2s&z0|J%?~MQJ3p$p!xO-gHgEI2~*VU3>@)TG>?EuLCW~$5g;K63^
zU**3pPM60@a>cGA3oQjULN9G}&8w(gEbTyisWF!(;!pRQ=54eS<ED0eDe(v2d$;F+
zjYb)OPfPPG!8pKX{tbBmDrDy!X?ZarJu5(qCU#k}=sW;2$4uf|;-#iszUR;0sw(Ca
z7jSY9Uqg7_|0l4k0=3menD$qO7}ZJ>4bVOB^OHDT)&=v!fb{4QZXmsMVEiT!{xypK
zy!Kpau4+IM8zhaoCh(alCx|*eKL(sVq|bnti5EE6-T(urDh`k7(6P-QYaXD3Ype|C
zsH{xa+n4?#W>LqP_5S6#W!oT2TB@ER+Od#o7N_6lFrpU!tY&pHTPeB?7*Wurcjl=f
z+snZEy8#kf9TQC9A|^Ezaa*qsP_X5~7oY*|Szi)<fn|*7F$?8xu*f>$O8E%}`_o=H
z?0JFEGBe_nh|;y1T8pr5U|EtqOSH@_A6DkLy=EZd$-1o9OVNC9+;U3~`=kl*58Cdw
zVe?QuOLbk63Dx-`z`lio?VSft!@=gN)?|I~j%R8W<%5vK><75#$7kLBMFFQ^tr(MR
z_+6TyKbQD_wN`ZMuVlOj8Mu-J80=$^L$2nt-Ed#Ej<~XbRbZHgB1UUojF*qzQ(0`-
z@8Kx(FKa@wAJq*~x0(6JE`ceKsp1d}*PK>jNg;WE1jwXn?nqnrCBWvvX0{$`x&Z7l
z0TkxRu=g9sI}7cAHIV4u&k%}_?nfwg6j?oxY;WI7+PJqrq7jJAT}NkCHy>cUGMEug
z#(3;HVI0`88vn>n38(4y!~8CPz+uunpAkrN<7n2~`}jw+#`k;vil>L0lt*8HXF=h`
z#o6A#{8#fdTu@<RfMb?(O*j`&x`=XC_3>HW`_RGXzJdGf84b#+HU<))Ta8{%#N<=Z
z_4V8Zjv9Qc>SR;1kMWP)*x<7u-LbuW0-}DgB-D)MiQHF;zBwJ&pstOWeA0f6W~1qJ
z%a)y91g*J1;&{$)9{j6<Px*}qP9^SQleKCsF%Q-b78p^l&~^`TZ(;l;kY#BD7F4a{
zQO{(PPgCW@Bc)4=I{z`);9tZ94=?*Vem^`e-LEO_qkMxvm|;VvE)alxSG{pvP7%5u
zng3MLtY|bpbF9p$dB1$^RTB0-;bfMb;d7l$#_0s>0vXCLr|jP*#<#tHoj%Ivk{T3p
z+h4KS`z-X{;{8|0Z2&=lsv2GzV~mlj`8WzOEdj{7tHxPV-O((8fJ=|-`Z7E93L}Yg
zzsRyb;IV?eH|l&H;x6FpiDK2m(Ox1uUhD^+BkGr65~s7zOef9a4cn0GWob#Ma6X1_
z(&yX0sftv#axY>1@4%K}m7ICvhl>__6Ke=|k#m?aZOXLa^5wqnA~MFixIF+D8|80#
zDe2s!pI2aNUA}{cA94)|@gcg-H8%FxeX|@v<4%(F?b(Q{r#ld5CKfR^cG}^~J*b+W
zmUW{G*Ds@Pn-T-kM(&}FQkR5FdN?>FYhdL`|Fw3Tu8aXG`v4+S6+*zPVnve80Fgat
zqV+na=N<v#JF2ikt+(BZdr(m?Vxjw$Khy{mLGn7TjS7c`*n+B$rxzwAULK!sk7E@6
z%!$KYgl&Rkd@TR<8@Jm`bRWB#72i@6=DAi3_RL2p(bE(kbj$#d@yy*PzSn1K5O3(D
zk8_(aQ4Fw`$_P%#bx8ET0Zd995BgQEwkTsFg&)I{YL`>p18$G$F$OGxN+UlT;9GkG
zO1(HAOo@T2>+gJLC%|je-ik>}MOy|A^@<Q$&tG8AwRc4_n%)+B_87Q_U<6&BZF-hx
zF3Hxw-)AU%zCVI~wg2PMjMyRlXT-{oA}y}}3Cd|FP+u%reW(YT^;Oy>;w_htB0OT5
zeL?X_<Z5-XhRV3{M-xQPhoy(2v%4Jo)UH1wZE;YC0Ticy7H>f0;{J<;4|#mF%VhJ5
zK%!MYFN(UkQZ!c8<#k#2HUsuQ2XX(G|N5lOhxvdLPE42nW$gV2Sg5-3EJixjeb8^j
zs*b?=WX+2Y8u=SQpojQyvL|S5MwFApE{i&@#bE2+o_zAGErw0oMDMNRSD|&%HQ7SC
zgeAgG6GIwet3=}f3O<AL*~Z1|NWCP}Q$RQIu4qD0^ZDUd^MOsqgFtrgH-{+(<z3W;
zZ?p>FgPMi6;%QenDxW8x&H3R60NF2y(xuJ-!BFp;Y=!Wfq+7r(ItWCdO%Ge10EE$4
zVJ>8citp*&iKo$Oc8zu5d})u?kbLK!ycKhEg%PJ4`u$!$wS6B9P_bS(XSGoE0*TW&
z|A|;5tcV(<5*rW6lZPKglj5(iT<sXWI4JYJCSz8O;)q0vI?D&G2A#1_sP6v|oV3U)
z2T<VxdT^M_Mg_uzzz)=@^nT{Z!(XI&Q9Dd?6+tqY#rSn@M)Gun_<@0S%|x@^m~n>*
z;-*mIi62Glz0TVy>u=CJ#kYZzb++yq+p^s=UD8GsZ;-4Yg&4oppJV0zJ(#6!qQcZv
z&khu%^ndONI3Y9g*nu*94tQ*j{h*r~XaIZ!PO_K8S$+)A0F;#)s4s2Of1dV#wg3tZ
z=qOhV08&;vm3`k(emjPn9=vu?279s1l-B#q)&;~GOu7Quopf}2Ujb*mP5Q2pR~8!C
zRc<t&6Ru_k8hRMoeF3yLI&4`QUlp3k^q%-^WDh1v(^4|bMHiAkd#{nOU67X=a+-)z
z4U{=bMxQi)mY~yAEQcQgnmBqC<LZu=imc}TDscRV`gEfTi;PR_mqOO}0$2NH0l!u|
z7{A!0)Gf`SEgwu82nwM^7;u-4{P+oU1kA$IoI@ZO13^%lxBAu~LK1bf1@0^$5ao2F
zKOk}yTYSBo(Sj5lOIo&YCs5%V*yMS-w3H*}PmEGg%Yih0a~1vir*58)`}c}jyZm~2
znm^ppduyg2r~rLy`@ArmMnZHNQ0Zo9q)E<$N$<TjnxH-y3S&9$+IxM4rB|z<Y*bQ{
zx;)wRBS@k*nRwgtMhCy|mH%ieKni(V(xbN~`kDI0Bn!d=pnxLS-zE&Mj04U+YD5I$
z>kUt~w@L*3QgVpO%`cEWEuWq9qgi~woS{dj_wMEw@-n~>c8t73t9_+<tj&We!ly}~
zaOSd|?uc?0s$U;&|BEB;(h|{3r|_||2$#o_`wR|e+`@FyDY+Z_ivxM027)$11LAw1
zUh6*O_D<uJ8d2r%Q1-N>khud6HY1N?Knw6m`hFt9`a&`x&Pcwj#z(UD<ai;%p16{}
zee62&9_tIxMSL)T9^A)YqNHi>z<lrevlK)rlygkw-%})b1=JCkW_$|5v&{;l0rC)4
z`8JT2;_fk`CiDh0rb^yN63v6{XY9v;P87>+IZ)>in?!x$bFb0ApA)MK)9<f+;g}h?
zpr5;Y7pR#+xPF+RVxuy4Hvh_dz<Rs%;ACp{f@}*&T4+nwcjJnl*TpR-ZlHPy;aKo^
z0>n+Gok~H~1N3pP+Htd3vNk5IhfVQlLg7G7HvS?Fhp9|?RF97iLEkGN{@m+8kjrF{
z_l=B-VP3tdSz4IHvh;HdDy+=m%A+bDNp?y*JQO*kaWGRH<W<wjvtv;B&<6A-+PFt{
z-Ox_8sTW04RtXW~e7;!)mCpqG-h&)Oe?Umh<CI0#?%^;kZA2yc>$2W|zz8POt8<v~
zzc$O#*{=FJ>wIT`L|dceG)g-!zUbTpw5-V~)Z5FL$ov3#=vzUvo|vCN<@{7=C7#Pb
zWz2K)lrbA2nSW!VKs!%3Wsd9%;sP)r2to!IF_D$0?|6-6is?G9H#l#n7S(kuC}SXj
zI{gkRkiAWXj?0hi=9*_N9IC0-!iu<I4DzBMPJm*<n>r;gl+w4ss>d2P^7a%T1A<pX
z*O+_rS7s=82;w6k99;JGwWN8EDm--A=^$Uwdj!tPull%K@kA@89bk2dUq1ptrdnRh
z;cOd}@>0ltPyVeYxpnjTVd#9bjImtjuXI1WkiFBRZCZ$UZd{v*_%oX2ufTwyEINU0
zq6|V!+_#n|T-0KBN&)P4O{>-6=VsKg@t|wL<vJb})Ki4{VFAi!2n5!fpZ!Tj>*!ZN
zi%2^U1}ALcSDw^4n>x2+$`U+QnzdE)<`{skF!kV~%f9Qf3qV+d_Ku@*_j7E_UOt<l
zuB`z|(~rlPyU*Su>C1?9mQKMLWx^{$xs>HLhtlEGs=o5likXR$-gTuRo=J8b(>M4Q
ze>edI^r5B`?Ii<LS_~Y~FBGkuFZ%>8FOEyUXb~WzOHc?}2FThW#tkyxR)I-IcQ+ol
z@hQ5ej>r1fDs#X4C^$B#hbhBV&au3lLgzPQ^sERsj77-lP^FKguklIjH;5DAJqDKv
zpybx!d9xF)FrFz)*alEMv(TjAejiP=<$+wyo6paJhaeQX!h4K$rYQxj^rn^VeVTZh
z&-kvx^%9hwIZCSUWqkyO;JMEYD}h6Kv+7Zl4kS(5IEo_s)uVt(^v}c*tr)y0sc8eN
z<DH?z8PJz1qUKoAJc!2hOvB3osR!buK$Ly~b>;;ClapV4nF{bYQMRLWU|Ql*<uCgQ
z(g_;?Itsh&<Ke^Pv~yWuJMif_$;&=&*`WR6s{WAZexTw(bw2lN98sYAx&sgx!wIXZ
z&%-}l44%{Wh>Ve-iGx$a{_3z53(8f{QU`R%M?MSbu~m5Mq+$<h2fSQLwP2C<Qd1BT
zBF=0xbWcWwFvtQTB4VU~J0W%3#cpdTLH(drS?Dq2gjAX7N$knQ_@`jvEJ{)?5u+;W
zA6=0ZQFQbp8|wy6@1B2nm0osJu|x8*CRRcK++a7vb?^VxiTwL1ogQ!v!UYsy-HM(`
zu&by!9-85s*DuCl7ntY^4x(h9;EjuNnmaVL-J-=wv-zDz-~d6G<<}}&gg@XMj4#=b
zuLUCK)wMP|f4q48XYT~+8GM>Z$g)G`fHs#l<y7DdsngO5Z@Tw}TR(<qh4H7;xS?}L
zI_Z~_To8j+_(nHzmtYG-R)h8+nuOKaGGBp4Ldoe3Vxr{J*Rf-XR>Urj7i70*K2lmG
zVd6{dYZi~j<EEJvHN!eV=-9Y;u*+4(h3~e9O{V9?$udl@^W^TrqpX{ZQzjl38)*cO
z#!C$s7S5zFMu0Y)QrDaz8@M#kFTqV__dlxCjb*Krw;@g^;k+Bb$MX=lvU)_V-cMA}
z09|+<<+%VvdfHIlC(>TzCfCF}ljCsr!*AYZhkpSo!7QC#uie)*Po1G8a1y_J6sZ7>
zSstT&^+AibPd73}XI=pWHRjVNm@er`>9;{J;bGQt!=>%^S62UM#Ff}GNQT-wJ|>z`
zU?E2p;l!hPV)Q-$mWwZ@T1P4Hi*?NDJq)<g-wc?mk0YcFGQiyEvI<Yc`84UMy}dF_
zq{69{U&NK@1b{&U0kAfcCViOllY$z7K}EWsK&)Z;PhQ8Tp%VP&Fj5_?@E)U^8@NuO
zF1`JqVtVXRMvb4ep3^&r<{0T&6r!6j&+gQESu>5_1Jyh!KyzgO=ttVGGjp6A!XJHt
zg&ahElpy$x&H|9GY}#>cwF;41K3p{$lKE!}uVKx$r6>j2cDDjSLrDfIRL<n>B0}Fx
z_iV}*AVfWme+qX3vGS@l<gHF;mJE%Lf{ax)P;EcTAbMOCYGhF)=wqx2h<%_u!gOt6
zlq;r>D-;G}Of${UOFH6s%#gt8LR{K>fwX87);_5$P_Q^g*+c84jW2Po3e{NM9^fC;
z{hVbw`};PK_TfWCwtbe^|NB`<xYC*&Z{#I%*q~eX*ccevoNgEc5=W1Yfz9y!?8^fK
zE>)>>P~7VyS@=xLm4_0UMKtxJJ9?oOXm?>Zyv~3DdDI59O3i8+qe<u5ENjx0tcZf=
z(|`;edaV0Wg<g_<Kud}DHkuA9y~CppxF!_x0P<ArklqIXZ8MLONxh)_-qIK>ZM7s;
z7ZuH)z)KsNp~QX*HGQzR$3R)YM04?<guEwxk%NImiA!FfblsjLS&(76GYLGKv>ZD*
zZy+IdFF_6<rZFm$UxlkiM-Rx?KkQAWR@A8pq@1Ab4$3*m-3T^&AXn`KgEfP*%0xs9
z^%n{(iKX|zL>39aC?IhGPmo(UGaFuzp%+@J$;Hp1TDL_|llvve2H;>NSyd{nJ=i(~
z^5PG_50wTxY`8~1@ZbjZ3&AID|InQ)E&LSje+IM2ul~X^J>m`UfXpJ6ohwXVlt&ZM
zP&tljRmiZ=BrmJoX4i#7VGP^@_EE2`O9at9Yeel14KC@TtSIFb5*Gct+x;&^V?Mu+
zj?YV=j9BvF>;Z>ipoZkhEt+$1dQ%mZ<Ve#Jk5qfmu*YZxhnLeB3$mXRvS&H2k8uU`
zK@M^b%e>DyJb)Cs4-PsxMQCU(pt=<_IN_j{&6pxt;Sk{Qewgug6y-~X7A^U@=*Tb<
zDBb=bbAPY234lx_pbjv*jN-HNy}B4_+Ki8>67GVNeL%~h;MhA~(`~*15R#3PHR|3<
z>w!#=HXQ!G!6(%0LPY&YKzwf!D~txO2Ar`xsj|Qy!=Z<@J0D1kr~AlOUfdkzh`c`a
z)aVa&)J+lkR`ZE=iDp;$V(+S~2m%@3e8%)Md457W=Yy5^$QjBvq07Xh#$iTXtG}TK
zgzlj9&l0`4@EwueE=JS$!}UEcTI*~VzSofzMs>AWHiEJ+6;w9P0<z@1eZBiCCKZ8f
z<%wxp^CnPF=V<3O`aeIk1`$-p9~<v4dqCxY<C+l4ayel57h6!%9L;|n(lH2`Cj7uL
zdhd-S`<@n41Q5zvu$L7bWG~9ujfG`wtGB*22FjD^-%?E$hJ671(6*B|IEo2SK4+IS
zDgUxmKtd?}MYQmc3t&XqK6Vn@yr02ypDY_)XEY0GP)jJ8W!4___2lFT5kZ?K#U`1n
zY*P5a8Rg=9OKF^SRzCG}zTK2He9hDEIxc%17c-oKpZSsf=i5WuK<|iXZK~!qZuY|y
zWer$>@GsJ8KZH-+1M<4g2|QtAZATK(aeM?)hHR{=QEpVY?XQdTcHn)d&C8Mc8sY`)
z;m2E#vKi>VTrbd5bbNp21kS=hMH<&zsm)YIwVps@g5Bz>o;}}khDj%;q<J^%b(JJb
z#@H7FOWz23`?(HIVueO3W&z<{MiNC7R9zeJ7E*4<SK)DxiH(iEVrNbT*cIXZJE(JD
z8szxd&aw?EPLLbjT>b`%p{cj?q;yQg!Ku5@9v3G2u&mzTF{Rh()a>;_9+Z+UPhR$(
zmlpGRdfAGN*nc<&)PQfaM&#w;!%Lg+FAPTzXREv_)@G^maD6CC5u`0sEac|%9Rg<c
zAM9TUNA*pDnZAFpN{_%o8Hjo|t=G^qq<jT+rWmso8Cz}xQT7Jq<NmPQ!(n^#{?!6u
zMN=vNmHZ{etcK|jj#6m591yDD(&u|wWvI{q9francg@)x{az~Y*$XFml~xZcTMk9N
z0{iJ^zPKXelUUjxrlNv6^jWr3qZg9Ml`I?Jed#oN4|ur$AA9c|Pxb%*jhDzO86mqt
z%C79JD9YYRB1!fpn~sdiY#7-RaU>%|I954II94*s%HDf(Jzl+|ynomAxxU}q?fcK~
zcDw%5>2+S`IUeizc-$ZNXLkFYn2)L~Ya(0gb5)S3AeXu2J3%tR`DVE`0rvOx)67BH
zZ!-|Z6XBPcwoms!AhS;k&&+o!&7;OuuIJGnhSu)cDlcDbQvuQ5vkA~VH>(yGTJSqR
zkT!O0V4Dlz!`jFptUGv=4*{CCwflcIoo(;Db?lw}s%m}Cos4b@{qJB?jf^kX*8*Dr
zhxSx*S&#w5IYw=@*8->mnZ_Vz#h@%U^PzC!py0Qo2be_po!^ibvQ0<olFXl>31Z)1
z4SNswdRK=r)iz3^iM+t@qQZLMospgT9m*Eg#DVIqb8wrxQ52N=7_2xL2K(~^9|h;@
zJ?CXgL}6HaGk0q-J47_2w~vY+*cIsfCp)WqbQ+8+A%~?v8>ICPXC$Ve9ajJxHAXet
z&k~GdG%P$k&sjj$OdOl}eiVI%F^XeDD_7b;W9<$zwipT=A8ifqwl9*+8*{c%lX4rc
z$=wX$yJqeirLJ``mEzv#{dCRt?ValTJYOs5gxIB)osRmnOfe}R%z=U<f&BsJ6eps)
zxn#lep-^IUxldx_OR+WoO7i1PGhKh&=^+-wGxYr*>?lsw6+m3at_POwOt}-Jt?!sn
zml$yU)K-RYUt81(bjuHjc?H_aew4Zog~UDVv?=ra=F%VRFju5eFZwYgd_s*a?6Wt2
zrOiAlk+ZkZQUN8AO3^c5XMTD)re3_j;pbyHfBjOoluI1;?jYd-paWQvMBfFZ>e+SI
z^~MR@m2N1id^$3C!uR^2Iw=G3pcmNPb-mjVw;1!4Wc;T9^6z7w6d`#c33G)6x?(_x
zV=U#f=jkulv!MUeA?1g{OLM~~)X7h2Q7UvAv-o<e4+oBD;=g=PL%j!KrPc8a9<g|x
zBa5WY1EnBKH`}&0RmLLfJjn<^#phk)cKy|U04F`IefGq3IDPmYcEzYL=0R%_vPvn4
zAg*_5zq9^sKbPyACc6j~p=t@rTS#8;?OTQSp>0S(xvdBsCOtt@R0ReJU<$v$-RG2?
zOx1*!HpwN5;Sg5^xa9Wos5A6Xy%}zPrSe!{YwIWIo*av(<AJ&qRp#pz#*noMu6R2x
zQdRQn2#I5M8$QN7NZ-187l@GcK$C+ECT;7<`I6Q33+zG;((|7>z$l!pHvMEE%^d3`
ze5dkuUH)NvhTl0}dM<rrx4&w-hN<3I6!AH|p$n%aBdsdSp<ksA<4&YmMJzzw^og06
z96}~x+jNC}hEH3I)%Q95P&r%~iaRRxSqv?&K4Jq|NBKbQ@rWUEY64d#B$c8AUSnUw
z4YZbe#p5BFQoogQ2u^b}Y7Xu`*0myFHPPw{{l9neKkhj02Oj=?qVjdfV4l$v1lTV&
z`$m!%a(2I!Oz|dZmHAlt`q8VKf;u4s#B;`+87UIkHrH41eCgPgq7PTuelK?BJ>tUY
zJr>-?T2j49WDCY9Hkh_1rcy-qqtNP=Eh0_9HesP}SmexDb$$F*(cJ4Is|eAuUyf*=
zRAZCE_XMu(iFUoHl9lu9&riL;{2JOy1#CcHQ)%kzPCU458ZpF&Dl1?<f%mCR&q2ur
z{F;Mj4_Ou`zQVf>^Nbf+MhCUik_aJeFwGuE?=m<zO7Jt>YW8&hWqTu9AeAI#fTfuk
zrdGYFOQL!#%p0o9L9O`dUoPw9F~<N(i%~0dSEWxBLCTPcuQQkI`t9A?w|hUh0squ?
z{3I;`oZ>)WeSI&jRd>0<r~(8MomuAoU$17-fP%MHo@$pPIGiR2^Ii}}Ge7-%NRFoP
zDU;i>xXXWeIdoZ(g4guKyKeuH)EFMek*1VOV})VJTcyU3dfN{i_wK1_vB7lE{#Et6
zKmHG)+V(<;!01Fsej(ETJiUMjv_y4d@_&d8b}iiR=TNH%M%&wHl;Up%ks_ppg|ATA
zb7(gN_5c2IG=iLHB@1Ty8`ZRz;4|_o@i7*E|Hyw-x9!PVal^ba{k~)V-k?*sdf4?2
zx~}j4?e}pb5IE^mw7Z9AehEgM8T4%9zl-GwISIo{4lcy*au-erGH5d3+rIzq>*x`X
zc=A4zce{>Ps9P}GN1udm{O>mKKYjLpP>%n-CjH;{_Hfz*(BMaW?<scaMWP!hlvabz
zH>zD9|EFl>cc$PdU}1SPdOdf2FOdw<&x)gNM1Q3Fzx0=CBoZ~*>xiHF`+dVT@TutR
zu>1Ib=_T882Tccc@I|z%<9N(2(Np!8BAE}*L1+B8^_C<r36@XrgH!Xak(B2_icyHW
z<@>8$zw>7x|Hr>@B~q>XSEHxiEByDz^6zh`p2utUgj$f~RR`y8h`7F$exx-hi7qz#
zS5^m2;R2^3BLk#jf*yIw^r!5RW`OdBo}!r^kF9HEM&plnRI`<}XRX1U{@~gUBDAIK
z-jw8+caXUB_D<c@1#npm&FG}YE_pVU0b6JFU@Y~IKF|}pTu?Yc$^K^ou&h&(o#bsw
ze-6F;l+zm~yB^Y%0u3i1|L(hMkyVwDJ^mYWrn-fvk5k_=8mHJP+Wo)r|G)Jo`7J-~
zA1V)t|M(g4<GM}o$)WSRrS<<?%11>IfwU?-)BDq5l&Fq}*?Rr#W#=EwF?|REeEr4G
zvw!SD3IYJpdR;RMhQC~RSLZ>;4UpvGr^Pbcx|1kF3O^FS>~%<gTX{^K_jY?1oG&`R
z<A5Oenvy&}KkYuf{UcH_zW}8GZ6&$zepldUnh$nLo9g}c+W>JziC>bKg3m?!rwjrQ
z!Jw6f_y5WI%9DfE5$o`vcRV}p7U(Z`s=3>ReV$5d!hf3ez&UiNz&zHj6|-;8LkdM+
zlKAoMAHR_(PT;ZC9^Wzf2kc+Ilo0<{r-Ky|^+==luX5mz-1##Q=s#)&#n%1z=zJn5
zv-LE0rUegSwk!pb82-4vQy1VD*lrf3?U)IvKqP#NLXQv}oEMnr*V`E_yv9@#rZ-_S
z5&!3>U*m>E{upc=22Wr0JLbov()N!I55sSc51ZV>?%>7qf?>HyiU0`oFLW<T8_>}f
z{~jNyu6((@+RiZGq2<lDt-jWOts6e<e#E_j7u#EX504u#{%yC=skHpKw!Bb>Kjsvc
zY6JXjenP(cpYDWg>4c%3Jxr8ZBXPOlwB}YUwEd%r$;fXeC+ub3>EFN!!h&=B?9fMH
za*VM4r!`U!k1s&-js;M56mCnCOz#~>)NP;Z#xFxE$`^Vr1(+*!Ei$!c&-D4TSV3+1
zkOEiE#gcA)Z?E+ktSw~n1;2fF8_T3c%FFwkquo?Z82|#Ni~w0RGFpF@Ah2afx$vy^
zJnMigEV(|QDVLEN^g)LPap&mP7oFp-Er3qrgXY2lQ}JI%h+o@eeg))?J2=8~N0!YE
z%1L$2y$^`v?MyHNAH0}Cva!MTN})PvLAWPEljL@GS2G#ew_tg9g4GCJ^K_}B=|!^T
z7D%ZN(nao&nq<N=EYr0C&M-CM;l%rWhI&q|T81@Y$Jm&fpj1JNxX{A-f=+7&*$~F|
zn;Qri8;bkYW>yrlP<qd*1PDvp%>+PYntg*(8-t?7XvMxmV&wY|(zo1v45{m5&Uy#$
zv@G8T(ljm)NI9K(Eo@ln+tsKC*H;@tkMe_11e+9KSmi%3X!okfP7Kg9`?gF0R6(|n
zkSu6vW2eWb<ta?Xjb17A(zcwYdF|yjP2W6|8?Y@zQsTt0>G877Mmv@ZZ2;QRDk;0T
zb0$l>1R6n2pKne&Z+ysNAp9yBf*nYtzsCKpfNVY!vhuw}L-BzZpR%ivA?SxoMI;2P
zY5bQjXQe=1^)9J5q($kutP`^g6SIo!Kv;X`XHdU&u_e7ck7WMF)|5&m+msi%{>mNu
zxX*4n|BSDtOscH@ZGd4Z%>kLgDg7<NF$!%(*j5hN5&u8hse2Sc5t^e#5B{k4M`WkZ
z$P2yP5%@6Zd5o+ut^009@lUNZLBiAi`z8R!@n|PQSR^I(Hy73ws3pt#ApqO+>8X?N
z1aV7H67z#f9zU}~5{YLWL4|Ih)+roT&;Z(K^~$0<2*Za!6B<n)htLccL!I0~>maGt
zu!S`vSQSyf*lnS5I)(dz!en`^pRt^4aoE2yI;}Sw_G|p2Z}wDR%Cd5$ht+#oN=FM{
zEO`EeNTs`ftnp8zbrOlR7%0~MZ0tL*(qrx(;-cH4eNz`P`(f*OMDxK+=LO02Ilfwh
z%<@@&%brHyLYD?1B_y?+VbS*Hqjvh}Nan$LJ6j~x3(dJxaK=rqm=UDKx5&1oLjb9e
zoGrBdat6y^{1HGzV&g!U7925Vio&pX7wLF>es;RC_iEXyRurVU8KU@wLiVrH+pcU|
zs!`3v(tV!$n7upr4C3Mym?Mx~k;2j(ZCUD}-?;8K4Uj(}FZLyfM17~qwj>-UyF|Ps
zMb^gOMwWgD^tH(JjZP3RnnFn|qXRx>Ef&Cv(OrYUiJHI%icmNSWj30%#uSCflWlf)
zY7rM^tjzNVu%Su^c2c{#G4rojQV}GY+rj?y_Hs#~n!r4U4{AA%m>-yLY=@i~?`yb%
zY^INfnQA&2eT};Sur|@6lY>4tl`=b@xPHX*B<0GM>e*-DK+Bj3a8m>j{jj*N^3~y%
zaafqa*D(xOP!FXzQ335%kFm^meRI#k`j8ssDxkRm!1JT4S9@a}v2E*f30pxMdUxB6
z`!Bg`b&rrZ^0{|$w|61Mvpdg?BO<tTX={;;R4)i;2(}4;;QL$ebgu3pE>z8Nf(+Av
zjnxH<yz_&>3_?8mU7!3?^?E+#B8A)O;|;QU%HY^JXiOWVy4j{Y$wRy+>Nvh;Tm05{
z96ib*Vv?$0sy+F&5_!BkK(h8p;+}O60w5&2n%?5sx@*)F-7+&<nG#Zz_1@YvO!A}o
zoIvf(IEsM&@K<LU5)@Xxv~@W<tc<M`Cf%dW<DZ{?{73UTNx+_1d47NY=OY=|5W!s!
z86#)Q>NA=aiJ?_%wFaA%WRmOCU61W66dK~i9YUMq9#XOULzXuH1VIona5K^|kO20B
zG$G$OVE+g>*8o=J)C-xJ5LCeZLlVLs;9pRIMhFk>a;UruzSb{jDBPc!a)Y`A0d7<w
zfW{-b?xX3i@G4GL(jeDnuR8;~i!V}wdgBg0#_yPn|A#gF0byB4tYy+5I3)=vVn|&o
z23N%APY>X-K^)jwTc}-P2asHDp%NE@etnhW>D3x@7Z#I{2d}UDu^e;zuur`GYk>E7
zor&##4`t~7O#}_+(yQvB+S7pa4Iq`aA<Oq8sWXuuGzzoQrMjUtf-&0zFs7A&^Eh&_
zF#F+6>dxM_+Bt@@XfGJ{ers?VBiT?fC#W8To`FY3pD$_X4S%>R%&UFPIDB1+-(?-{
zy7_3AawztCEY<7c_1W40fK>YW(drjg%#Kvt3go7d0Kq!4r@t?&B_4_vF7dN2xWo%~
z=Tm}=yb?nl{2h7#mq&u2=)|UGdVO*|7u(t61I)U^F~(hXr@y+8jTiQZ7(#vPbiw6T
zm7#@Re+KIZ8?1UIP<7WXX?S{4=}{JgOHSybrP!f%Z!od9$j?ojckeRO)j*2ht`)Y|
zG{*9<B?Y;q0Pn_YFs9FXyUrp2c#3@NF+Tvui>?FyjZD)=7@&ssYd>Gq1I+Ze2bA`T
z%vPKs;)q)-W3S72smAs$e02!wD~@`o#RK!#;<EbVO_+=RF*W3;-pR9czn!^ZVD7OQ
z14hay9K)Zf@k-ec((4v8CejiLNP~*mLLoEtYOd8&B|)?4`_djiiUCluPZIr%KH<XK
zF|e8*h6u;Y+fr^Ba5<l)9R9TPpmHIGy(SpPxpSN(d3Fq}^jVzVj;~csyarygjxyg8
zWllD)_Q~q;C7MrKj+}>?`}?B78jH}H)=HYAYrZ3hEk2n7E5I=J!bLQP@LPyN?RphR
zJ{m{%Zz6$&%w0dUDsqW~;qr9vO2$v-ETq_@bNNUhRCFm;kFQ@kaag)0h}1cE<7nbH
zq_^eg6NT!5;aY&g|7u7BkbjnN=C=f1%<^$tJ2mqGU4R!N*pF-|`eN&DB~*HvNw#<P
zhe}EW0a8HuxW`@S?HEKaP(vYXFo4niGJ>kejWM%L$oaW^Ro5(Ke{=ffO9CA98>@9T
zUQ|^92WdWw2@gEe=);_S+IIsFh0jfVQLvRrpNvT~ukkooPv6T+-;6>F#ad6Mi(dOI
zd}p60z}DT6E2t^~aN2CTrKn?`o>B?ESC`*W$2NsVaUHB*W%yNny8F+~?BZ!-lQr)e
ztxqT0q3BTG4i78Vwtcn|{PtBFB_W-PRCHNIM$?iz!Uv(W5NssN7v2`%l8cS2tT5A0
z>a;3oKZ#MMjTYj5l^;F<Df;FfJY|ls*(`D0Bq<hJyII{Zj)Z#tW<``dh1enD2PT|X
z$objtJ=bLm<|xW+k+h*!P3$MhGfB)^D>XXLlPCA)S6CrMNU?moe`|v*<5^WHuum8n
zi+#NYZnqL(L=;ztirKw{IXw6GpAVh?uCLLFG$-^gg>uOVIAH>QVs)3}A&}B19`PaJ
zn4h_E%e>ud`h@a$pt;Am_xQJ*BDAN+*M6*z<p443*8GNr>vcQs9q+Z5Ko^M#B1DS*
zw9iTzBOwo++`FB8%qbkwV=EJy%cA9w7<&K3XIiaMQ(qeOqmQsFhH3S2=doxD{Px_|
zDxwLyoP)jPE$WG(Sl>;ew#Kx3*pKIY>zepeho)l8koK0NQQ=ma3;mVRP`ff_ADW~k
zou7b^uNNW(Q}RKh6nTPGU88W8VFaur-Hq{dFg%O*z>i~)g0&)cC8!Pw<uk}>GH}>J
zH3Du1bt7?(Mmt02#j9U(j_F*^FK+=$SuMs8^wOM(R<ZXv&-+{A^;DHM?Xra<Kne0m
z`4Nh#AYnEBj=Jk(^u6}4%qYzGrm(k&jfjB(RK~hqgCCIapR%XoupI)p+@sb#?T0U5
z0JA7g`$92>T}t7S41E%2Yb9apA#HlfO1^!}!2+iiDTC`w&hD}MtSI>0AxE~aLXV#M
zylZg3z$1r}UxKl9RV@=$y>(;dm=VV+1>VqcHTqt=&CnNGO=(X$AH~gug8BJZQGtU;
z)jZJ#3OPj$*8sI}b6uaK^RySv6oN#%CIpjQR{Oc0*KrG61aX4L?3MNSpU`0D6a}DE
zhrk&bZlulYbtN`5&oaMN+wKR_-#7`d1eY?vP9fg7$f>Yu$z;SG-FnqP+^ekpChE(+
zRmIv#p}N7h+}*Bfs4#D(e58kGI<z5tKD?oN0SECBvuvZ$&S}_O4uYZlbJ~kLN9Hs_
zaL`|?PPrY}ITb7*;Uq3f(>*bi!%XwZ3$>zPw0Rye)yWU4*GNpxY`iPmu$q*n^q~cy
zoPi5qqWWjo-wRNWs@LvcjS{?Beox0Ngw)OyZ0<pb;uLwZ-$4tA5Rt;nSbIB(9Z-;6
z7w!)_XJ5g_<C{eP{a2@5<mO@0`IMEe+)TvFXRB1+!9Ux?d+`;CCl{7LlE2dYbom3h
z_fh_Q)S3C(D<}y|5kU`A&Ee$Z`d-N|Zjp2$q`tqvUKNFoIci>hm~oxA+-MeL`HJdh
zd+lGv-TfTAFR2u1Nl0WS1zf2a8~PUnwaDP~t6QyM$7Cu2#Y<1y%ZLP{&3!XITpVjD
zvZ?`}T}FIfA?G+ar}2?nDg=l90&Ox^8z`{r$>9jsT1d12UZ;RPb3G|^Ax*PpCGRsw
z#x!36U>tKQbjaluh8%TMyRO%B`v4Bqz|%xA@WwFk7^)y$x*u-X%`jAS$qTGr7!byb
z=C;zqm*X^!gYdIy*%=^Hiy~!ZDZYs=N_{V+(Z>Ya3ROdgCIDgkl+<P*tN96u=LFqw
z)uYXVJT=rL7;rLiw|Ud>2>{&D8BC>{N9vyoTtIzJV)2|;D=~dca&RQy5>%8d$WDP?
ziw;ijPu!Kv&6!qZ6~g66309FdxAG$S^Yb_vqwc=@Us&)8@U6_?tbQ#Ky)X0GT-K8z
zdy$6TZ-#)EmkNLrRhyxI$G(4#+=(~v<qyu!tOT&^PoCM(TUDfCU+-q|@(5~3@n|R?
zdYwuynLWEywk6J!JNzQhpZcm^PGcWem5CR90-f2qX*0mXX{K!8`l_O-wLV{i&FPH;
z-F_6=V=d_#>tay{_cl3y9dh9EOqnT)oT5+fsXt^R4#C+$G;Xe637>G~b)1!DfVYRx
zW2^OY+V0TV=kFt8X!BZThufXE?PlU4Vj|s*?u71~|8;5L{LdX4@3~WGL|+A?=~tcF
z0X5+AWJ+c$02~~`k1+K`6Fq4pvZUBwYopgvdFfPoZfCQKBopWJ96dpt9N#D$XDko<
z_V^q@XQs&8=;=JIv~PzV25BS;BlVXj%3<=HZ!CHvKs%g#whN3@^!51jnjbh14=a$)
zaCV)>MNSw+ck5<Iqa=7%kJ(=Sh0+Fj3%eUx%;pj4V}n1OreBkL8Xth~utwKsZoSB}
zKSnrcXGnP=T2p++mr9waZ0-A0pE;<8&toJ#(Q$in5%@hrJPq+KKzgR+Tz&rwRv(^p
z=^jX8K<7gd^*QHUY{3t=F+T*qA@uGUiDyx`S5b49@F3wTQMmmv9k1H>7$a;*aA%!C
zYAhC?lGve1SdK0rwF5F2k;O-XZ?f*i0-8fye@``m0Rl%&Zf%g<S{JU?J5c@YW%aip
zzyOo`%ZFDJQBjV*d~pjcTuoq(Z($a@<cstmFH8d*mfinsk+C=j=b%vhc?7vva?~2a
zu7{l`xh`?CeOrX6fk59vnB{G7;FxW{t4xA5OusPe@G$Gk%l$sZ2*cMqj<RYMTY($=
z*c^L17Q^<H!5H3mjM)0I6Irw)(&Be=#nu$drW?jiR<w+t9gmqg)}yN&Mj<4v`>{Hl
zk1k8C#EbHaa&X&?>QmS340{12$lTCAP7op{C(<pVk5)rM<|3zWG@e6ZQ5J8CbZ;vN
zsPd$;_Qb5WlU>ll%k_OZ$DofyGkVMo7w+#Ls4Uk!&IHJd`&!H10c~gMh2o#aD#RL@
zc`G6%7>1I`oW0flqshqW+dAk7OZ-^LCdG91FfhbFSW`GA$*ek(uX^eUhN^lXf+Ni(
z|L1zf&!VoTAzQIizqAgi;r6QR-TP1eBP~rT9VH<VI&(scU&4KEg}ZZEFA_7==l&3G
z`Vu-{+~_Icp2{Mwi@**KRZg#jjPQFE1~izhNS>{7R`U3I=f}wyOUg3Zo&ba8_z*Mv
zgTy_?*9CEraq->>4O5IJIn_(F9*Zw$#PwI-y#6-((I*tzs0&6QM`3m1miKlJ2DOUl
z(JtZ~%^1j0WLRAs2QT^0NnDi$Nhb1#v@#DDLx6f~K%6Y{vl(z_T8Y_9la9Qg@iG(L
z>Wlv103>?8NUx~GOP_8aIDmUw6Q=;C_P{VKPkAwd)Vb-Bm|jUGuO^jZPv!HLG}#Ca
z<`lD$yO5VK1oTmer?IXE=s=+NaurJxx~LO6qI4}FZ6h~Cq?3wPS?N8?BR>j_8Dje@
zdDqdxnL3G6yxuDc!m!Q#XZ7$u6jEE@8^P&3QaHwdTE(z>U5bT8HmAV`zfwJ=z_L3{
zJKRSsh!+(rT~=AYScofuezYPbD#!lgJi6}qtBAlJ!F_T%>cGx$oXz)LonYtf!3gh2
zTkfHAw1G^3AFi`Lbv?xfSTP@!No|26<?f(}<v5H^p29aH&Ym5c<Hu9RzoU`J?Trnh
z?v?Vf2;xrSq)#ezw4RRQJA#tZEs(z!J6+4d^Q=E|#LPE^Xf^^`Dy2lMy9x_~=L1M*
z)gJ_|`7#iH_;I(>%Ixv}cVcqOa>?>+Sc5dX63yyy0Aa<}S+HHmZ^>OVrmrwYYs~8P
z<CU#fTbwd1w#W0-t3jA0s+A~76kG9|B$_0i*L%?S)+5PXZc1J~MP)@++?As-5%mms
zD9O!bqu7Wo_JxFLBaH(5K;oFxiI=1&->xwOQ60~G%%Y568@Slx5{lF-L<bzsuo*qp
zFKe2`kAOVL<Z^bp*C7T$uKg6($Yp%|+p~>3^cMvghTQ?~g;#N&e!rizPrFl5cPzz_
z0b9d>dVJN2=Ryv|T#9>tm^3yW()~baHUA1C@=nuQYwh1DPV}Wj2<#j-Qq^y)#s^NN
zM8J{x<keoe^roRJUKjoE>1_^4NC?fwX)Xr1FF!saQ9S#dru*UaKHe28IMs(rK6>iK
zdP)+94JFM{k&tT&oiCN^APZkz#@uW8py|AYbm()1JTyv9&^+;S=D7nzX@RJVt5wdZ
z)5BQjdn4!b3vGipW_%*(8Tg8K&;u=npbHirdnR`94yr~-y(nkz+v1&&D;Oc-z)5LA
z$VFla!HTY!E*=$ooTK_OwD$FXJgNBe#Z%~jeUr97DK<Gol%t-#=0b|FE;ygzfIv+W
z0knkO-h;ttU4&hsM2C$#;h-<IZi1O_LfXws792P^TVD5GWxd`=$xPyTtxB{`VyO8|
z{70ri6$riI)La0^P0h>6XS}Kc4uviS60GKG;a?N75_<D&386d;EYpua`3mjZF2!;z
z3&Oycm*~G!o=WD<F2^f6sxEk{W>bTpMwWA{+jm^#S*C9rGxX)Ps|9DMp1A!C9N}Ch
zE;mK-#3{L(){x&IbDzk+`ZmL!g6;wjJ*|&gBNxG2dE)t$MEVL&uZ8$P>@f6ZCmZoS
zoN%yXXlMmo>j*!Ob${hqJTIgc-T%?lmzVg`o$DlV6n(5mb(FJ%NAKxbVhJ4ZGa__u
zJv^eB>|=~pe2(6*JH}kWfnI5B5((u^y5jTUbl7E=XM4CSasb|VmYk>mRxz;B3rM`n
z{mRHobFPkG<G?b-=uZ(JmT*PljTy!lB#gaRq(9?-E948EwplDJb{f;37C2p`_B<qq
ztvn{4R};4et;umT57$mg$thr!_e6L#ek0UUcC9}fM@o<m4$A4ik>b$7W??c8dfN-f
z8|-{gBSrB((Ms)oahoiO9Cy$!!_X(~bB>3TECK<<-AIe{K32n?`HpHGF^IwJ=zMB4
zBNVxzc|#WtRYrcY^^Hg>%6Z0wL^oHFq|QEp#1C?f`mfC|C$y`OFu}1}XJ$(QZC~?V
zujY!&leu{LXaqd{lp!xQNO+6Gl_zqR(HeJ!1qt5(mC|My@MGUUp--X`J_p@!KcU+)
z*X91X5t{3kYV&f{O%>Dk)mARN&+}T1jf@y@IGRxRJRZF#)paCd<4#-uIj-&tI5@ju
zXfJoOy>wdHH1GTHaOPM-_ghKrk3(8sxBaV!E@QlV*L>%V1U%RwZIMyEGIA)vW+fgq
z#mTX7U)`*8!gY**dm?=Tv+YdwLT+W`?{Zr_QqWVeTH49+Ix_yMM~s;tng>5&edbF=
zFJ;=%%kxMdGKW`q2hj}qktE|P9pBBu<rGzjvHJTWm4v?Y^VD0Btp8F9{xpnS?f%!x
z#+H=^d6wOMzF!i);;(WTY9YN^5|{Zx4teX9%bYm%wDK~(FA-fMB*l-YtjhP_N`z*$
zb3fqzG{3<pl~a#Y6~R{?w4v#cO_J(9{VMiJMi}(8u~$!PBsn$nb7TUcfbq<+cS!qU
zNy2Ib3kPfYGwIv0(53jf;Po|<(|3m86!;0{&Cd~*5}2P`Yg@(*xsq6QCk1Ie=&7ym
zwyMDTiWJoZ;l4p)E}HZXv?5`7OR2<?snOB0L{H}e_fVk>R-bds=hQ=cc56=`fAzh#
zJv?4UEAym_vHhnVV|=c)-@H!pBB$<Fu)Ni`=Gmp%yxSL!M14-+M@#>JD;{RNNSLvN
zW$RaJ9V)tTNixkpU1AOfrW`dwpS-Mf9^IKz{n6iyIh?zQFYGQTZ}m+B5}DM2x7fH;
z>f$>8^BPB=@SSK?)ljIo_{Ej!Cym7y(l|m*eWVAw&-Ez_zHyA1N7Lf9_<NkexI1q8
zhu;nqRaxv0R<)e%Ig=%>cl%0WFh_7Y#Sz5@Cu#S2RCQ6^oKNRre5JA5*fiXcRP$#5
z{k-bIotS$8tr#70-YW1qUcgA@XD|w`df8XXMYYs#>K4qr|1w5-LBS^`o%$U5XOZ5h
zL)2UqLS%Skc#iS+$rt!VG>_x3LlT%Fs6f86ztKPE47Zg-Q`PsRXcnyv_5Slx=yr<2
ze)j_fwI+7-lIB$=LaDDB?kNtfk>cH5w<Qg3u<{6v@FEXz^_|W|VIa1an3lQSz6c$B
zLlRVT-!CqnJ6=1JEMoa|YGpb8QU73ob79I%)g{ZP4V(PZ_tujKY&ex)KbC~x>BK%`
zp`AmBC<O#haR*)o?9lMFJU|@A?Zolao#4snSMYSRm2tTL)?;8H5u8@|ns@jXoNkLh
zZ?>9nXiyxf*}s~`bE!2&I#V@q3L^dkoz}%xda;29>OE<=RmEbs=dcp`7-&r(dt-O3
zA?l0~iS-Ks;3?3dCY^7*=5t=t-~~ljQU^6HAw^p903kEt2AWwQrM^G>O`rel=Itt*
z^|PY1r{V0peoI*E%cFd06vk4Z#!{=|R%haIq$WZK8T-ZdqBGW^QY@__M1|DmRus@$
z@G9QTyz7S7p4X}@O?i3BB>XAz7Jfc@ub)aQBU~ciU`P71%Bh0-L}H|E%{sg6lRBV*
zl7-!_f<X(Xp2*7}s4%7)J`VNMkV7Iv%W_pu=OTKjE=cM^3J@CH$6k_-5cxJ1rcFTF
zHmG%l=xd`c<?tGkaU0uT`_=~iZ9s@87owcQ*{nL%M)cTvFBkp8?pDw!9U3@OSe~t8
zq}N<X?ymsNB(|5!gJr*k;BG8Uey_4F!_PBI?>Iqw+<xzEFQe<PpH`wqN_z3C4~Nvh
zdFuUxFH1)!_~-({`gL`vq|ZkV&k>3<wFFPzb`E-VNG_!pMps8-5k3^2%W^f5M|r`P
zcbdJ5>-!jHSG~1yWh_T*>`Xi~XPC#vl!fN$qkDj8Lt)NU!nP~c0m-IQA88h|Hi@;X
z&Iaz2pQ}1w9vE{v@o0QXdN#yIecTd>9#9euJN4a{QWkJ-O-ycbUyc1>as}hep?R=W
zwQOZ?<h*NW!&A*VBZl^BXCDvAE6J;36%sy<>C_~LC`T0_1d@^Wz&?hqYIAQhhAv7j
z)|;2tAh+9`#^N#b>5vQbbk&MO&H^=c@@;TKBtF%dp0}Q>sI2j``HtOCXE)0m&)AXG
zH6*y%I5SVQZPHP?2y|<Ac^)BE^s7ck@AGVzcVR?&TaS1XM<1fA<$kAkxTo)#=Cuqq
zR!#fl{F7i|UklJO8X2^IHiuNQ-WM4^knvxFD?mNWAY`p~V5V^jNvZ2WHn0~0@Pn{o
zvSQDmn)~slw&)v6+t4t=3U2z=i6%dJq3@0wNBB)ubyFI-FR1WX$_Jx0eFJnEhEGIA
zaReS@kr30qtgv><@%-(Yup?)UnYHLOTC3d-&Ywv|qTOL2T{%ENefE>`iE=^vpEbf}
zK+W>7q3CdsIW*x-GaW05;P(oSpS})M4p-3%otMLZB3Twv!&KYrU~MX)or8szn;nOJ
zv@R_{J4$g`p-eqtvCc{X%)TxQbkmNTUz%z_Y7daq4yC2d)n9F0&%;X+<N46<Bjs~5
zJ;kX0kZFW_7o^sLQHqNy!Xa(^crD`r5onrC@PBqiEupE_#8d$1Pb!jPqLrY$TWbS7
zjXII!g5mwM411-k1F|g_`#v9*y1g>)$ZS&j)+pxSFHW1_ul-v-2-~v%7}0zRR_`#v
zR+Td%%k8%Z48iQx@&pGT44*`vC0j)*o+sD!hYm$8LwsAWI+0!8=edL6QN4n)-{<M^
z!yVXU7(Bdwo}=y-f64O5{EFmCBQ<)uahc+&gy$;1Zg4|@?|w56XIUxUHF3kFX{K8B
zk+`X<<&qMBKJXLGXF`EuhR|DwCm&8FuWOwNJXoTogU#H#W<{P)c`i35=vsj#%{XCF
zwqr><4uhCkg~XKZP+^*DA=%Eq8y?fFe+3T-=1@EF?Ug>vA<&o`{)5OD>fMZXt9^mI
z@oUB-!pEi#bzY$*pt=Y|S4Dl45Hl;qixpK6oY8DP&Zzm?3T~u5bEIPZ77CNfu>m4_
zl{f<yMX&3kR^H)|q*dx9no32@_~R|%r8Vhrz&zx=EQ8cXj`t#Mo@riJUn5DJGkhOB
z7SA9>ZBtO&b$ZbxcKQvE9u#5UhO{?Pf!$sjtb&4}#$=mCBCnQivOK-k`0RZpM)!i=
zB_Njga78A7ESUB#l2YnBR*|fk#7hwM?XpIAUzAF?0?$$CqN#pDlCTmS)&z+`W_GHs
zpOHhrnn6@e5tbZIZ`of2jbgMP93-uKA|+$jd_SZ`VF^4makc)KM+9i1If@T21f08k
z7l<O8(mq$u%#`$BU3qJG&Wqa{P$)H8YZ;DkLDd!nzz2~Y0n~NNW|6Ri=`&*DL$MAz
zV(~IxIKL82PC{0;NXkIrexhYolvM|9)-}x~R`OWy3&*oK$0HqL(f3ETlJcx`Izb-2
z1aZzSYH|X)Yy`{dei+9$ROMno^+eCz;Cna(W~1MYT$NhLHME8FY1@j&YamU4V#*;8
z?;XpGQiC(1quzK{k;8MN#8otc&AL4Gz4?%2UdrVqcWlAH;P-;x7D^K(B?3tk+Kqme
zts2jv4<_bb!f7d5E4cx6&T>c>aqNdBn)PgDuVj*&tK-ijda=pNn-3f8@K;F$IOf&T
zdAXlv;;w#@GUyPJWSj$fwvQ|Z&;u7r?=dw!X0RpdMffBlGvzu!ZBiO@vvy8+69~R)
z5sN$VCl`MOkxqWyMXenR&h$$<GXR!g*LWbjNxjBj#uJLa^u7*-!;%~C7m&8?M_SMr
zU|+tM@M0!HW~CYz_!XFb#<H?2NetJQXSxm|?)DctX=7ay;nOMUVOO%5l44o2%G83~
zNP!9wC&4t4{twTk?}}uaakumfAOb*uSw){s@%6IV@U_yA@=nob8$GUP9BcV-$Yd)-
zIz`_pgxn8IAQA8pfacl2x5K=&XDB`-wOag&H@^6pfTtVK`N+jQy|vL>wFQG?D1aI~
zg4U|W(V#jk@1+$<b%Xz-;oYx*!XSNI2KR<4MWlV}1QwxLABf+SD(+Eb@dD+;gSB4Q
zFCR_Ur8f}32N{q?bn!rf*{+wVmzREn{iVY4nS9OOY&M8MoMh^kpv=o|mHP6!zW1`9
zcYIZzr2wXYXg)`>o~i@keoDiwEVLN}AXpwtD6-gzdQY==J=m&du!)V)A6Ir5Qc9;@
zon4aY2@d2WPNal8dOzsfeWW%b&{;oHtLb;ndjsJU#O&vbvoi@&I5-cpvYK%U9U>ia
zTkN{BrjXQec-Y9zt^PJvkUOEYPMhtg*pLS=%`oKdAQ~0!G@0TA5DYZ}%h(?($)vw=
z#~%A}vF0Dkl?Fr}aVCe1z78AAs?EASWO?>trvD~_=rA26G>h)o+kJkz{XzAi94=_E
z45k^9GieGvJ`-mQePVr~6f>Sw!BfJ^)~ub&UiWb<-G0+mCLWkKpq&kq>$oGCP&%49
zAKyZ_SPRd}nDApbN1<;i`i90kFrOM##wI)FGa5m|)g4MA+y*X@v^+xsQS-v_*E`Pr
z+^;}XvlUKN$0?+#A-!|!%?qZ0BIv0mM6ciOWOD9$O!`1IC6rW$NZptryfckiE?>9<
z@EX+jJ-TeHz-UYWgu{}fV7#E2-wTCw^feG{Mg5?GoxJ?3hM{@<OTkxNuxlqgJ{<We
z#4~!$EG3*?v3@bUTmfBDc}WW4y5ZrIRh@#++eUMRT+&B6UCk^QcQ6y-Lb?!k(odp@
z=wX<WS1V^_$~<q}Rs5RV3)$e6uEj|AyUc!KC){i-B)9A-N6mO`CLp|hw0w2(HY>H^
zt5KvMiA-n5rhS85^m%~Zw5aaos2l#1!Z0qs8eu}gF59X?#s&rq>#4)KpZzP;6Q__O
zZfsc|WPbgiWnJ`#ig!Fjz~xzu%|trtr{5Mq;1t3ra#lMh*^POKnYv?{z(b!_ol^i8
zAkeRU%A+Q0;2ck01bAp3f_&kGS?`A|oe&EN`}YkC4M})uN^xp_KS&iql>qA-S{H!G
z^!LMhKV9d5S>xPk?ftAjpVK#-;W_(de{++Vb+ZSWO<f@83FJTfG$j%xEzftReU04A
z>fIv4EA<^VjUCgB=4x3k8_ccc34BX*d<r60HlP~XTP}S8J1&uH{pr!1HE`txYblT}
zJe~`k<8$|-A)VKn)#F(ZiA4|7i0*%N8U&f*Z+ub6{jtaY03mP{`1RM`t9-+lggyxo
zWhn1IUjP6Uexh7~{4)|`snyP89$Ods-5CpNjl76XUayFqy1qPlLMyZz@HTxO7^5CJ
zH&YLxa+8mi{qCd~+N2p7Q4dM&5h)gwG{!v{A#X%AjI>t|4I_Qt%D1-WlWCA5PhVQQ
z$xe~yE&|OcHZp105o=u}D&P)@_S}gI0OE)QpQgI_tJ*i5TC?G>Iv8?~^lDn)JY@I=
zrPa+&wqiCB3OIvQ<>o?TCZ4n_wBdXc4^D(UpnXSfea!LFKmi2gLDvicu=UWeV&0+y
zAaxWHKZS`JU&`Vvfzu`k^$^LKY9CQtQ>neZPaA(zo#=6(Yg(da+Kj@t)64#ZhA%CX
zXSGPgDM*r!v$Xbwh|Sixbn0dd;(5;d_1q%So{7S#qZ4cTTXK`~;k>(o(7GUmdQC43
zQ3NzZkf}RV>a{{(Tj}6Cwj9|3C%&%4M@DRTo`^s8R527^bOK0HfY-+AN4o)!eh<Z9
z97}5v0x{!H9|t&NaDN`zyxl#dGjmb|d&x9*3jeckcVXB#QbRxX*7vc&LP%f<ky{gy
zqTP>dCY~;n8<jqJp!Z%R@e~(ttw0pVC$fpR4k;0;wqEH%gw56eD;&J~efYmB1xqkO
zC%;`#a~_P{=*yv3sRM-rPJSFueXqmCX+Sv{wA9`fEcD}ujI&~mc7{8kg^h4gBfhn{
zwAE_gc{WJA1TBSKXMZl!0}mCecHmxdp2QPeS86+Emf>g^8b0g7l!!ObQ*r+}V?#e6
zlRva+qa#zbKij<}>pfe2GT)L-IMr_N3F@nFxfxjdx9}<(eJ>I-srQDqsn8F#0V3vx
zc4Ai4L^tr%>EXf>&F<>G9b!+qfj*;J`WWdp|Je3&IkB8YpLY7E-iuqDV=3k-2q|c?
zfVyc2NrL9NlsJcakr^>VhIN+KvaK!c$49@&X{mA$H$iHEklE$v)sLv5V_Up;GQ@2!
zoZ6BsSsC2&*HZ5Zn>c(<9(6S${X>!7XNM@y4v3x+2vd?(xI}}<KT<rZk=Qlr;SsOU
zy|l18aQ-yM3t9`1eT&Het>vydXNGhuAaU5rDm{;Sm&v@0ExSxdZ=L-9#ql?+b^GW(
zH>a>Zd-3wg>_t~~v9q;-B_%bgE-qqiGXDFouwHQx_fJ!mPPwqfZ}Duu#`+o;X6;=)
zfGpoGwXemyFG@^JjX5>6yJt>C|LD$dxZz%kNm#J@&WmL3M%Uz?M^0I<U?Q_`V&n#y
z?_kDW-M(H#81}?HbVcH-?30F)E}Pn=p)TH<eVX#8JLw%0dAm;oG00E1bM4j`nTELY
z?|uYy@qWQXk{eBq44j|jqhZqZ&uJY$w-*nei2SJ3O&0<^Z!(+Cc87odeXpF4q;Qk5
zTKjFoum8Wd2lbryC{8F*P?r&fv99;H5xUI<a95m3>g-u{;si(ClfBQ|N(Kco@2R~K
zeID^R>g=om@weN$Whl(+evoXm40D~TVd}fcApOXY^QQxk<02_?e$2%$<=GT4`*9dX
zKl6J1o_L-u`&p0NX=r?3mOd8$CFT?^as_zCEXAX*>Z@q#1J6iXZ{!`hqzm|~xpi=J
z_$HLE@4mQ^R+S7**7MRTED57To=@}p6^Z5ve#Sb~r=PbJu%k><IKmlM7tJ>D778HR
zp>8e7qo0UbnSgsS(~X#i?SQtAhid|vZJnyFXcJq_^O4i(32w&E6*((jkIM+5FkBk4
zg0*93DfRRtn+_C&oAqzh&kS5yKn}YZ@~OrwwO$cMwNv<Gls%Rq@3m>ZxD@~&!|JDv
z6^3tm5)~Dcqw*VfgtPHKpB3+{S?pI6-89yf)#75JIH&#{@!F$KS2pk)-9#q23gK|l
zmTa-<z^&EFIKa~^8Y~z&LQ2Dxwk12@BrnPz;aRul`<2qM)=S0M7c*lyx3moGWae@A
z&(y6jwB3qZq8pi}N?5)qUi$5JSv4n8&B$=LgXa|L-qr6$HlK3tj;z(GAvaNlL4)?J
z+=)Ahyv>x0wOev2x1YP*5jXY?_@Xc^+*`+$a5i2o#5rupo;RV_4Z98!@|9H48IDtj
zHoQxl{b;X7Zp<VwjhJ}u+jCUv-k!a9Cys{kb-xqrPK%zyIb-w{nD013uJ3NX4zp}H
zp+(Bi^WR)hm@g{JnPP?PQ=JN!jDbyA*CW>k*{34|KIV1XnDN9+eIGy>-UrTv^R73q
zx?WavD6VTOT6zz#O9A$Strj4$=%R71ZHXO5B-WEkbkDZLsuF|5q6c%|ZcA(u1^F<8
zs)o|lMBX?B9-yMOTM|7M2*GHXSJQln>)fdA7w~?9)*EL}TnO;Rp6$?nwdd=x07iAf
zS_IaadwHXnZWw_kUPmRL31@bD3t;6o!zYR=Lw$>$@;ZGGOJ&IIv+VYgB~ZM~iH%EC
zls|`yY+OSa-P~3$o|tD#d%Q<g{uRY4vr7+Z)N{(+pfsV5VlTwx^=^)@-6{gS6k9ps
z0<`t)KDr5vq_w0|sf@Co(v&SwICTmYv3S;dbA1#jLJh@5j)mqkyEVDGf6Ar1F<lZM
zJI*=A>15B$1n+GHHYfd)`CdZMI@62RPVp|SSqXC*D&bVQVe|D<2||nOMd>E1ZW$5u
zIr3)9`R<i^QU2!G$TqVjd||3P3O7@q1s%{{Jl~rEaZc73w71IF5Fm1pmHAQwV3<mI
zU{LTD=S(h{0Cv(i^nw8QkXt-uf+tUJa&9Glw*5Gr(>XS#M#>WIgbLq?Cb~>Ib@@G~
zRT3LYzQKsD$;lSgK8O)%h-RzE5av(NPbeciO}sC%LCN9yEW!%(hUfk8i`5B0BF}Po
zJ4dsf=w!Rup>!IRg@4FFUr5Q`5;)k*zOPIIFQHlK`*3aDbTaJ$rH)gb9JQ?T(cY(5
z)}G)@=fpnG9~nwtcg6EQJ(3scWc=*Y;uDH}$N}i<u>bRfG7}MYg$F!)vwgpTj=g42
z7-2Mc%U|WU(n*`3%%-x^(2cbpPf^~iM<7dDi{~M0BO1IIxkCJ~8og0M*C6O^qzoyz
zuS7)G<W4&beo2>nb%vMe%-r+(2cA^n9yDhuZ^$p6W1?fHPvYG&87QeYNeRM58dG8F
zi{i^n>Yg;H;JL22w_p@U;{fmBbfk!GnX7il<4U6VR>SS~J9G`ExFP#@3B7|DSEYO9
z<J2`<YNuC<CE^AwgnsC87f$Z4z>Hjc$Nfvt`EH`{R~E0&&9qu)G{ntDs{mU3t9bAy
z?WP0sDb^T;0r$I@fB`Q>hl^`p%7AueY|699<f=So%y@nk;9c4JZxngU&bb+2&JS(g
zTp>$XF!3#eSfFt8#^~VrUciHCmU%V?>bJNhDm+hcrx}p=#sln0xJ3*=T)r;p+lRgs
zS#_V>HxRFcSrb&4Wc&F;cT(+kf`9%n?(A!BlQLrPCWt+#I^DAhr>{DFs(=2C@2^$9
zFqB;oyK(Z@|Kk&VvD}No?9Fd*Bv{>_S4{Qh<b_1w2Yo%M_asqIwDqOYC&zHd>eEbN
z&5u)j)lx0iN-a(7t~(WH5x-+)jO0@gby6{LkMLA19xRb};hK041~>egmn3VLgM`(y
z9{u<lOugt>3)5H1uJy)HFU>NqDuy4~s_eP=*d{vey`O@@+LS&~jWn-tS703PavrO&
zj<Cvd04#gku^!6X773EAOT-Z$Src;bNfeF<jwz&#Kp6<#ki4ku`P7Ox!7s>5$a%6$
zG*&=guE<g{DGPsX)^Tf{N%6AP$;c>#YpE4ZNCpA1uf`nW8|56xPI^R#gspfRl}z_K
zuBA}j5!J0e9(RK3z(nfp!w+LQiCItTL^WtfOf7iS7SMbw602#j6=EJGKgt)2x8luj
z{2q7}WH){<C6VHNDQ3&3wtAHwr5*+fiM+LUiax3_1HwT&jc@5wbTTnd$Oc`Dppz-x
zvzH6MHw2%^@kGz+TLv~K#Ni`bgukB+_DJ=~SD9b`f{$^B$%ie>=V#oW;dy%U={(zm
zx^y%Qr!P<U(4NBbmVAaHVd05Tv0P>W8=xv-I@k35nYQV%-x0)3JUq{R$$7_ro8vtw
z9(+gB5xgtZJ0GAB1AQcgnqqqzj>lj?q^w#y19GDwA0B)DdJbL{cSDg)>gGH?f}i3@
z&Sf)ofy0p&-M}PpjOtse6tIXJ%3b_5?p0d&Qa6r%=PmDPaY-reoC4dTh`&l}yU|{j
zB(ObiB1c~M;lH1FkG$iDd1W+&@Uu}@?v)b}bLnI!z6Wgab-6sCpwONhYOpC?@<Eo3
zh+JnKMi^LjaD4B!HxeWU`H#`<%%S`}IhwM2V+|j>_b+a*Enf`2o_Brf*SoK`{nH2)
z_@}UZsqTA6GyV6V=y(wl6V0M4Ie^j82KGX)e85vREPEE*|MiWV1;j-2bp<reCEG9f
z$eAcp+qdw}>Gu+sTT<C>h;-_9&FqXK3mL_j&Di!=)Tj}kSW7a0kf_OgXT^xfhp$T|
zX4yz{Zf_0w@CB)W2<B`}w%^(cPn0j1bo**=|NAKJKD>L^6T0vY3-m)yl6j?ZBc9(&
zgUr8~9b!k%uLb{X82qfNmS~<Z>CW$`YJhEUH1DL@{?5HIm7#x@8U|GUf*e^W^zVh*
zer68}Lx8`!dNL1hN5Ys9k=v`FwetZ6f`mmSQ-a^q--{<sRJb|*_Y-g69q&`43tvce
ze~=j5UN;ySpC{~bvx$|$&Ua;p9msY>NHOakS!?r{mBseTw)2286(sBpN%%eRY*T#v
z)%`y?a6(5fXB^vEHmO7#uzMKd%x%GtN+jINr0Z6$9=<(ec%m)AlS+46y6#CG!h0j>
z{`V6W@Qyd>nyi0bd#~J<h-m(x-1&BeZE;8=0-@yK9ueMN=qgY+2jPxS`rGrAy#x!N
zmiMxMd*8os!Hb`_GB5gX4+a^gCn?eV!tP}p#x?DWK<})LCn>CJ431>`m+;-lLFfK{
z;s(6KNh#&{&i26{{xYx{yT=1kr)G8qd*S!$93g|HsA^PReDZrdd>pV8aw%*~guh>O
z^A4gGuNSk0b_N4~{LjfV2lb8}L8O8aq~c%W5zPydR}dxJUUgd#F~*R)xsltFp+*5(
z`S&)QQ-EpT$qy*~ODgV3b;FzgwdtaH^Y>CXwpR?Ecq6&CTi!vu|9+wk-f=@C{ql}%
z!NIK;(VpGo!6g}M>NeSJ3$worNP%tM)6#8m3VH;p{r5IRU4s{&%Mc$wxIGwTJ5~{M
zy?fKa@)B}s?X1`;4M>4d67}{k5zQY3%lr2e?~!*z%U{}7S>*i(0$?@%H6AkV<fbPf
z_qU~h5tbq#Ic}Wo_jqn<f^`k~dmFw`k`KpRn=rK55lQ$1_wJsQ!M}_c8MpY&bouQS
zgD1us{-vF53A>)ihj+ATa%}4m^8TB`pzHp-$Kz;97arlrxGe>}pb#RA6SsM{7bp2R
zDC6JTa3=z${l)x{`;I>b?fxG$a_WzNuih`CQFRP#^6v3~H~HHZz&4CE^qR7}U)feh
zYEZ^q+wc~qeMj5<8^hn*H_v?(*S34p6J-<*>7aL3Y@QpWV7Ipc=koUxj>tO%!XN#u
z(_jlwn179jNmur7TObTlK(^Zif-?TS4bRzN+HX6me(Z=O{P90%l25)V9CRI0(dm7k
z`7YKL^L*C?p1}quvStQs??WO3SpN5F>N_9YR0X*vWHS@r`Pz5HVaZ-$+FoFkJ~%8l
zYKB)BscvDuliSfXVwxsEN1K(c|A`hra;|a1hQ7A;B2@Xke^GZ~|8_qCBB(7?cAx#X
zAK^L+OT4%7|AhzJTKoU&@ZjW|Q#hfV(4rk-!Y5Kp01=$O)=9Z-$L~?B#_rpBpcx{{
zg?YD5`famX0De7o!sK56j)c%75>ljmHE{|W{xXp&bBO!Kbbw?@^m%9Ng%9i!Inn1#
zl(C{!Vl$BH%|uR*tK{)>u|xA7L3V)#pEcrVpPBzCaa|Z2S)Bn8cBUv?4urL|hHTLr
z1T%c5`J`L38B>G%k2y4;t?s_~=x_f6ItMkxfxJFnC04!&S|>9UcuVuViUF>m$(dDR
z|Gc$zYS1+g_)oP$J;#lTb(PtE>q|KwL<a{oEZVPbe=to7LD9O^FwwSOsA2^rh|;aa
zY?~*418|4a?~+dJ1VtB-Ip=?dlWP28-YYF)RwZr9=?=JW3f+cT%x}%mUqEvhmKpLU
ztaS~cp((mhA_#!(wXJYu>}WoQG)%W@B&@c$JqM^vk?3HQQ8kx0u*_>ApxQCBz~es`
zAkCAc3w#z>%Q<LL_sEAH-H_*kjWLjX&OjK&t2I|X@_IFjzBzNwf4%N$!uroqXaG*D
zdW7_?jhTjsFMcxz{8xIn_ozk!8n7;9UrS1&UNcE9R~*t_iL{5?fi&R=lVz~B6Irpr
zq}0rnu+rPsVFgb#@5{PiZoszit5Jf>lFuB`*<Px1$jW@NIpMw?U>uSCwK5%wr1JuY
zEP&ZEGeo%W0Y~!VgAUhPP#6>Ov1_f+pQA_h9r_bnPcY9sK-oKy%p1R~@ST6p4WBtL
z<Y#$WDLMho2@3P6LoLzf%Siw4AcOA(7_<`|*<^vg?IrGh`Hawab?)V@H(6dASa)|F
z`JGUITb{Q~>$X4VE5?t<jGt}wvJg+|Sg^}b>O_&wAAH~jkODH`jW&ml7c-^`*fX7!
z{z(89>Jv847GFbn7|bDgYOYL=Zrj+uY7KWWm`~)ly$LO9hU_vy#p=h=!^5?r$OgV!
zw0L24`)5&+!w`?AaRI@qGF(vgHO)Uxb~xrW6TvEtsGs6*Ns?6sSA);cdTjd${QmHA
zRrjWL>3U@#!OHu2_27RG|F8J|*MD6+T7j{##;6IHl<)I(YN8$<dwJH+Ebs)*84#k{
z#Js^*F$<YCT1a!?0#<S}=gsUltVDtQvV|YD_7R)MbmFDmQU+v`Kst~fdM0^FeDF*t
zokIyss9H0=GG+$Pa|+Uw8P3qq^z_Y4puzerCjdKeY1+WFHWyU`UP3mo?5;~oGP4$U
zev7)1;z<iR6y}WHwl_V&0p9h+!LjUL5>tMFrV!HYXvokPAgrY9rCZC8$V>lRm0@2t
z-h0T~B_DdU+G@Fg_FbH*7vTx`x`COY(Su4Enhg-;N$tiglhCl>wh2|!>zP2Ktd+Hv
zjeBHL?rT+Fb~(ysf8PBY+gkd!L*L$gB!KFrx|}vO&MkEP=XoADi}gK+3zU<7>7)bb
z>nJqFd3wGiUN+WzYsht<XkN*&3zp-gZb6J|Kfx;V)-v{L*-D*_=v^-JH%lT7uF4C|
z37dKG&&Xui(nLD(Je8g*XR%vgpOX)-&4haEZEQ?=XUTojgSxpWQ?;t1JAjJN&}=o-
zKWQQQlkIW|a6JTFG$~{p21JDB#sIH3-BMJ{xwFTI%`=p`!;0&y+?9Tf9)HI!mphWK
z56elSt%0*ZG<L~{wj{BCkgB{G8ap!pBbC~}4lK%v(Ly%*Sl76dgI=9*M`ELywuSDb
z64K0?xzrxerBV!aB;Dp{-dRsCxR=iwk!#X7#Te1~M;QU;2@~6C^}iNww?;)>!IOLQ
zPC37G)PZ({hZt1IId%W4y$f<dUWN>r$V;b}BYaQUjo(Fr?I{%n#1%SWW89=w#cTE{
z$B86_xkF2|r({$sYjpYcgi)w?!RmyK_jQEeyfvnKeQ*?7U*15*?llLK%PjASUMrYv
zE2+V9<tTNQ%cZrG*pf2f-bHV%m2G9nSRn0c;f-za@Z(o`-P>zNy^$IT8C+NA_I%s&
z>Xs>E;~0%TpG#QzV!pY)(h8LpHRFrLaA`&{nE-MFN-K~bWi)sUo^vWS5$mU05H+&v
zvctAE=WbGZc(fJ&(2`^lL%jqaH9_h(k$S5YGOQ=mLdFmST{;0hh@Ohr7jC!uv546~
zaA^zAXiJALwF2|bSCS>A!^3NEyOY7kvrH*GFw2gcz=4qQ6`IdD#*kih`s*u0wfJ=s
z>%E8CP;a5^lhpwTwT!$v=<vJx&J0hFILTOCgjW!`gY;`-tnEkYt1T1Jo;5|`4##Ve
zcF|V2tx+ShXfV2NG_ws^<_x%RBuU=o4EFkG;3Za1c;#9KH?*~kjifMj0>!C`Jy|S}
zinl@o^Nob1nRd8XECX7g7ZzM~3dA3D6W@xM<Lhh(m@;8b$EEin_*dpy)}cH4m0KlM
zJg!4qV$EP{we9*8yIT&3k~Xr+86pmYikeo{4Qn-W66z`a210B@)8O!%lv>R)pB5;_
z4BA-Yb(0{16qE{ahiO4Z^6_65?VQ%pB~AM&cDCe+VA|nY&y+h+yh{&f#n<3gK}#mx
zNj8a@hboq(3B#9|??kv8P=zB+s~AG9(c6-5%TjJ%#K5?-2CO#RArF~3r4aI5nAIkz
zUWoOevC#JMYA2@I{m0E}5=HDOPsml8Z$4)nR=78OMa50s;^I5y-W>dqMTlv%6Eox3
zl564s*F@@+y7JC@?cFFekY~yD%DaraLR?ZfhVVy?g?&H4vQo3ydUb0~a!Z5B5qE)9
ziI=M2+pWO&CsZqSPD2qz+JLzJC2L_+SUZ50gziXEs_Gf~QKMf;ETlV?j@JV2Q3GjC
zZ5`8JdhgcZA~^|_q0uv;u8J0mnwiiCi)HclSO73YnZC`qLSa^ZQgt9w&k==&xv|s4
z37&9oQv#4H33}IF|Do==kYQl;+IYOyG3fJ*P41_YMt9anRJ_sW^^I>Wa-v@P9>Q|6
ztl|Pjt@0aG`1mf=;vZ+{Al66hE2$P8FD{aN1n^oKZ4IVR>6~&S?^^C(l~`>I)kf4f
z11Yj5O_)?RLjW}T@~JYtkmEPIq$?-Mj1s3H8o3?bxJ-%pGL-QQDvG^r>lP7<WUecW
zr6Wo<=Gt`c+t_eR*#Y8~Nr^UV6|CejWtYxnzMQ*FSK7lg<$_i06kq#w<lP5Hb_Vj7
z|Es<4j%zB-{#L{i$CUw{(I}uZbVR`*FoYtYQbO-dief+nj8df!ii6HXPz;0)N|PoC
z2!aq43qokpdq9B@LX%J=6yI|lnVo%i=QCS=```LkK*8kR=RVIl-*(!+?Bsl#XA_Sv
z4!b7{bn+oV6FCDwA@*6ndGXT7o$CxzFCN$zwTO6cvk|L}WKICR@kB_^v~-(vZxHFn
zK^S`{J>)Z!D?QQ_yyoEr;-k)dj|dD`eBHG=cSW>~)S|qx(Qz2p=05-OyQN1x4EoO6
zq#oV%PU~;?zrUU`QXZQxb}uCRCZBp72FoZiu{MOl`!2_BbzD5PEnV=L35TICjsK^*
z8|TqphgC~v&rP;bAGo8_4-Y*v!lg(~?_z6fB$0?emcmJ3f>wH{L!Dc2bL#0GSSS&H
zp6gh9_X!!iRAw4UqZJkWlFg+EVGX36&+E`7qhNy&N>DDW4Bx9)k``>L&!PDWT5tHG
z#g2}~CYK<UG1G9HbhR07d0U`!snhZ0?!5U7y=bbYC&M19OLM$Yj-W-?jtLp?;Il9n
z&V2jkffr+VcavwAxDZ{dn$QqA*jWk{#Aw<$a@0l5lLIBCrDxBoZ+4AlY@3rQC9~w{
zZjGHl*OBuqku-t%bjiT1c?xY$>b^0mCP99>N4s%3z!vhQ*8I${%#qW)9gu&H(9hWr
z^W-ARbs~iz8sDG?jcbZi$mzv&(pIZB0uao<o;*7=Q_w^gyAin4P}mnzBBI#7&rDeS
z4jECGY;EcH&cHx{3$}U|v(MuC{4HP}NjBI#U8~DuW0x<K<wr&O@6O3B#I2UEJOL2h
zYOxU+X)zA`k7w-T=4qmM%<Z$|!^)0Df(K*#ze8~f|InpeWhuwJ>BJun<&3r|`e;G&
zK?veD?sWOnWgr8kZ}9a9XlF8YbT7q4$1y@E#K>sF5+jVewYsTx891cI4J=Vkct7=K
zzn231hD}Yak6_&Jr*VY%q&cNk5Dm-`%GlXA+e<x8yGv*U2DG@$Px(|m{iaJ>LH30a
zvdz^Hzb*yl+5EtAS53B}se;f@K2aSR_R&5~jev>`9~9(dBYEZdSJ|%8XNnZ@Pd(2c
z5tvK2#df6$=9+O3KZtHg-=te*HW@<c8tr;_m#36?qth01|17GH@6xl4?IN((V)vS>
zU50<)XzgT!UY(K2_tkZIjFdgu&{(9l??g@7z(98|1Ieg37&&a9t{(t1Z|s!D4I{Ee
zkXa8%r(=b5fw?p~ul&0k0=Rj$GZWpNPm-g2xMiyT`cPsDLavTK+r*U%-yx91y_~WU
zK60KyR<Zkbyo4`aU=CkKG>Kt}eb-@xl+Z`r?%WxxleZbynfH+lp(yq%e`-<hk50%$
z=B=6QKbrPV>?bzM*``SLK7m{VtKbtYmeW!B0r(nSx^!fdjaq7dbe6>3aCgg#aCV+P
zFD?Jcf&Yab9xDHqj7h#73X|5c(K~+x#lMFPP<i2W)zT0OI4+;A?WX{7y4ZfFV1i6T
zP7kR{1X)?f`hkZT-e@vHF_2&Ao35RMJ5ZKs!Ot7T{jul7#U*7VFsdUb9;&g&&$sL+
znMN!tEJnNQi*jwHDkat8qzlOY<|7n4%Oi_#f?&sPC)my1-}89NaMXfjqp%oONI+^E
z=Fq9Em6#gE3<_7QL!D?ajOcY?+M14nUuE2MDn~X#z9xRz|CQjp5e9Q691oUetMi}M
zEFIn@1A(<1qy2|v+?VI99HbGu#`b$83C5VIdn&+KfE=V(a~O?S6c!l73s#WG6~jWN
z9NJ<iQz6ah$Y*BN_LD*z$%;ee#EIxjdg>T$=Pm4(x5F*kl6lGB_n)pob_DUE+!px}
zWLWq0kvM^^j)KLKK_7nM+a!v5RLQiNzL4i#2*QG&v@_!!Yj0dh<OodHlhB4>!U%j&
zl^J~LD}Du)<2YB2G`+45=M{#Y@&stp?|B;F`t?F6NU7_`ZcQGrD!F^YG2?{+_Egl{
zU9?5YqUm%1U4c0Q+agLmXv!nmC9kMLlb$i5jk}tFjG<w6qAnw$OL?rhJVKxg-)N3g
z$?AE2>!hE9dYm&KUzpoBcIYd;j&OJYLUCY=pDet>^kMZ)j1+ZPJ^w-S^;dHv9x>sB
zqq@t(S@Vx{wCIJND|HXT(%*N&405BukwbP_^z=<u>KXOML)ryL^@E$}o=|W)EC_~l
zb5gvarYzrN<>A<v1-iA&pnU=Ax)RGIc6yqzrFwwQ4QbUHc<h<;xi>iiyE8e5jNo&~
zhbLrS7VE7&n!&PL6Fz4>nNG&L+h*KinS2V64Vt-jpiRHB3F{%zo%}J~c&I68p-4P9
zAE$Nu@My!2XYxZRpVp>7J&ZV<N}vq=g&>-9aa9>6EHJASP#9rX8{ArS#^G6Com%<Z
zDISf}2>Wl6bKGa-W6J=d$+M=W8LhbQKuL}!U>Rw5$34;#M2>1|eU#_hEL}~PgIz)V
zUcZqDYuBjmn9Osh>quYvw?s~eA&+QCCGueS+?g&zRaJg`C22d>%u*`u+eAHa!TpoV
z{T5y!sm1#|!u6F=qyR2kx5w?@`o-vv(kXTnC!`NNP{i&de;in(m~Wk*RSHB}@dnrg
zIzo3!VhU}ioJrHOlcw&3?*+Ir()6l-@f~ZvBE+@W&AzvcuLL=>?W?wR!0Hd$!i^kX
z0U?E&yCr`pW8au{fxW8ItgTt}OdSMI+T_deo#08+8hZ~a_1tTU4zeHf<rtS(Kbu$F
z4<d7~Aenh=FX_*oiT3Ar^r%juWyE9TNR_kL5y9NH9n9g{4n{z0Rv!e6wS$eta@u87
zAM!>neb{v1<1@7!hFo)JzG<e4oi}D9HPp|j=LH_sF8C&@g}47wg$fN?WX3qmyvt&W
zq}=u1pD%a=w8H-FMIN?MQV!VN>F-U&&~GIdCWZLIam`s}1oUc7Xiy6Ubyc>r^RzxT
zE+3wsJT3I&0;0McUkC*>(V6TCr{ldnL9A2KC;ebvkeRRh%E3A~V?!@`B#GQ0?{rSY
zC2`pGNF=9%VTEG+_-Jd=OsI$nN+fy~+Vm$uu`CA&YZJT6q%jhrgBfknmk5oSp<ne<
zjkFJ&hBcM{Wp1S62>snog$V-)67!oEXT|DU6ry=uE*?5l^n`v|G`$lKNLjwAd{fK4
zixqnAsHuMFH0FifCmL2Qk(wCkMDQ#xsiUlTx3_WKL#MFc+X7@TpAdSGULPx+VRp1}
zu3ldr5(#q<m56D6U@}jN?i*}6uN`(@#{09GMm>j8cLYG*>6A(OlwhnBBY3ql_;JVG
z$&6!4JYOGH9SA>!`AmQ~5>?hdX^U|cKcIbF2g58vIqq&|*KK<3KJ5e&_wgqpzF+H3
zp~LrVcRfPlo@@dU!f4ucTl%s;f4DLK`67esVZ!G^*(=l+IkN?6B*oq5kw(OhuHl~|
z*=gacX+V*7I!OVsPo}Bd;)_{>gg!ZHW#s_?6Y|3LMj38oRUTi|Hk!R%>?%o{WThlR
z{hH!M5Pe?5dC*^sBe3#()}L<S*e}ah8kR40n=3vvmy-h5zu<Ettw21tiSffMJU>45
zC2<5y`GCT}hj2fmYF@_6NKgt1eI32ZG{LBus1M<4MGcXmIeZg`=Ho*OU@`X~{kV9l
zO}Y-UW+nA2E=sNflYK@8j@ranq($=cY27j%wD{EeX%cq}24j!wnTD~n*>(|o#$3X`
zkHJ$R*`_Bb?V3?FE}dkEg6SomDmeOs4><KnT<xv~?ekga-SrDa2X;k`%D3^N2w09l
zXwsds!4*^behP0=O*A66DAKTIQ)k|t1NEr*Jqm)Ht|bV59cwsR*V<2|7brE8pE;}d
zTLaLN%_$&VXJsjzSXK8Rt`d&-n-X-btJlzFM^yVb%Xz-0E(k-wm9aJU9=`2~;)wFb
zN1PW~3W+QPtL&HyRq?PDA$h}KK$EJK5lNT!1c!9Kj6xk$1s>0+Am34NxZ^;*ks`!N
z*7HE^7q;J5nox?8cC?EZAV+hv3+)(B<ZQKSUMPAU=Uc3EI@s$Lu5!ESG>4}cKJ-Pg
z4M&=oGZlk7laUO7L*CB7CF>c$a>~CM(_WoGkc5@;R(s`A)RyR`X+Ul5wq_6j_z-_i
zAr{9Ot*HyP#W;@;{ILqrjZye>RmEr2_aUR8$MK(~Kq0ZflF1eI)(m{c6GIGlkAW#Z
zj7y0X+ccqhn|pRPP<}BDeELtrz3N^nhKdXHks}AKDcbY_Z4STmVO3L9mzQ{1>4&1^
zRoD950D6<s)qtiW>F6=)rrJ0zUCo$$#Wxa*du-V)<PNTH#i^tGHBd<zRyAC!EL=OV
zGPt!JIRN#k*P7k)(@DhduXI>CwG&FA1!k`q#CA3iKr2jY`#sD35An@iUzYL-SjxQs
z3Nr^a%%V5cB3&`|9IJW{<x)NSsqfv)^CG8V@j3|2kqb03TOhe<#;jUKY{`ZrI5E1Z
zm}nm@Wq^~m_j>UcxnlL13scrMk3OU(tJra9ty%%f;1aGJV}BYefIs$#6ndfVjW5HV
zl=*We5%33PM?2$2Xnx1Ln}WPo{VEO!9Ru~>l3-W&jyahZ@rfOGc4|UDPYY9?yVtSp
z<}^%vtO>n`&3k`~5a2L>T{7TaQWOn;tgMhQd9OM&5l$#v<4r_P8DtZ5tO2L_sGd5_
z1;_S$RYl`@JYo*FJ!0}LqDy@>F!Pff5nVf}ahz?~|MrkV?1r<|o&X&X-*rhe%=AJJ
zYrI+$yrQA>6zXJ4n#l2`T5K12=cOeHP$JLCD3nDlHQedP98cvyFzWe<E_#AC<CvAb
z8d0c?Y;W1)B~CrIDG)tLKqwbkmVo}MB;PnvoY`VoQFo*r_P>Xrw>dPO83DeJ1MH5{
z4w*X90ER^jx1P3^Qx!NvjIV~H4$}#zl&yu{Sd3?$IIB(~x~R+7E6YXX_di1O_n%L_
zWkvT>f2(sIlg@dPRnD+(EYi`hrQHR8dD_nSXb(oR#|ok>2#H0|VPospLBB57x8KA-
z=)=~gm8GWOhbMf>GP|-!Dt1$N((+G<<M<u~(i#V^O@5U*T~&32VmUgpQLZ}~4t-`F
zwpW|~>|F!<Eb~$NcNU0w?NFOJM<D1K!=z-UuPJ^Uyf%L=s?e2xt*BNwp48q}#k!Q(
z*Q-_Qv!s|JvM5j&7%^c!MIH{)M{7xOnc-?zuVeC(nqP5+P>$R-`O^h^7nC+g>2NB8
zXG}Ycq$rOS2uvq^-I|q|WF<6+iHbCQ+Hx6rj5JMvsPe372hKyN;O<T-F<%}p^^Nfl
zkRhS%FjCRZ906V$6~MTuSJOTRC3$E&9Lp1~YRU+pxv216GG`3ZCYYky(u=q9*8tCw
znVD+fzxBQ!{?>tp8EdSn8(@JBP&{wHf<1oA9J`qvm#QkO@eMD54}E6_e3n;~6a-45
zWRt=_=VTZ~{Ltx4L)M<aQ=#Tg2^9UCR34nm7WS<C#9Z1EF*>H%MmYyhDm(Sfn9CF)
zvvaAp0Oa?@>eq>cr0C?g+oai!{<X9`$jq&nG9Ys+L6F7b1#v3rS#PBF5zAG2@ESd)
zA;D+B*Na}I<GQ7HmLhLt+gJ-w;W_t1&E#e``A<x=+i{+OL#fAwcbKcc)b7o=WsFff
zr{;s0%c7dS>fl*;E6%qOOg!@!5YRMJCit9<Wk%>G>W6d>JCvdvGx*)E<Odq&^=~#J
z<ByeGDpc&Mp>PZIv=O1}sOxB6ngQg?@6frv)1{rCJGdfX%^kg$yA(dR^-@10$34<y
zvI}-q?4SP5q|E&8a=7pLt}9$;#J$UWW)Na>XGK()LzM4QAb5&QmUbpY`n&qxZoeYL
zL}Y<E$YDOeU@V!sZS?772+{G{zR$6n5b%D#S0%vl_Vq79F&|)#teeu0Igs6~lE>>3
z?H0}J6W}hH*7wW##_7?=Dt2->P<VbpKK|fE$W3_Xob1;L-x;IkWHhR&b+c^@qEXbH
z!IoM<JOYAYFeatxo<A}~nmc#o>VC^h4U*9?+b`L?7)h>36HmO>6Oo>*LOYl%3}Nr!
zJNCLExm*GCq8UO;<SbUumD+2R7ap;t5YEeNugI>r=aqIV@AjBhW5v+@Q!jVv=h|fo
z7-CVk%v;Wg>T^ntKrm?5HT4E=wy+DC=lq56X0rWR^z{j(0@!4L-=7#X4V}K%n`0~t
z&cT<>c7<qzR10)^!4;vws1$P?)V<P|%%W?jJ!Pp?Bw}{7p3^Vl&EzWGhPTs&4V+Dw
z?KH@eNGP2b^}!2150zt?tbx<7jQ?oy!{&ul;*OY|uJW(5B#b!-iRu;F8ZSKCT(<C2
zj|`(e)j;6oY6-(yeN3BP4ro)3N%D*jM=YviM4Ei*^qO9}$Jua6JcOL(!a0?Qy+Zcu
zC-E3a)X3RN6~gXX5^wKSeIMG~>f#S8ORO5ATC;7FKeAwA<PrH(=HUs{XHYQaFtU8y
z^Tlbmq}etUJ8$u_{CVJ!Dy_h{%H*EKwlln~s)NcpNreD;b*2vLmbg;#Mb5*H%^$5o
zL%1y92)rH4?mCDAupz+|)Ay~5S%MBY>-O^ZWg_qrmyiUvXL02=6!3a+8S8xEIuZM~
zm^N&!10cjfR>{%IEo>f6KZdrQ72YJ1G^$AeoZfCc)5ggU8@Dn`uBxYo3o*qKueQbt
zCSEXNqvjB1)+;8{RVvY3*l%_Yp<Fszb77;=J=S7(JcOcZ0ZfK=O){j~m41^}t7+-X
z<h<WvC^~eXC+U|?*sQn$1~;`MCu7`P<#Zq+>h1LFL%ulQ>QiFfui;~wRZYo((T4{T
zRhRv2x|@*ZKB%9efi)9)Lwc_(cBZu=35KR%_Sx*f#Y!~$0AJ?K%f|x5oe=1V5ZP=G
z1;do-t;avAM%@7-<D!s7DUx}Ob}J$g*T{w&jFW(M4`23N<usAL>V#9T)!v)O*?>4W
zIyAk#uD_1Z4`#QvLuqOLDG>|BQ${hAc842*24w_)S`5xX_^O?6g~#HWULpT-oIB#m
z$@z{_%46m6fCa=F$oA?v$a;MU_5Ss_`|qiRugc=N*>xp6W7HdosHF^#GeyIMQzh}M
z#>maFcl?UvvN$^!7F$`m5dLemvv+eyEk_`2d0;WSkE`CN4v8kZcjDrfnb~Kl#M^R)
zwX5Z%KBuucC7I{XW(RXZS%`UP9C8M7$-XUjlT2v-*|g%-vEpQKOqt8Zuhx(KSSA;@
zOfK7Fgp)S0$k1J|P9b2vWBm#9@#C{$Sj8NH4vmqv9D%5t__X3sXI$A#^;bEFC#E7V
zX+GD<bdO!Y<<&LtdgN$~yEM2~Xzmpf2uNQ6+RZ;IMHdGPLZa%e`u=u-D<fW4)gcq>
z5|lDLvVC9`w`$E)fW(kR7a7A8NhWNM#|O3#3SEh9JOTZx1S<{?y)9xeK$4@p+Bpdr
ztd+V7(_{hSf;+aIL!csSz63TnyggfWO#$4AeE+8Pr2-&&BZo|{I*N7EkW^Qg43a*+
z$Pp-WV$}iABo0zTlb7XeFKx}6$(j%MsYp#yG)&2F(-(Py@OF5ac};g;wN(><hET#1
zF|(grc<~^0&IYIN1*)RB`K<S^bpArb#W*x=yA63e8cIK(&!_6_z7qeCo9+I?(*A}1
z>7Y{OLZm%4RqsbO&I`Z9klbNBNT&ezFyTFX7Hi>%Wq(*1YRC>P7yWj#^v`Fi!_rgF
zRky=CzFZXd>k+gGIfCp{yMC3XNWDY$klDd|eal+4l@I!MosS;X^oIT1cC{d9bY+wo
zVAl{>KvERUQ7q236yd6?TWxeqBzBPp2XTk;OlY019g_J~ti)-Pvg6SygRps9=<|c`
zk4w2OFhubOHwbQ*jDHv`$H%`m6J`$mD=TK{Zsk%bOEUY&^83oWrqb&=<v%`u_G;Mo
z<H=c#-Ls$YF|V^aI)}rQ+T@201m>OLmJEneq3s>I2!%(s2ghEdB(_fVzzd=L0%7yn
zS48nN5IYk|Y$59%Mj6Ryt6$WviSfLuB@an9)_k^bm+}4$+*NVRBg?nn_S()yl8diu
z#Zp8$0vm4k!S;XJ=YiPrZQ81p?zdx9(E#Ub3z1T75SWQ^boa0e24#W?emE*Ld3J6V
z({mQ95eVNoeSY}GM<Ejv9M&Z2M?J)J$N~_>`MPrQ$Cn~zsGl3+<?>x0t|eY_UdT@{
zM-lL8Wk%3&xb%2ls6mQz)3x*j(8;vJ)=w3#Sy}6&D|c>>k!>nH8kI34T(cb86p&$y
z`K|&JH-r#)<50fkXJFLqu)#9DMH3y|8oY)P1<|*Mtp{eK{bn_kXYYugp^e^fOAkVG
z6Pke=m@`{j5KN0MutMF(_X-W0F9=k{fA=iZw-j_slxWL7fw_?wIEf?L6SuG|p8y0c
zoz(-OPT9!uJkMMabt|aHP>a>t@?vi-AvDN*PbZ}W-@^D<cp9Lw&?@QgRTe~;7o#2b
zkAAvFT**c%if8qpuZ{1XbeR5Cks-QW2QFx3K5<)eAw(q;eG4q6Oou7<Q@7HF*Pa6<
zY+ltk_5n4NKD<1QWPMZ}8LA*&O7;(%6zE4r?6yPTw{2%2xHip4p3Wgu^@y!g-B-3V
zSFcxlF0EPf<BDO{PG>4ReJW*BV3~RJR{HQpa~MQyBYQN1WH59k6&;?{x8V5PtJ6Hp
z_Oe&63E|JStqma;H;11lBwi?Or+Zrm;^sNo881s_)LfE$)~a{yH-6abM}7S?s|xr^
zgnS)G>0eQE!81NaiYR0{=%BQsVtpe$I8OMoJ&Vr8a^iW$#7Eo+{?>d?v<eR0%h{!`
z_=pXf4ZEhG*V0LbVVkwL#s<M|abWZZ+=j#ZwWJtGPuMR{hf_O8W3NQ7<Z{;z3!xcv
zwHt`=AREEUO!l*uq2zJdOnY}3!fc4K$p(g4YMS8gXznGR&^k^0Qh$Cs;25rQi4l#E
zAYGBG-Sd=~Vv#Z8BVF79<B<n^xx92Pcyrc`Id0|iGytuHOFR7(cE^aF7+iCDj3Mt<
z?lEnrtO<XX;;+Jl!%B${-6<mh5<>T?kUq<ASm1!AaJySviEd3s%Uf}2NQ&vgaz=Gp
zYM_TCr;UYOY7BSGjXYtYyFgSoTc~W#LTiZ>lXz}WnZ$*d%r3h0N?d0}#*_+~f$;n}
zw6%p&U(3f^*|;vL;NuuICve0<5-1$C9m&0z{3V6tL{5}3qE5hmo#62vK@ckQyUApL
zQ1b5Zc{C=&sPsHvl)&iCIb+KV&aDcY2iWL)l8GCNns!CG7WwMY39SHY_FAR2j)flo
zwA93O&`)0p)E|_uNh-Lg{*g+1Nm{u@Tsm1p_JhI*25>m{YhhTs`lyf60$A}h*Uf)v
z&3c3u)N^_rDq+m?vf<O_&q9OGuCmyE<u8XX@%}To@ByU_N?9v<TJ}d4rDxY5(f0iH
zX+&$-vFWyv<YIYU<vqYL3T=<HviR75*Va<8W2IpF>EIf}ZIEK&V)D^(PvL7T8i5t4
zKnrA5{;(*@N>p0Dv%FVqX`m8@mbB+@<r||ab@t`x)XL5_XY%??b)QD?%b;_%{N+X9
zVI9HizY+~px}2r+XXK0(N9$3novbj7>Ybi`^$u(kMKz3q$hXgIm+H*Jt(Ej^^a?W3
z<4hBIK25M#V|z(01UPWDG~(c%JG4wX&6Q)$zQ4b14nLQR)b}IJR#nnxO|Qy{ty*~P
z8dDew-IdKbfs+RnL5>_y^doM}#gicvlM6o$SbgPn<-@pnRXKL{jR3Rxbga@16}yh8
zaAV70>1QS-Z&woE_JqUKc|;3$Q7_3og;40t+pc0a!}HSGy;g`RQn9Cgq1gi4h|q<2
zZ=2)#W%^E!Ll3v!d<b~mo!%)JV$y*T;_Y-xk^CwMVuBx%gZtj|o5QcOoo9ZQ)Y!{B
zr%Y%ty2v=R>QTN~9K$R2CI<lk$htM1IfXMlD{hC<<?C!RM*5m=Y?{PBzq)x|(--f@
z@yho?uo${>8i8c`^>*>ujr(k$OW!rThca`$u$=9qF8+qr1@W!fQ!is~I!7evGjuG}
z)!r;~N_d2e{$Y>kUN1Lr<$c=XhWJz#J{F-g!Y%=4WD0~VA=^u0NiWg=D-<=#sM>cV
zFq2lM{}{%blsZP?<A$DLq?du{3`h-n3bR@|X!+EN6$;3Fa*(5PK8?@2>z?0S>H!-e
z&lqkLb#c;hbrwsjWF0JlVC{Q4<Lt!){N<fRzgn#?2gE-#&-feb%s=V`VuUzQtUwpm
zt!#O&`E<LsQz-<M>Q;anH>9CoJJVJ<F|#M>#Bf=_g2cbOH7m$iC~sI~C`Qh7!BDC8
z@Qt~Mj%#Oh9-Isfug33`zyLxp7e7q?NhzbcYG49Fh2}MleB&EmO)iu4B*OT-EtK}z
zpB5v9ZrJd(uhvOb6aNho0;)o3QSU*Bb&V+_3&2w);g(FtgG~KnYIawz>HD?Z9rpua
z<Y!gfBD4$hL9!a0uyYsAn0(d}UjV8$w=_9LKdGeO*2oDuZ`WI{WYAO?0Nrt?<&LP3
z{!TmQy|;@!+R+VTR~d6{yke5`e|16+P%J>EHL_Bx>HIz_fy>gT@#(1OFX=&}d=Sev
z|0WFWmT!p>r(JwHX;ulvQ?7=JYa^9e?9Y)f3H<Z!M}@RsWt#w56Mu@y%<~aP$s+R&
z=5vt*V7T*%SW~92l@3p#Zmp9S$Sm{i$c8g5(U+a06@`?On@Gg%<R))r5&W?_-V@a&
z4SM#Bxu_X8wWx*}(`t8AK*lDUqL=cMiH3tl$LV@vOZBfSL+e`5NA5jDh|2kLBgy64
zhVNe=60(T)qEBljT}_T;3DYgq92z2<?ZY1jl*F_AU^^Y?xpvEv1`q8@<;Oa6iV;DU
z{yah6W4zNjtXM-%XxX;0-KeVAV}L$w8D3<JuqMnZU&ih(iKX5$NH^+8{DcXFigiUG
z{D=Eff*;#5mA&fa6Fjh!<{inmFRhJCt7CV2kMBTV<JEukhW**X8B>Pzz@=^z=+8ay
zUegdr?|DF(E(HmvIeICuBGUgxS;X75e(A*q42K;Dp}`$O&e~{aqz`lLK~Q?DB37mB
z;%m(PK-RJ!ztP=0(;i$BTv8V#eb09y8sT84i;dLPJH(#OAg*HKmM+-S7RiMKv9d5P
z7#3wRxmb^p<T9gDBq55)S7BuK@qWU?Z%_MNv!{_ZA6^$E{}Gx}q<T=5vf2?4fCv~f
zCoh5L_@t5zsJbVoLUXhRMHB5?vIIj&p-nu4u@56R*f4%fus`cBlEmnVxL8v3B(J=?
zrcFyEJ@kDGTI_ie!?md}!nrfS96fYsvWF0sRyfejhOhtym`ga`q}2nyO~S(yDNB8N
zCbLUx5awvSmD$4YOnzm_GhT_9vR5@#ijAF~F097aVJ4X|jA;Zi8Hb}Ms|Ae-j>u#d
z#dDw9bpdHN#_wz>ocvH7Cf(n>1~7nq$#xQxae7(o3j`1nD|%bXmR}+&Y;Ab?A}i#i
zaW_qIOq5FisR+8;1FyR}G%rg<tu`tTt7>$A3SD)N$guk*YeJ)nI;?E`GqWj3=<vEf
z<Y4nHaQyrpvENIAZ>VZnaecn$aB)?P!Q+|rwuWiAeQy4_y&(r@hE|Gk3L#F`P5Re&
z^z3WIhNvpgy>96OFD)qU)ow8i*7vLvU;^{lwy*Zcy5e446LjU<WHL^yBd_%b5nZ{=
zPwU>;hX8MspEbJqxuLGNg1@xvo!go4g-s~10N<yj#Dwg+i!0_a*l3*GS`{vRQSm%G
z3m&{Oi!L|PmmiUuh1I<20;C9TRJ8)VxeeF^yk4dKYKwp$=Ljq+VetL%<zw4F0w-nR
z?|<9{jHQh|89sySx8W7y=6Y1nE4uz8!_e++X?qF(%hlq296&?CX}4woa&t>BB=h5u
z)!Cau<-Zvf?7iWtHH3S6lh%J>?Dyc_(Q!uoCfJYRLq3#-f&y;+L+K#KLSsOh*0Do%
z#8{{@DfVBqoc=SO5*1TM^E-*(9>o|v#CDYBX8q-(ZG_K}_*c)#FKWn}&R)Mz>x<<7
z&|P@p-OvBOEaur9+;H9G$A^DI*7n|OlK-t02E`B7L_fbYZ~d<wpTdLC9kk_#p!JJ?
zA4wOvo%3HbCI8Q_hj2#!WxwA4EYt{S2@k`6$wEDN#tDWN9xs2G_owCSZyUxxcp-(j
zmBb!d-uSYA|MsN*3oqt`X9|K9P5i7+x_x<w|E*t^Vh(TRpAi`Sw%z^x3uEwP$8*G;
z`faoO*I%gQ!y#4qbJlN%_rGu@*3XNKgFwh`KK9G{*Z=yv*zm=VYW(MD;y<{nU*58g
zDx3h@&wsmq`u{s`r*b<)6o_XiuNNx+#;uLH3Lb^s%@XT|xIccwfBXjhAYX%NsuTDh
zEHJ6f$Wl+S4E;Ag!ungD1E#$*Cx-p^EAOBFj{SC6U&bm$+x~+qI%W#hqWRJJKZW@I
z`2Tpe8%n>7A04^>gDcvc1CIUtv*W)rOaH46ONbcMXSy33zW=AM-`}G=|Jyizf7SnQ
eHjcqhTPX)e_N0uD^=*JZTBr0*K2)>6`u_m6T}m<l

diff --git a/docs/tracing/opentracing.md b/docs/tracing/opentracing.md
index a2110bc0e..8528c2ba3 100644
--- a/docs/tracing/opentracing.md
+++ b/docs/tracing/opentracing.md
@@ -1,5 +1,11 @@
-Opentracing
-===========
+---
+title: OpenTracing
+has_children: true
+parent: Development
+permalink: /development/opentracing
+---
+
+# OpenTracing
 
 Dendrite extensively uses the [opentracing.io](http://opentracing.io) framework
 to trace work across the different logical components.
@@ -23,7 +29,6 @@ This is useful to see where the time is being spent processing a request on a
 component. However, opentracing allows tracking of spans across components. This
 makes it possible to see exactly what work goes into processing a request:
 
-
 ```
 Component 1       |<─────────────────── HTTP ────────────────────>|
                      |<──────────────── RPC ─────────────────>|
@@ -39,7 +44,6 @@ deserialized span as the parent).
 
 A collection of spans that are related is called a trace.
 
-
 Spans are passed through the code via contexts, rather than manually. It is
 therefore important that all spans that are created are immediately added to the
 current context. Thankfully the opentracing library gives helper functions for
@@ -53,11 +57,11 @@ defer span.Finish()
 This will create a new span, adding any span already in `ctx` as a parent to the
 new span.
 
-
 Adding Information
 ------------------
 
 Opentracing allows adding information to a trace via three mechanisms:
+
 - "tags" ─ A span can be tagged with a key/value pair. This is typically
   information that relates to the span, e.g. for spans created for incoming HTTP
   requests could include the request path and response codes as tags, spans for
@@ -69,12 +73,10 @@ Opentracing allows adding information to a trace via three mechanisms:
   inspecting the traces, but can be used to add context to logs or tags in child
   spans.
 
-
 See
 [specification.md](https://github.com/opentracing/specification/blob/master/specification.md)
 for some of the common tags and log fields used.
 
-
 Span Relationships
 ------------------
 
@@ -86,7 +88,6 @@ A second relation type is `followsFrom`, where the parent has no dependence on
 the child span. This usually indicates some sort of fire and forget behaviour,
 e.g. adding a message to a pipeline or inserting into a kafka topic.
 
-
 Jaeger
 ------
 
@@ -99,6 +100,7 @@ giving a UI for viewing and interacting with traces.
 To enable jaeger a `Tracer` object must be instansiated from the config (as well
 as having a jaeger server running somewhere, usually locally). A `Tracer` does
 several things:
+
 - Decides which traces to save and send to the server. There are multiple
   schemes for doing this, with a simple example being to save a certain fraction
   of traces.
diff --git a/docs/tracing/setup.md b/docs/tracing/setup.md
index 2cab4d1ef..06f89bf85 100644
--- a/docs/tracing/setup.md
+++ b/docs/tracing/setup.md
@@ -1,14 +1,20 @@
-## OpenTracing Setup
+---
+title: Setup
+parent: OpenTracing
+grand_parent: Development
+permalink: /development/opentracing/setup
+---
 
-![Trace when sending an event into a room](/docs/tracing/jaeger.png)
+# OpenTracing Setup
 
 Dendrite uses [Jaeger](https://www.jaegertracing.io/) for tracing between microservices.
 Tracing shows the nesting of logical spans which provides visibility on how the microservices interact.
 This document explains how to set up Jaeger locally on a single machine.
 
-### Set up the Jaeger backend
+## Set up the Jaeger backend
 
 The [easiest way](https://www.jaegertracing.io/docs/1.18/getting-started/) is to use the all-in-one Docker image:
+
 ```
 $ docker run -d --name jaeger \
   -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \
@@ -23,9 +29,10 @@ $ docker run -d --name jaeger \
   jaegertracing/all-in-one:1.18
 ```
 
-### Configuring Dendrite to talk to Jaeger
+## Configuring Dendrite to talk to Jaeger
 
 Modify your config to look like: (this will send every single span to Jaeger which will be slow on large instances, but for local testing it's fine)
+
 ```
 tracing:
   enabled: true
@@ -40,10 +47,11 @@ tracing:
 ```
 
 then run the monolith server with `--api true` to use polylith components which do tracing spans:
+
 ```
-$ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml --api true
+./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml --api true
 ```
 
-### Checking traces
+## Checking traces
 
-Visit http://localhost:16686 to see traces under `DendriteMonolith`.
+Visit <http://localhost:16686> to see traces under `DendriteMonolith`.

From 24f7be968d6459e887de7c18e17a2c7bb8294796 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 11 May 2022 15:46:45 +0100
Subject: [PATCH 104/391] Fix link

---
 docs/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.md b/docs/index.md
index d77af87a8..64836152c 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -12,7 +12,7 @@ many Matrix features are already supported.
 This site aims to include relevant documentation to help you to get started with and
 run Dendrite. Check out the following sections:
 
-* **[Installation](INSTALL.md)** for building and deploying your own Dendrite homeserver
+* **[Installation](installation.md)** for building and deploying your own Dendrite homeserver
 * **[Administration](administration.md)** for managing an existing Dendrite deployment
 * **[Development](development.md)** for developing against Dendrite
 

From 58af7f61b6c3719e15cb4088a343cd08b404f5be Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 11 May 2022 18:15:18 +0200
Subject: [PATCH 105/391] Fix OTK upload spam (#2448)

* Fix OTK spam

* Update comment

* Optimize selectKeysCountSQL to only return max 100 keys

* Return CurrentPosition if the request timed out

* Revert "Return CurrentPosition if the request timed out"

This reverts commit 7dbdda964189f5542048c06ce5ffc6d4da1814e6.

Co-authored-by: kegsay <kegan@matrix.org>
---
 keyserver/storage/postgres/one_time_keys_table.go | 4 +++-
 keyserver/storage/sqlite3/one_time_keys_table.go  | 4 +++-
 syncapi/sync/requestpool.go                       | 8 ++++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/keyserver/storage/postgres/one_time_keys_table.go b/keyserver/storage/postgres/one_time_keys_table.go
index d8c76b49b..2117efcae 100644
--- a/keyserver/storage/postgres/one_time_keys_table.go
+++ b/keyserver/storage/postgres/one_time_keys_table.go
@@ -53,7 +53,9 @@ const selectKeysSQL = "" +
 	"SELECT concat(algorithm, ':', key_id) as algorithmwithid, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 AND concat(algorithm, ':', key_id) = ANY($3);"
 
 const selectKeysCountSQL = "" +
-	"SELECT algorithm, COUNT(key_id) FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 GROUP BY algorithm"
+	"SELECT algorithm, COUNT(key_id) FROM " +
+	" (SELECT algorithm, key_id FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 LIMIT 100)" +
+	" x GROUP BY algorithm"
 
 const deleteOneTimeKeySQL = "" +
 	"DELETE FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4"
diff --git a/keyserver/storage/sqlite3/one_time_keys_table.go b/keyserver/storage/sqlite3/one_time_keys_table.go
index d2c0b7b20..7a923d0e5 100644
--- a/keyserver/storage/sqlite3/one_time_keys_table.go
+++ b/keyserver/storage/sqlite3/one_time_keys_table.go
@@ -52,7 +52,9 @@ const selectKeysSQL = "" +
 	"SELECT key_id, algorithm, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2"
 
 const selectKeysCountSQL = "" +
-	"SELECT algorithm, COUNT(key_id) FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 GROUP BY algorithm"
+	"SELECT algorithm, COUNT(key_id) FROM " +
+	" (SELECT algorithm, key_id FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 LIMIT 100)" +
+	" x GROUP BY algorithm"
 
 const deleteOneTimeKeySQL = "" +
 	"DELETE FROM keyserver_one_time_keys WHERE user_id = $1 AND device_id = $2 AND algorithm = $3 AND key_id = $4"
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 8ab130911..30c490df0 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -248,7 +248,15 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 		defer userStreamListener.Close()
 
 		giveup := func() util.JSONResponse {
+			syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached")
 			syncReq.Response.NextBatch = syncReq.Since
+			// We should always try to include OTKs in sync responses, otherwise clients might upload keys
+			// even if that's not required. See also:
+			// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
+			err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
+			if err != nil {
+				syncReq.Log.WithError(err).Error("failed to get OTK counts")
+			}
 			return util.JSONResponse{
 				Code: http.StatusOK,
 				JSON: syncReq.Response,

From 3437adf5970a11342c22089ec1a18f46d0708740 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Thu, 12 May 2022 10:11:46 +0100
Subject: [PATCH 106/391] Wait 100ms for events to be processed by syncapi

---
 syncapi/syncapi_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 7809cdaba..d3d898394 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -199,7 +199,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
 	for i, msg := range msgs {
 		test.MustPublishMsgs(t, jsctx, msg)
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(100 * time.Millisecond)
 		w := httptest.NewRecorder()
 		base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
 			"access_token": alice.AccessToken,

From 0d1505a4c1ccca7c5cf4a64faf5d1044d8aa96f9 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 12 May 2022 11:35:35 +0200
Subject: [PATCH 107/391] Fix `create-account` with global database settings
 (#2455)

* Fix create-account with global database settings

* Avoid warning about open registration
---
 cmd/create-account/main.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 7a5660522..7f6d5105e 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/sirupsen/logrus"
@@ -99,8 +100,14 @@ func main() {
 		}
 	}
 
+	// avoid warning about open registration
+	cfg.ClientAPI.RegistrationDisabled = true
+
+	b := base.NewBaseDendrite(cfg, "")
+	defer b.Close() // nolint: errcheck
+
 	accountDB, err := storage.NewUserAPIDatabase(
-		nil,
+		b,
 		&cfg.UserAPI.AccountDatabase,
 		cfg.Global.ServerName,
 		cfg.UserAPI.BCryptCost,

From fc670f03a2ac13adc7e022bcb21b82ad874d6706 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 12 May 2022 12:05:55 +0100
Subject: [PATCH 108/391] Separate sample configs for monolith and polylith
 (#2456)

* Update sample configs

* Update references

* Remove sections that are dead in the monolith sample
---
 README.md                                     |   2 +-
 build/docker/README.md                        |   7 +-
 build/docker/config/dendrite.yaml             | 348 ------------------
 dendrite-sample.monolith.yaml                 | 279 ++++++++++++++
 ...nfig.yaml => dendrite-sample.polylith.yaml | 215 +++++------
 docs/FAQ.md                                   |   2 +-
 docs/installation/7_configuration.md          |  10 +-
 7 files changed, 377 insertions(+), 486 deletions(-)
 delete mode 100644 build/docker/config/dendrite.yaml
 create mode 100644 dendrite-sample.monolith.yaml
 rename dendrite-config.yaml => dendrite-sample.polylith.yaml (54%)

diff --git a/README.md b/README.md
index 9c38dee90..ed09e971c 100644
--- a/README.md
+++ b/README.md
@@ -71,7 +71,7 @@ $ ./bin/generate-keys --tls-cert server.crt --tls-key server.key
 
 # Copy and modify the config file - you'll need to set a server name and paths to the keys
 # at the very least, along with setting up the database connection strings.
-$ cp dendrite-config.yaml dendrite.yaml
+$ cp dendrite-sample.monolith.yaml dendrite.yaml
 
 # Build and run the server:
 $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
diff --git a/build/docker/README.md b/build/docker/README.md
index 7425d96cb..261519fde 100644
--- a/build/docker/README.md
+++ b/build/docker/README.md
@@ -27,8 +27,7 @@ There are three sample `docker-compose` files:
 The `docker-compose` files refer to the `/etc/dendrite` volume as where the
 runtime config should come from. The mounted folder must contain:
 
-- `dendrite.yaml` configuration file (from the [Docker config folder](https://github.com/matrix-org/dendrite/tree/master/build/docker/config)
-   sample in the `build/docker/config` folder of this repository.)
+- `dendrite.yaml` configuration file (based on one of the sample config files)
 - `matrix_key.pem` server key, as generated using `cmd/generate-keys`
 - `server.crt` certificate file
 - `server.key` private key file for the above certificate
@@ -49,7 +48,7 @@ The key files will now exist in your current working directory, and can be mount
 
 ## Starting Dendrite as a monolith deployment
 
-Create your config based on the [`dendrite.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository.
+Create your config based on the [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) sample configuration file.
 
 Then start the deployment:
 
@@ -59,7 +58,7 @@ docker-compose -f docker-compose.monolith.yml up
 
 ## Starting Dendrite as a polylith deployment
 
-Create your config based on the [`dendrite-config.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository.
+Create your config based on the [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) sample configuration file.
 
 Then start the deployment:
 
diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml
deleted file mode 100644
index 94dcf4558..000000000
--- a/build/docker/config/dendrite.yaml
+++ /dev/null
@@ -1,348 +0,0 @@
-# This is the Dendrite configuration file.
-#
-# The configuration is split up into sections - each Dendrite component has a
-# configuration section, in addition to the "global" section which applies to
-# all components.
-#
-# At a minimum, to get started, you will need to update the settings in the
-# "global" section for your deployment, and you will need to check that the
-# database "connection_string" line in each component section is correct.
-#
-# Each component with a "database" section can accept the following formats
-# for "connection_string":
-#   SQLite:     file:filename.db
-#               file:///path/to/filename.db
-#   PostgreSQL: postgresql://user:pass@hostname/database?params=...
-#
-# SQLite is embedded into Dendrite and therefore no further prerequisites are
-# needed for the database when using SQLite mode. However, performance with
-# PostgreSQL is significantly better and recommended for multi-user deployments.
-# SQLite is typically around 20-30% slower than PostgreSQL when tested with a
-# small number of users and likely will perform worse still with a higher volume
-# of users.
-#
-# The "max_open_conns" and "max_idle_conns" settings configure the maximum
-# number of open/idle database connections. The value 0 will use the database
-# engine default, and a negative value will use unlimited connections. The
-# "conn_max_lifetime" option controls the maximum length of time a database
-# connection can be idle in seconds - a negative value is unlimited.
-
-# The version of the configuration file.
-version: 2
-
-# Global Matrix configuration. This configuration applies to all components.
-global:
-  # The domain name of this homeserver.
-  server_name: example.com
-
-  # The path to the signing private key file, used to sign requests and events.
-  private_key: matrix_key.pem
-
-  # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
-  # to old signing private keys that were formerly in use on this domain. These
-  # keys will not be used for federation request or event signing, but will be
-  # provided to any other homeserver that asks when trying to verify old events.
-  # old_private_keys:
-  # - private_key: old_matrix_key.pem
-  #   expired_at: 1601024554498
-
-  # How long a remote server can cache our server signing key before requesting it
-  # again. Increasing this number will reduce the number of requests made by other
-  # servers for our key but increases the period that a compromised key will be
-  # considered valid by other homeservers.
-  key_validity_period: 168h0m0s
-
-  # The server name to delegate server-server communications to, with optional port
-  # e.g. localhost:443
-  well_known_server_name: ""
-
-  # Lists of domains that the server will trust as identity servers to verify third
-  # party identifiers such as phone numbers and email addresses.
-  trusted_third_party_id_servers:
-    - matrix.org
-    - vector.im
-
-  # Disables federation. Dendrite will not be able to make any outbound HTTP requests
-  # to other servers and the federation API will not be exposed.
-  disable_federation: false
-
-  # Configures the handling of presence events.
-  presence:
-    # Whether inbound presence events are allowed, e.g. receiving presence events from other servers
-    enable_inbound: false
-    # Whether outbound presence events are allowed, e.g. sending presence events to other servers
-    enable_outbound: false
-
-  # Configuration for NATS JetStream
-  jetstream:
-    # A list of NATS Server addresses to connect to. If none are specified, an
-    # internal NATS server will be started automatically when running Dendrite
-    # in monolith mode. It is required to specify the address of at least one
-    # NATS Server node if running in polylith mode.
-    addresses:
-      - jetstream:4222
-
-    # Keep all NATS streams in memory, rather than persisting it to the storage
-    # path below. This option is present primarily for integration testing and
-    # should not be used on a real world Dendrite deployment.
-    in_memory: false
-
-    # Persistent directory to store JetStream streams in. This directory
-    # should be preserved across Dendrite restarts.
-    storage_path: ./
-
-    # The prefix to use for stream names for this homeserver - really only
-    # useful if running more than one Dendrite on the same NATS deployment.
-    topic_prefix: Dendrite
-
-  # Configuration for Prometheus metric collection.
-  metrics:
-    # Whether or not Prometheus metrics are enabled.
-    enabled: false
-
-    # HTTP basic authentication to protect access to monitoring.
-    basic_auth:
-      username: metrics
-      password: metrics
-
-  # DNS cache options. The DNS cache may reduce the load on DNS servers
-  # if there is no local caching resolver available for use.
-  dns_cache:
-    # Whether or not the DNS cache is enabled.
-    enabled: false
-
-    # Maximum number of entries to hold in the DNS cache, and
-    # for how long those items should be considered valid in seconds.
-    cache_size: 256
-    cache_lifetime: 300
-
-# Configuration for the Appservice API.
-app_service_api:
-  internal_api:
-    listen: http://0.0.0.0:7777
-    connect: http://appservice_api:7777
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # Appservice configuration files to load into this homeserver.
-  config_files: []
-
-# Configuration for the Client API.
-client_api:
-  internal_api:
-    listen: http://0.0.0.0:7771
-    connect: http://client_api:7771
-  external_api:
-    listen: http://0.0.0.0:8071
-
-  # Prevents new users from being able to register on this homeserver, except when
-  # using the registration shared secret below.
-  registration_disabled: true
-
-  # If set, allows registration by anyone who knows the shared secret, regardless of
-  # whether registration is otherwise disabled.
-  registration_shared_secret: ""
-
-  # Whether to require reCAPTCHA for registration.
-  enable_registration_captcha: false
-
-  # Settings for ReCAPTCHA.
-  recaptcha_public_key: ""
-  recaptcha_private_key: ""
-  recaptcha_bypass_secret: ""
-  recaptcha_siteverify_api: ""
-
-  # TURN server information that this homeserver should send to clients.
-  turn:
-    turn_user_lifetime: ""
-    turn_uris: []
-    turn_shared_secret: ""
-    turn_username: ""
-    turn_password: ""
-
-  # Settings for rate-limited endpoints. Rate limiting will kick in after the
-  # threshold number of "slots" have been taken by requests from a specific
-  # host. Each "slot" will be released after the cooloff time in milliseconds.
-  rate_limiting:
-    enabled: true
-    threshold: 5
-    cooloff_ms: 500
-
-# Configuration for the Federation API.
-federation_api:
-  internal_api:
-    listen: http://0.0.0.0:7772
-    connect: http://federation_api:7772
-  external_api:
-    listen: http://0.0.0.0:8072
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_federationapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # How many times we will try to resend a failed transaction to a specific server. The
-  # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc.
-  send_max_retries: 16
-
-  # Disable the validation of TLS certificates of remote federated homeservers. Do not
-  # enable this option in production as it presents a security risk!
-  disable_tls_validation: false
-
-  # Use the following proxy server for outbound federation traffic.
-  proxy_outbound:
-    enabled: false
-    protocol: http
-    host: localhost
-    port: 8080
-
-  # Perspective keyservers to use as a backup when direct key fetches fail. This may
-  # be required to satisfy key requests for servers that are no longer online when
-  # joining some rooms.
-  key_perspectives:
-    - server_name: matrix.org
-      keys:
-        - key_id: ed25519:auto
-          public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
-        - key_id: ed25519:a_RXGa
-          public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
-
-  # This option will control whether Dendrite will prefer to look up keys directly
-  # or whether it should try perspective servers first, using direct fetches as a
-  # last resort.
-  prefer_direct_fetch: false
-
-# Configuration for the Key Server (for end-to-end encryption).
-key_server:
-  internal_api:
-    listen: http://0.0.0.0:7779
-    connect: http://key_server:7779
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_keyserver?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Media API.
-media_api:
-  internal_api:
-    listen: http://0.0.0.0:7774
-    connect: http://media_api:7774
-  external_api:
-    listen: http://0.0.0.0:8074
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # Storage path for uploaded media. May be relative or absolute.
-  base_path: /var/dendrite/media
-
-  # The maximum allowed file size (in bytes) for media uploads to this homeserver
-  # (0 = unlimited).
-  max_file_size_bytes: 10485760
-
-  # Whether to dynamically generate thumbnails if needed.
-  dynamic_thumbnails: false
-
-  # The maximum number of simultaneous thumbnail generators to run.
-  max_thumbnail_generators: 10
-
-  # A list of thumbnail sizes to be generated for media content.
-  thumbnail_sizes:
-    - width: 32
-      height: 32
-      method: crop
-    - width: 96
-      height: 96
-      method: crop
-    - width: 640
-      height: 480
-      method: scale
-
-# Configuration for experimental MSC's
-mscs:
-  # A list of enabled MSC's
-  # Currently valid values are:
-  # - msc2836    (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
-  # - msc2946    (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
-  mscs: []
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mscs?sslmode=disable
-    max_open_conns: 5
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Room Server.
-room_server:
-  internal_api:
-    listen: http://0.0.0.0:7770
-    connect: http://room_server:7770
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Sync API.
-sync_api:
-  internal_api:
-    listen: http://0.0.0.0:7773
-    connect: http://sync_api:7773
-  external_api:
-    listen: http://0.0.0.0:8073
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the User API.
-user_api:
-  internal_api:
-    listen: http://0.0.0.0:7781
-    connect: http://user_api:7781
-  account_database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_userapi_accounts?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Push Server API.
-push_server:
-  internal_api:
-    listen: http://localhost:7782
-    connect: http://localhost:7782
-  database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_pushserver?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for Opentracing.
-# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
-# how this works and how to set it up.
-tracing:
-  enabled: false
-  jaeger:
-    serviceName: ""
-    disabled: false
-    rpc_metrics: false
-    tags: []
-    sampler: null
-    reporter: null
-    headers: null
-    baggage_restrictions: null
-    throttler: null
-
-# Logging configuration, in addition to the standard logging that is sent to
-# stdout by Dendrite.
-logging:
-  - type: file
-    level: info
-    params:
-      path: /var/log/dendrite
diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
new file mode 100644
index 000000000..e974dbcba
--- /dev/null
+++ b/dendrite-sample.monolith.yaml
@@ -0,0 +1,279 @@
+# This is the Dendrite configuration file.
+#
+# The configuration is split up into sections - each Dendrite component has a
+# configuration section, in addition to the "global" section which applies to
+# all components.
+
+# The version of the configuration file.
+version: 2
+
+# Global Matrix configuration. This configuration applies to all components.
+global:
+  # The domain name of this homeserver.
+  server_name: localhost
+
+  # The path to the signing private key file, used to sign requests and events.
+  # Note that this is NOT the same private key as used for TLS! To generate a
+  # signing key, use "./bin/generate-keys --private-key matrix_key.pem".
+  private_key: matrix_key.pem
+
+  # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
+  # to old signing private keys that were formerly in use on this domain. These
+  # keys will not be used for federation request or event signing, but will be
+  # provided to any other homeserver that asks when trying to verify old events.
+  old_private_keys:
+  #  - private_key: old_matrix_key.pem
+  #    expired_at: 1601024554498
+
+  # How long a remote server can cache our server signing key before requesting it
+  # again. Increasing this number will reduce the number of requests made by other
+  # servers for our key but increases the period that a compromised key will be
+  # considered valid by other homeservers.
+  key_validity_period: 168h0m0s
+
+  # Global database connection pool, for PostgreSQL monolith deployments only. If
+  # this section is populated then you can omit the "database" blocks in all other
+  # sections. For polylith deployments, or monolith deployments using SQLite databases,
+  # you must configure the "database" block for each component instead.
+  database:
+    connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable
+    max_open_conns: 100
+    max_idle_conns: 5
+    conn_max_lifetime: -1
+
+  # The server name to delegate server-server communications to, with optional port
+  # e.g. localhost:443
+  well_known_server_name: ""
+
+  # Lists of domains that the server will trust as identity servers to verify third
+  # party identifiers such as phone numbers and email addresses.
+  trusted_third_party_id_servers:
+    - matrix.org
+    - vector.im
+
+  # Disables federation. Dendrite will not be able to communicate with other servers
+  # in the Matrix federation and the federation API will not be exposed.
+  disable_federation: false
+
+  # Configures the handling of presence events. Inbound controls whether we receive
+  # presence events from other servers, outbound controls whether we send presence
+  # events for our local users to other servers.
+  presence:
+    enable_inbound: false
+    enable_outbound: false
+
+  # Configures phone-home statistics reporting. These statistics contain the server
+  # name, number of active users and some information on your deployment config.
+  # We use this information to understand how Dendrite is being used in the wild.
+  report_stats:
+    enabled: false
+    endpoint: https://matrix.org/report-usage-stats/push
+
+  # Server notices allows server admins to send messages to all users on the server.
+  server_notices:
+    enabled: false
+    # The local part, display name and avatar URL (as a mxc:// URL) for the user that
+    # will send the server notices. These are visible to all users on the deployment.
+    local_part: "_server"
+    display_name: "Server Alerts"
+    avatar_url: ""
+    # The room name to be used when sending server notices. This room name will
+    # appear in user clients.
+    room_name: "Server Alerts"
+
+  # Configuration for NATS JetStream
+  jetstream:
+    # A list of NATS Server addresses to connect to. If none are specified, an
+    # internal NATS server will be started automatically when running Dendrite in
+    # monolith mode. For polylith deployments, it is required to specify the address
+    # of at least one NATS Server node.
+    addresses:
+    # - localhost:4222
+
+    # Persistent directory to store JetStream streams in. This directory should be
+    # preserved across Dendrite restarts.
+    storage_path: ./
+
+    # The prefix to use for stream names for this homeserver - really only useful
+    # if you are running more than one Dendrite server on the same NATS deployment.
+    topic_prefix: Dendrite
+
+  # Configuration for Prometheus metric collection.
+  metrics:
+    enabled: false
+    basic_auth:
+      username: metrics
+      password: metrics
+
+  # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there
+  # is no local caching resolver available for use.
+  dns_cache:
+    enabled: false
+    cache_size: 256
+    cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration
+
+# Configuration for the Appservice API.
+app_service_api:
+  # Disable the validation of TLS certificates of appservices. This is
+  # not recommended in production since it may allow appservice traffic
+  # to be sent to an insecure endpoint.
+  disable_tls_validation: false
+
+  # Appservice configuration files to load into this homeserver.
+  config_files:
+  #  - /path/to/appservice_registration.yaml
+
+# Configuration for the Client API.
+client_api:
+  # Prevents new users from being able to register on this homeserver, except when
+  # using the registration shared secret below.
+  registration_disabled: true
+
+  # Prevents new guest accounts from being created. Guest registration is also
+  # disabled implicitly by setting 'registration_disabled' above.
+  guests_disabled: true
+
+  # If set, allows registration by anyone who knows the shared secret, regardless
+  # of whether registration is otherwise disabled.
+  registration_shared_secret: ""
+
+  # Whether to require reCAPTCHA for registration. If you have enabled registration
+  # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used
+  # for coordinated spam attacks.
+  enable_registration_captcha: false
+
+  # Settings for ReCAPTCHA.
+  recaptcha_public_key: ""
+  recaptcha_private_key: ""
+  recaptcha_bypass_secret: ""
+  recaptcha_siteverify_api: ""
+
+  # TURN server information that this homeserver should send to clients.
+  turn:
+    turn_user_lifetime: ""
+    turn_uris:
+    #  - turn:turn.server.org?transport=udp
+    #  - turn:turn.server.org?transport=tcp
+    turn_shared_secret: ""
+    turn_username: ""
+    turn_password: ""
+
+  # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
+  # number of "slots" have been taken by requests from a specific host. Each "slot"
+  # will be released after the cooloff time in milliseconds.
+  rate_limiting:
+    enabled: true
+    threshold: 5
+    cooloff_ms: 500
+
+# Configuration for the Federation API.
+federation_api:
+  # How many times we will try to resend a failed transaction to a specific server. The
+  # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once
+  # the max retries are exceeded, Dendrite will no longer try to send transactions to
+  # that server until it comes back to life and connects to us again.
+  send_max_retries: 16
+
+  # Disable the validation of TLS certificates of remote federated homeservers. Do not
+  # enable this option in production as it presents a security risk!
+  disable_tls_validation: false
+
+  # Perspective keyservers to use as a backup when direct key fetches fail. This may
+  # be required to satisfy key requests for servers that are no longer online when
+  # joining some rooms.
+  key_perspectives:
+    - server_name: matrix.org
+      keys:
+        - key_id: ed25519:auto
+          public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
+        - key_id: ed25519:a_RXGa
+          public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
+
+  # This option will control whether Dendrite will prefer to look up keys directly
+  # or whether it should try perspective servers first, using direct fetches as a
+  # last resort.
+  prefer_direct_fetch: false
+
+# Configuration for the Media API.
+media_api:
+  # Storage path for uploaded media. May be relative or absolute.
+  base_path: ./media_store
+
+  # The maximum allowed file size (in bytes) for media uploads to this homeserver
+  # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least
+  #this large (e.g. the client_max_body_size setting in nginx).
+  max_file_size_bytes: 10485760
+
+  # Whether to dynamically generate thumbnails if needed.
+  dynamic_thumbnails: false
+
+  # The maximum number of simultaneous thumbnail generators to run.
+  max_thumbnail_generators: 10
+
+  # A list of thumbnail sizes to be generated for media content.
+  thumbnail_sizes:
+    - width: 32
+      height: 32
+      method: crop
+    - width: 96
+      height: 96
+      method: crop
+    - width: 640
+      height: 480
+      method: scale
+
+# Configuration for enabling experimental MSCs on this homeserver.
+mscs:
+  mscs:
+  #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
+  #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
+
+# Configuration for the Sync API.
+sync_api:
+  # This option controls which HTTP header to inspect to find the real remote IP
+  # address of the client. This is likely required if Dendrite is running behind
+  # a reverse proxy server.
+  # real_ip_header: X-Real-IP
+
+# Configuration for the User API.
+user_api:
+  # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31
+  # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information.
+  # Setting this lower makes registration/login consume less CPU resources at the cost
+  # of security should the database be compromised. Setting this higher makes registration/login
+  # consume more CPU resources but makes it harder to brute force password hashes. This value
+  # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds).
+  bcrypt_cost: 10
+
+  # The length of time that a token issued for a relying party from
+  # /_matrix/client/r0/user/{userId}/openid/request_token endpoint
+  # is considered to be valid in milliseconds.
+  # The default lifetime is 3600000ms (60 minutes).
+  # openid_token_lifetime_ms: 3600000
+
+# Configuration for Opentracing.
+# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
+# how this works and how to set it up.
+tracing:
+  enabled: false
+  jaeger:
+    serviceName: ""
+    disabled: false
+    rpc_metrics: false
+    tags: []
+    sampler: null
+    reporter: null
+    headers: null
+    baggage_restrictions: null
+    throttler: null
+
+# Logging configuration. The "std" logging type controls the logs being sent to
+# stdout. The "file" logging type controls logs being written to a log folder on
+# the disk. Supported log levels are "debug", "info", "warn", "error".
+logging:
+  - type: std
+    level: info
+  - type: file
+    level: info
+    params:
+      path: ./logs
diff --git a/dendrite-config.yaml b/dendrite-sample.polylith.yaml
similarity index 54%
rename from dendrite-config.yaml
rename to dendrite-sample.polylith.yaml
index 7709e0c87..4b67aaa94 100644
--- a/dendrite-config.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -3,29 +3,6 @@
 # The configuration is split up into sections - each Dendrite component has a
 # configuration section, in addition to the "global" section which applies to
 # all components.
-#
-# At a minimum, to get started, you will need to update the settings in the
-# "global" section for your deployment, and you will need to check that the
-# database "connection_string" line in each component section is correct.
-#
-# Each component with a "database" section can accept the following formats
-# for "connection_string":
-#   SQLite:     file:filename.db
-#               file:///path/to/filename.db
-#   PostgreSQL: postgresql://user:pass@hostname/database?params=...
-#
-# SQLite is embedded into Dendrite and therefore no further prerequisites are
-# needed for the database when using SQLite mode. However, performance with
-# PostgreSQL is significantly better and recommended for multi-user deployments.
-# SQLite is typically around 20-30% slower than PostgreSQL when tested with a
-# small number of users and likely will perform worse still with a higher volume
-# of users.
-#
-# The "max_open_conns" and "max_idle_conns" settings configure the maximum
-# number of open/idle database connections. The value 0 will use the database
-# engine default, and a negative value will use unlimited connections. The
-# "conn_max_lifetime" option controls the maximum length of time a database
-# connection can be idle in seconds - a negative value is unlimited.
 
 # The version of the configuration file.
 version: 2
@@ -44,9 +21,9 @@ global:
   # to old signing private keys that were formerly in use on this domain. These
   # keys will not be used for federation request or event signing, but will be
   # provided to any other homeserver that asks when trying to verify old events.
-  # old_private_keys:
-  # - private_key: old_matrix_key.pem
-  #   expired_at: 1601024554498
+  old_private_keys:
+  #  - private_key: old_matrix_key.pem
+  #    expired_at: 1601024554498
 
   # How long a remote server can cache our server signing key before requesting it
   # again. Increasing this number will reduce the number of requests made by other
@@ -54,16 +31,6 @@ global:
   # considered valid by other homeservers.
   key_validity_period: 168h0m0s
 
-  # Global database connection pool, for PostgreSQL monolith deployments only. If
-  # this section is populated then you can omit the "database" blocks in all other
-  # sections. For polylith deployments, or monolith deployments using SQLite databases,
-  # you must configure the "database" block for each component instead.
-  # database:
-  #   connection_string: postgres://user:pass@hostname/database?sslmode=disable
-  #   max_open_conns: 100
-  #   max_idle_conns: 5
-  #   conn_max_lifetime: -1
-
   # The server name to delegate server-server communications to, with optional port
   # e.g. localhost:443
   well_known_server_name: ""
@@ -74,105 +41,90 @@ global:
     - matrix.org
     - vector.im
 
-  # Disables federation. Dendrite will not be able to make any outbound HTTP requests
-  # to other servers and the federation API will not be exposed.
+  # Disables federation. Dendrite will not be able to communicate with other servers
+  # in the Matrix federation and the federation API will not be exposed.
   disable_federation: false
 
-  # Configures the handling of presence events.
+  # Configures the handling of presence events. Inbound controls whether we receive
+  # presence events from other servers, outbound controls whether we send presence
+  # events for our local users to other servers.
   presence:
-    # Whether inbound presence events are allowed, e.g. receiving presence events from other servers
     enable_inbound: false
-    # Whether outbound presence events are allowed, e.g. sending presence events to other servers
     enable_outbound: false
 
-  # Configures opt-in anonymous stats reporting.
+  # Configures phone-home statistics reporting. These statistics contain the server
+  # name, number of active users and some information on your deployment config.
+  # We use this information to understand how Dendrite is being used in the wild.
   report_stats:
-    # Whether this instance sends anonymous usage stats
     enabled: false
-
-    # The endpoint to report the anonymized homeserver usage statistics to.
-    # Defaults to https://matrix.org/report-usage-stats/push
     endpoint: https://matrix.org/report-usage-stats/push
 
-  # Server notices allows server admins to send messages to all users.
+  # Server notices allows server admins to send messages to all users on the server.
   server_notices:
     enabled: false
-    # The server localpart to be used when sending notices, ensure this is not yet taken
+    # The local part, display name and avatar URL (as a mxc:// URL) for the user that
+    # will send the server notices. These are visible to all users on the deployment.
     local_part: "_server"
-    # The displayname to be used when sending notices
-    display_name: "Server alerts"
-    # The mxid of the avatar to use
+    display_name: "Server Alerts"
     avatar_url: ""
-    # The roomname to be used when creating messages
+    # The room name to be used when sending server notices. This room name will
+    # appear in user clients.
     room_name: "Server Alerts"
 
   # Configuration for NATS JetStream
   jetstream:
     # A list of NATS Server addresses to connect to. If none are specified, an
-    # internal NATS server will be started automatically when running Dendrite
-    # in monolith mode. It is required to specify the address of at least one
-    # NATS Server node if running in polylith mode.
+    # internal NATS server will be started automatically when running Dendrite in
+    # monolith mode. For polylith deployments, it is required to specify the address
+    # of at least one NATS Server node.
     addresses:
-    # - localhost:4222
+      - hostname:4222
 
-    # Keep all NATS streams in memory, rather than persisting it to the storage
-    # path below. This option is present primarily for integration testing and
-    # should not be used on a real world Dendrite deployment.
-    in_memory: false
-
-    # Persistent directory to store JetStream streams in. This directory
-    # should be preserved across Dendrite restarts.
-    storage_path: ./
-
-    # The prefix to use for stream names for this homeserver - really only
-    # useful if running more than one Dendrite on the same NATS deployment.
+    # The prefix to use for stream names for this homeserver - really only useful
+    # if you are running more than one Dendrite server on the same NATS deployment.
     topic_prefix: Dendrite
 
   # Configuration for Prometheus metric collection.
   metrics:
-    # Whether or not Prometheus metrics are enabled.
     enabled: false
-
-    # HTTP basic authentication to protect access to monitoring.
     basic_auth:
       username: metrics
       password: metrics
 
-  # DNS cache options. The DNS cache may reduce the load on DNS servers
-  # if there is no local caching resolver available for use.
+  # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there
+  # is no local caching resolver available for use.
   dns_cache:
-    # Whether or not the DNS cache is enabled.
     enabled: false
-
-    # Maximum number of entries to hold in the DNS cache, and
-    # for how long those items should be considered valid in seconds.
     cache_size: 256
-    cache_lifetime: "5m" # 5minutes; see https://pkg.go.dev/time@master#ParseDuration for more
+    cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration
 
 # Configuration for the Appservice API.
 app_service_api:
   internal_api:
-    listen: http://localhost:7777 # Only used in polylith deployments
-    connect: http://localhost:7777 # Only used in polylith deployments
+    listen: http://[::]:7777 # The listen address for incoming API requests
+    connect: http://app_service_api:7777 # The connect address for other components to use
+
+  # Database configuration for this component.
   database:
-    connection_string: file:appservice.db
+    connection_string: postgresql://username@password:hostname/dendrite_appservice?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
   # Disable the validation of TLS certificates of appservices. This is
   # not recommended in production since it may allow appservice traffic
-  # to be sent to an unverified endpoint.
+  # to be sent to an insecure endpoint.
   disable_tls_validation: false
 
   # Appservice configuration files to load into this homeserver.
-  config_files: []
+  config_files:
+  #  - /path/to/appservice_registration.yaml
 
 # Configuration for the Client API.
 client_api:
   internal_api:
-    listen: http://localhost:7771 # Only used in polylith deployments
-    connect: http://localhost:7771 # Only used in polylith deployments
+    listen: http://[::]:7771 # The listen address for incoming API requests
+    connect: http://client_api:7771 # The connect address for other components to use
   external_api:
     listen: http://[::]:8071
 
@@ -184,11 +136,13 @@ client_api:
   # disabled implicitly by setting 'registration_disabled' above.
   guests_disabled: true
 
-  # If set, allows registration by anyone who knows the shared secret, regardless of
-  # whether registration is otherwise disabled.
+  # If set, allows registration by anyone who knows the shared secret, regardless
+  # of whether registration is otherwise disabled.
   registration_shared_secret: ""
 
-  # Whether to require reCAPTCHA for registration.
+  # Whether to require reCAPTCHA for registration. If you have enabled registration
+  # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used
+  # for coordinated spam attacks.
   enable_registration_captcha: false
 
   # Settings for ReCAPTCHA.
@@ -200,14 +154,16 @@ client_api:
   # TURN server information that this homeserver should send to clients.
   turn:
     turn_user_lifetime: ""
-    turn_uris: []
+    turn_uris:
+    #  - turn:turn.server.org?transport=udp
+    #  - turn:turn.server.org?transport=tcp
     turn_shared_secret: ""
     turn_username: ""
     turn_password: ""
 
-  # Settings for rate-limited endpoints. Rate limiting will kick in after the
-  # threshold number of "slots" have been taken by requests from a specific
-  # host. Each "slot" will be released after the cooloff time in milliseconds.
+  # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
+  # number of "slots" have been taken by requests from a specific host. Each "slot"
+  # will be released after the cooloff time in milliseconds.
   rate_limiting:
     enabled: true
     threshold: 5
@@ -216,18 +172,20 @@ client_api:
 # Configuration for the Federation API.
 federation_api:
   internal_api:
-    listen: http://localhost:7772 # Only used in polylith deployments
-    connect: http://localhost:7772 # Only used in polylith deployments
+    listen: http://[::]:7772 # The listen address for incoming API requests
+    connect: http://federation_api:7772 # The connect address for other components to use
   external_api:
     listen: http://[::]:8072
   database:
-    connection_string: file:federationapi.db
+    connection_string: postgresql://username@password:hostname/dendrite_federationapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
 
   # How many times we will try to resend a failed transaction to a specific server. The
-  # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc.
+  # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once
+  # the max retries are exceeded, Dendrite will no longer try to send transactions to
+  # that server until it comes back to life and connects to us again.
   send_max_retries: 16
 
   # Disable the validation of TLS certificates of remote federated homeservers. Do not
@@ -253,10 +211,10 @@ federation_api:
 # Configuration for the Key Server (for end-to-end encryption).
 key_server:
   internal_api:
-    listen: http://localhost:7779 # Only used in polylith deployments
-    connect: http://localhost:7779 # Only used in polylith deployments
+    listen: http://[::]:7779 # The listen address for incoming API requests
+    connect: http://key_server:7779 # The connect address for other components to use
   database:
-    connection_string: file:keyserver.db
+    connection_string: postgresql://username@password:hostname/dendrite_keyserver?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -264,12 +222,12 @@ key_server:
 # Configuration for the Media API.
 media_api:
   internal_api:
-    listen: http://localhost:7774 # Only used in polylith deployments
-    connect: http://localhost:7774 # Only used in polylith deployments
+    listen: http://[::]:7774 # The listen address for incoming API requests
+    connect: http://media_api:7774 # The connect address for other components to use
   external_api:
     listen: http://[::]:8074
   database:
-    connection_string: file:mediaapi.db
+    connection_string: postgresql://username@password:hostname/dendrite_mediaapi?sslmode=disable
     max_open_conns: 5
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -278,8 +236,8 @@ media_api:
   base_path: ./media_store
 
   # The maximum allowed file size (in bytes) for media uploads to this homeserver
-  # (0 = unlimited). If using a reverse proxy, ensure it allows requests at
-  # least this large (e.g. client_max_body_size in nginx.)
+  # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least
+  #this large (e.g. the client_max_body_size setting in nginx).
   max_file_size_bytes: 10485760
 
   # Whether to dynamically generate thumbnails if needed.
@@ -300,15 +258,13 @@ media_api:
       height: 480
       method: scale
 
-# Configuration for experimental MSC's
+# Configuration for enabling experimental MSCs on this homeserver.
 mscs:
-  # A list of enabled MSC's
-  # Currently valid values are:
-  # - msc2836    (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
-  # - msc2946    (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
-  mscs: []
+  mscs:
+  #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
+  #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
   database:
-    connection_string: file:mscs.db
+    connection_string: postgresql://username@password:hostname/dendrite_mscs?sslmode=disable
     max_open_conns: 5
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -316,10 +272,10 @@ mscs:
 # Configuration for the Room Server.
 room_server:
   internal_api:
-    listen: http://localhost:7770 # Only used in polylith deployments
-    connect: http://localhost:7770 # Only used in polylith deployments
+    listen: http://[::]:7770 # The listen address for incoming API requests
+    connect: http://room_server:7770 # The connect address for other components to use
   database:
-    connection_string: file:roomserver.db
+    connection_string: postgresql://username@password:hostname/dendrite_roomserver?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -327,12 +283,12 @@ room_server:
 # Configuration for the Sync API.
 sync_api:
   internal_api:
-    listen: http://localhost:7773 # Only used in polylith deployments
-    connect: http://localhost:7773 # Only used in polylith deployments
+    listen: http://[::]:7773 # The listen address for incoming API requests
+    connect: http://sync_api:7773 # The connect address for other components to use
   external_api:
     listen: http://[::]:8073
   database:
-    connection_string: file:syncapi.db
+    connection_string: postgresql://username@password:hostname/dendrite_syncapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -344,21 +300,23 @@ sync_api:
 
 # Configuration for the User API.
 user_api:
-  # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31
-  # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information.
-  # Setting this lower makes registration/login consume less CPU resources at the cost of security
-  # should the database be compromised. Setting this higher makes registration/login consume more
-  # CPU resources but makes it harder to brute force password hashes.
-  # This value can be low if performing tests or on embedded Dendrite instances (e.g WASM builds)
-  # bcrypt_cost: 10
   internal_api:
-    listen: http://localhost:7781 # Only used in polylith deployments
-    connect: http://localhost:7781 # Only used in polylith deployments
+    listen: http://[::]:7781 # The listen address for incoming API requests
+    connect: http://user_api:7781 # The connect address for other components to use
   account_database:
-    connection_string: file:userapi_accounts.db
+    connection_string: postgresql://username@password:hostname/dendrite_userapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
+
+  # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31
+  # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information.
+  # Setting this lower makes registration/login consume less CPU resources at the cost
+  # of security should the database be compromised. Setting this higher makes registration/login
+  # consume more CPU resources but makes it harder to brute force password hashes. This value
+  # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds).
+  bcrypt_cost: 10
+
   # The length of time that a token issued for a relying party from
   # /_matrix/client/r0/user/{userId}/openid/request_token endpoint
   # is considered to be valid in milliseconds.
@@ -381,12 +339,13 @@ tracing:
     baggage_restrictions: null
     throttler: null
 
-# Logging configuration
+# Logging configuration. The "std" logging type controls the logs being sent to
+# stdout. The "file" logging type controls logs being written to a log folder on
+# the disk. Supported log levels are "debug", "info", "warn", "error".
 logging:
   - type: std
     level: info
   - type: file
-    # The logging level, must be one of debug, info, warn, error, fatal, panic.
     level: info
     params:
       path: ./logs
diff --git a/docs/FAQ.md b/docs/FAQ.md
index 571726d61..47f39b9e6 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -58,7 +58,7 @@ Bridges known to work (as of v0.5.1):
 * [Signal](https://docs.mau.fi/bridges/python/signal/index.html)
 * [probably all other mautrix bridges](https://docs.mau.fi/bridges/)
 
-Remember to add the config file(s) to the `app_service_api` [config](https://github.com/matrix-org/dendrite/blob/de38be469a23813921d01bef3e14e95faab2a59e/dendrite-config.yaml#L130-L131).
+Remember to add the config file(s) to the `app_service_api` section of the config file.
 
 ## Is it possible to prevent communication with the outside world?
 
diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md
index 868aba6ec..e676afbe6 100644
--- a/docs/installation/7_configuration.md
+++ b/docs/installation/7_configuration.md
@@ -7,11 +7,13 @@ permalink: /installation/configuration
 
 # Populate the configuration
 
-The configuration file is used to configure Dendrite. A sample configuration file,
-called [`dendrite-config.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-config.yaml),
-is present in the top level of the Dendrite repository.
+The configuration file is used to configure Dendrite. Sample configuration files are
+present in the top level of the Dendrite repository:
 
-You will need to duplicate this file, calling it `dendrite.yaml` for example, and then
+* [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml)
+* [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
+
+You will need to duplicate the sample, calling it `dendrite.yaml` for example, and then
 tailor it to your installation. At a minimum, you will need to populate the following
 sections:
 

From 870f9b0c3f288950ab843b048485a0767e177bd1 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 13 May 2022 09:33:55 +0200
Subject: [PATCH 109/391] Shuffle config Verify/Defaults a bit around (#2459)

---
 setup/config/config_appservice.go    |  7 +++++--
 setup/config/config_clientapi.go     | 22 +++++++++++-----------
 setup/config/config_federationapi.go | 16 ++++++++--------
 setup/config/config_jetstream.go     |  7 ++++---
 setup/config/config_keyserver.go     |  7 +++++--
 setup/config/config_mediaapi.go      | 17 ++++++++---------
 setup/config/config_roomserver.go    |  7 +++++--
 setup/config/config_syncapi.go       | 11 ++++++-----
 setup/config/config_userapi.go       | 13 ++++++++-----
 9 files changed, 60 insertions(+), 47 deletions(-)

diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go
index d93b6ebe0..ff3287714 100644
--- a/setup/config/config_appservice.go
+++ b/setup/config/config_appservice.go
@@ -50,11 +50,14 @@ func (c *AppServiceAPI) Defaults(generate bool) {
 }
 
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect))
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "app_service_api.internal_api.connect", string(c.InternalAPI.Connect))
 }
 
 // ApplicationServiceNamespace is the namespace that a specific application
diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go
index 6104ed8b9..bb786a145 100644
--- a/setup/config/config_clientapi.go
+++ b/setup/config/config_clientapi.go
@@ -67,19 +67,13 @@ func (c *ClientAPI) Defaults(generate bool) {
 }
 
 func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
-	if c.RecaptchaEnabled {
-		checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey))
-		checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey))
-		checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI))
-	}
 	c.TURN.Verify(configErrs)
 	c.RateLimiting.Verify(configErrs)
-
+	if c.RecaptchaEnabled {
+		checkNotEmpty(configErrs, "client_api.recaptcha_public_key", c.RecaptchaPublicKey)
+		checkNotEmpty(configErrs, "client_api.recaptcha_private_key", c.RecaptchaPrivateKey)
+		checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", c.RecaptchaSiteVerifyAPI)
+	}
 	// Ensure there is any spam counter measure when enabling registration
 	if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled {
 		if !c.RecaptchaEnabled {
@@ -93,6 +87,12 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 			)
 		}
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
+	checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
 }
 
 type TURN struct {
diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go
index f62a23e1f..a7a515fda 100644
--- a/setup/config/config_federationapi.go
+++ b/setup/config/config_federationapi.go
@@ -34,24 +34,24 @@ func (c *FederationAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7772"
 	c.InternalAPI.Connect = "http://localhost:7772"
 	c.ExternalAPI.Listen = "http://[::]:8072"
+	c.FederationMaxRetries = 16
+	c.DisableTLSValidation = false
 	c.Database.Defaults(10)
 	if generate {
 		c.Database.ConnectionString = "file:federationapi.db"
 	}
-
-	c.FederationMaxRetries = 16
-	c.DisableTLSValidation = false
 }
 
 func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
+	checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
 }
 
 // The config for setting a proxy to use for server->server requests
diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go
index b6a93d398..e4cfd4d3b 100644
--- a/setup/config/config_jetstream.go
+++ b/setup/config/config_jetstream.go
@@ -36,9 +36,10 @@ func (c *JetStream) Defaults(generate bool) {
 }
 
 func (c *JetStream) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	if isMonolith { // polylith required configs below
+		return
+	}
 	// If we are running in a polylith deployment then we need at least
 	// one NATS JetStream server to talk to.
-	if !isMonolith {
-		checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
-	}
+	checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses)))
 }
diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go
index 9e2d54cdc..5f2f22c8a 100644
--- a/setup/config/config_keyserver.go
+++ b/setup/config/config_keyserver.go
@@ -18,9 +18,12 @@ func (c *KeyServer) Defaults(generate bool) {
 }
 
 func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect))
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect))
 }
diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go
index 273de322a..9717aa59e 100644
--- a/setup/config/config_mediaapi.go
+++ b/setup/config/config_mediaapi.go
@@ -42,26 +42,19 @@ func (c *MediaAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7774"
 	c.InternalAPI.Connect = "http://localhost:7774"
 	c.ExternalAPI.Listen = "http://[::]:8074"
+	c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
+	c.MaxThumbnailGenerators = 10
 	c.Database.Defaults(5)
 	if generate {
 		c.Database.ConnectionString = "file:mediaapi.db"
 		c.BasePath = "./media_store"
 	}
-
-	c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
-	c.MaxThumbnailGenerators = 10
 }
 
 func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
 	}
-
 	checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
 	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes))
 	checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
@@ -70,4 +63,10 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 		checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width))
 		checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
+	checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
 }
diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go
index 8a3227349..bd6aa1167 100644
--- a/setup/config/config_roomserver.go
+++ b/setup/config/config_roomserver.go
@@ -18,9 +18,12 @@ func (c *RoomServer) Defaults(generate bool) {
 }
 
 func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect))
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect))
 }
diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go
index 48fd9f506..7d5e3808a 100644
--- a/setup/config/config_syncapi.go
+++ b/setup/config/config_syncapi.go
@@ -22,12 +22,13 @@ func (c *SyncAPI) Defaults(generate bool) {
 }
 
 func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "sync_api.internal_api.bind", string(c.InternalAPI.Connect))
-	if !isMonolith {
-		checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
-	}
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
 	}
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
+	checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
 }
diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go
index 4aa3b57bb..d1e2b7fe1 100644
--- a/setup/config/config_userapi.go
+++ b/setup/config/config_userapi.go
@@ -26,19 +26,22 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
 func (c *UserAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7781"
 	c.InternalAPI.Connect = "http://localhost:7781"
+	c.BCryptCost = bcrypt.DefaultCost
+	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
 	c.AccountDatabase.Defaults(10)
 	if generate {
 		c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
 	}
-	c.BCryptCost = bcrypt.DefaultCost
-	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
 }
 
 func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
-	checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
+	checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
 	}
-	checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
+	if isMonolith { // polylith required configs below
+		return
+	}
+	checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
+	checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
 }

From b57fdcc82d0c57ae3eed401e7b9891cc9b53a8d3 Mon Sep 17 00:00:00 2001
From: Till Faelligen <tfaelligen@gmail.com>
Date: Fri, 13 May 2022 10:24:26 +0200
Subject: [PATCH 110/391] Only try to get OTKs if the context isn't done yet

---
 syncapi/sync/requestpool.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 30c490df0..fdf46cdde 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -253,9 +253,12 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 			// We should always try to include OTKs in sync responses, otherwise clients might upload keys
 			// even if that's not required. See also:
 			// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
-			err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
-			if err != nil {
-				syncReq.Log.WithError(err).Error("failed to get OTK counts")
+			// Only try to get OTKs if the context isn't already done.
+			if syncReq.Context.Err() == nil {
+				err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
+				if err != nil && err != context.Canceled {
+					syncReq.Log.WithError(err).Warn("failed to get OTK counts")
+				}
 			}
 			return util.JSONResponse{
 				Code: http.StatusOK,

From cafc2d2c10daeeaf8012a50163d07815b5516043 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 13 May 2022 11:36:04 +0100
Subject: [PATCH 111/391] Update NATS Server to version 2.8.2 (#2460)

---
 go.mod | 2 +-
 go.sum | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index d14ced5b7..ca817b00f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/matrix-org/dendrite
 
-replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9
+replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f
 
 replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
 
diff --git a/go.sum b/go.sum
index 8b518935c..7544768c3 100644
--- a/go.sum
+++ b/go.sum
@@ -889,8 +889,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9 h1:VGU5HYAwy8LRbSkrT+kCHvujVmwK8Aa/vc1O+eReTbM=
-github.com/neilalexander/nats-server/v2 v2.8.1-0.20220419100629-2278c94774f9/go.mod h1:5vic7C58BFEVltiZhs7Kq81q2WcEPhJPsmNv1FOrdv0=
+github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo=
+github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4=
 github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y=
 github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
@@ -1280,7 +1280,7 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8=
 golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

From be9be2553f0f18baed07755e81669fd374f3525a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 13 May 2022 11:52:04 +0100
Subject: [PATCH 112/391] Resolve over old and new extremities (#2457)

* Feed existing state into state res when calculating state from new extremities

* Remove duplicates

* Fix bug

* Sort and unique

* Update to matrix-org/gomatrixserverlib#308

* Trim the slice properly

* Update gomatrixserverlib again

* Update to matrix-org/gomatrixserverlib#308
---
 go.mod                                        |  2 +-
 go.sum                                        |  4 ++--
 .../internal/input/input_latest_events.go     | 19 +++++++++++++------
 roomserver/types/types.go                     | 15 +++++++++++++++
 4 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index ca817b00f..ecdeb77fa 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 7544768c3..145c2a04c 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c h1:KqzqFWxvs90pcDaW9QEveW+Q5JcEYuNnKyaqXc+ohno=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220509120958-8d818048c34c/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433 h1:nwAlThHGPI2EAAJklXvgMcdhXF6ZiHp60+fmaYMoaDA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 9ad8b0422..e4c138d58 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -233,12 +233,19 @@ func (u *latestEventsUpdater) latestState() error {
 		}
 	}
 
-	// Get a list of the current latest events. This may or may not
-	// include the new event from the input path, depending on whether
-	// it is a forward extremity or not.
-	latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
-	for i := range u.latest {
-		latestStateAtEvents[i] = u.latest[i].StateAtEvent
+	// Take the old set of extremities and the new set of extremities and
+	// mash them together into a list. This may or may not include the new event
+	// from the input path, depending on whether it became a forward extremity
+	// or not. We'll then run state resolution across all of them to determine
+	// the new current state of the room. Including the old extremities here
+	// ensures that new forward extremities with bad state snapshots (from
+	// possible malicious actors) can't completely corrupt the room state
+	// away from what it was before.
+	combinedExtremities := types.StateAtEventAndReferences(append(u.oldLatest, u.latest...))
+	combinedExtremities = combinedExtremities[:util.SortAndUnique(combinedExtremities)]
+	latestStateAtEvents := make([]types.StateAtEvent, len(combinedExtremities))
+	for i := range combinedExtremities {
+		latestStateAtEvents[i] = combinedExtremities[i].StateAtEvent
 	}
 
 	// Takes the NIDs of the latest events and creates a state snapshot
diff --git a/roomserver/types/types.go b/roomserver/types/types.go
index 65fbee04e..ce4e5fd1e 100644
--- a/roomserver/types/types.go
+++ b/roomserver/types/types.go
@@ -18,6 +18,7 @@ package types
 import (
 	"encoding/json"
 	"sort"
+	"strings"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/blake2b"
@@ -166,6 +167,20 @@ type StateAtEventAndReference struct {
 	gomatrixserverlib.EventReference
 }
 
+type StateAtEventAndReferences []StateAtEventAndReference
+
+func (s StateAtEventAndReferences) Less(a, b int) bool {
+	return strings.Compare(s[a].EventID, s[b].EventID) < 0
+}
+
+func (s StateAtEventAndReferences) Len() int {
+	return len(s)
+}
+
+func (s StateAtEventAndReferences) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
 // An Event is a gomatrixserverlib.Event with the numeric event ID attached.
 // It is when performing bulk event lookup in the database.
 type Event struct {

From 1698c395794ab853bf377b293cc0e0cc074cfe00 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 13 May 2022 11:52:42 +0100
Subject: [PATCH 113/391] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ed09e971c..e8b7bd0e2 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,7 @@ The [Federation Tester](https://federationtester.matrix.org) can be used to veri
 
 ## Get started
 
-If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](docs/installation). For running in Docker, see [build/docker](build/docker).
+If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](https://matrix-org.github.io/dendrite/installation). For running in Docker, see [build/docker](build/docker).
 
 The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases:
 

From b40b548432b465e37c6045b6735f9eaf426902f0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 13 May 2022 12:06:47 +0100
Subject: [PATCH 114/391] The Pinecone `gobind` demo must listen on `localhost`
 for `baseURL` to be correct

---
 build/gobind-pinecone/monolith.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 310ac7dda..664ca85d9 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -225,7 +225,7 @@ func (m *DendriteMonolith) Start() {
 		pk = sk.Public().(ed25519.PublicKey)
 	}
 
-	m.listener, err = net.Listen("tcp", ":65432")
+	m.listener, err = net.Listen("tcp", "localhost:65432")
 	if err != nil {
 		panic(err)
 	}

From 6af35385ba06f75610396b91452cbf381c1f5443 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 13 May 2022 13:17:15 +0100
Subject: [PATCH 115/391] Version 0.8.5 (#2461)

* Version 0.8.5

* Update changelog

* Update changelog
---
 CHANGES.md          | 15 +++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index c058da6a1..3deebd8a0 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,20 @@
 # Changelog
 
+## Dendrite 0.8.5 (2022-05-13)
+
+### Features
+
+* New living documentation available at <https://matrix-org.github.io/dendrite/>, including new installation instructions
+* The built-in NATS Server has been updated to version 2.8.2
+
+### Fixes
+
+* Monolith deployments will no longer panic at startup if given a config file that does not include the `internal_api` and `external_api` options
+* State resolution v2 now correctly identifies other events related to power events, which should fix some event auth issues
+* The latest events updater will no longer implicitly trust the new forward extremities when calculating the current room state, which may help to avoid some state resets
+* The one-time key count is now correctly returned in `/sync` even if the request otherwise timed out, which should reduce the chance that unnecessary one-time keys will be uploaded by clients
+* The `create-account` tool should now work properly when the database is configured using the global connection pool
+
 ## Dendrite 0.8.4 (2022-05-10)
 
 ### Fixes
diff --git a/internal/version.go b/internal/version.go
index 5097bb2a6..04c9a8a88 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 4
+	VersionPatch = 5
 	VersionTag   = "" // example: "rc1"
 )
 

From 05607d6b8734738bd5c32288e3d0ef8e827d11d0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 16 May 2022 19:33:16 +0200
Subject: [PATCH 116/391] Add roomserver tests (3/4) (#2447)

* Add Room Aliases tests

* Add Rooms table test

* Move StateKeyTuplerSorter to the types package

* Add StateBlock tests
Some optimizations

* Add State Snapshot tests
Some optimization

* Return []int64 and convert to pq.Int64Array for postgres

* Move []types.EventNID back to rows.Next()

* Update tests, rename SelectRoomIDs
---
 roomserver/storage/postgres/events_table.go   |   6 +-
 .../storage/postgres/room_aliases_table.go    |   6 +-
 roomserver/storage/postgres/rooms_table.go    |  16 +--
 .../storage/postgres/state_block_table.go     |  53 ++------
 .../postgres/state_block_table_test.go        |  86 ------------
 .../storage/postgres/state_snapshot_table.go  |  10 +-
 roomserver/storage/postgres/storage.go        |  16 +--
 roomserver/storage/shared/storage.go          |   2 +-
 roomserver/storage/sqlite3/events_table.go    |   4 +-
 .../storage/sqlite3/room_aliases_table.go     |   6 +-
 roomserver/storage/sqlite3/rooms_table.go     |  16 +--
 .../storage/sqlite3/state_block_table.go      |  49 ++-----
 .../storage/sqlite3/state_block_table_test.go |  86 ------------
 .../storage/sqlite3/state_snapshot_table.go   |  10 +-
 roomserver/storage/sqlite3/storage.go         |  16 +--
 roomserver/storage/tables/interface.go        |   2 +-
 .../storage/tables/room_aliases_table_test.go |  96 +++++++++++++
 roomserver/storage/tables/rooms_table_test.go | 128 ++++++++++++++++++
 .../storage/tables/state_block_table_test.go  |  92 +++++++++++++
 .../tables/state_snapshot_table_test.go       |  86 ++++++++++++
 roomserver/types/types.go                     |  33 +++++
 roomserver/types/types_test.go                |  64 +++++++++
 22 files changed, 570 insertions(+), 313 deletions(-)
 delete mode 100644 roomserver/storage/postgres/state_block_table_test.go
 delete mode 100644 roomserver/storage/sqlite3/state_block_table_test.go
 create mode 100644 roomserver/storage/tables/room_aliases_table_test.go
 create mode 100644 roomserver/storage/tables/rooms_table_test.go
 create mode 100644 roomserver/storage/tables/state_block_table_test.go
 create mode 100644 roomserver/storage/tables/state_snapshot_table_test.go

diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index 86d226ce7..a4d05756d 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -264,11 +264,11 @@ func (s *eventStatements) BulkSelectStateEventByNID(
 	ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID,
 	stateKeyTuples []types.StateKeyTuple,
 ) ([]types.StateEntry, error) {
-	tuples := stateKeyTupleSorter(stateKeyTuples)
+	tuples := types.StateKeyTupleSorter(stateKeyTuples)
 	sort.Sort(tuples)
-	eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays()
+	eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays()
 	stmt := sqlutil.TxStmt(txn, s.bulkSelectStateEventByNIDStmt)
-	rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), eventTypeNIDArray, eventStateKeyNIDArray)
+	rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs), pq.Int64Array(eventTypeNIDArray), pq.Int64Array(eventStateKeyNIDArray))
 	if err != nil {
 		return nil, err
 	}
diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go
index d13df8e7f..a84929f61 100644
--- a/roomserver/storage/postgres/room_aliases_table.go
+++ b/roomserver/storage/postgres/room_aliases_table.go
@@ -61,12 +61,12 @@ type roomAliasesStatements struct {
 	deleteRoomAliasStmt          *sql.Stmt
 }
 
-func createRoomAliasesTable(db *sql.DB) error {
+func CreateRoomAliasesTable(db *sql.DB) error {
 	_, err := db.Exec(roomAliasesSchema)
 	return err
 }
 
-func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
+func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
 	s := &roomAliasesStatements{}
 
 	return s, sqlutil.StatementList{
@@ -108,8 +108,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID(
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed")
 
 	var aliases []string
+	var alias string
 	for rows.Next() {
-		var alias string
 		if err = rows.Scan(&alias); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go
index b2685084d..24362af74 100644
--- a/roomserver/storage/postgres/rooms_table.go
+++ b/roomserver/storage/postgres/rooms_table.go
@@ -95,12 +95,12 @@ type roomStatements struct {
 	bulkSelectRoomNIDsStmt             *sql.Stmt
 }
 
-func createRoomsTable(db *sql.DB) error {
+func CreateRoomsTable(db *sql.DB) error {
 	_, err := db.Exec(roomsSchema)
 	return err
 }
 
-func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
+func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	s := &roomStatements{}
 
 	return s, sqlutil.StatementList{
@@ -117,7 +117,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	}.Prepare(db)
 }
 
-func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) {
+func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
 	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
@@ -125,8 +125,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -231,9 +231,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed")
 	result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion)
+	var roomNID types.RoomNID
+	var roomVersion gomatrixserverlib.RoomVersion
 	for rows.Next() {
-		var roomNID types.RoomNID
-		var roomVersion gomatrixserverlib.RoomVersion
 		if err = rows.Scan(&roomNID, &roomVersion); err != nil {
 			return nil, err
 		}
@@ -254,8 +254,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -276,8 +276,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed")
 	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var roomNID types.RoomNID
 		if err = rows.Scan(&roomNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go
index 6f8f9e1b5..5af48f031 100644
--- a/roomserver/storage/postgres/state_block_table.go
+++ b/roomserver/storage/postgres/state_block_table.go
@@ -19,7 +19,6 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"sort"
 
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal"
@@ -71,12 +70,12 @@ type stateBlockStatements struct {
 	bulkSelectStateBlockEntriesStmt *sql.Stmt
 }
 
-func createStateBlockTable(db *sql.DB) error {
+func CreateStateBlockTable(db *sql.DB) error {
 	_, err := db.Exec(stateDataSchema)
 	return err
 }
 
-func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
+func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
 	s := &stateBlockStatements{}
 
 	return s, sqlutil.StatementList{
@@ -90,9 +89,9 @@ func (s *stateBlockStatements) BulkInsertStateData(
 	entries types.StateEntries,
 ) (id types.StateBlockNID, err error) {
 	entries = entries[:util.SortAndUnique(entries)]
-	var nids types.EventNIDs
-	for _, e := range entries {
-		nids = append(nids, e.EventNID)
+	nids := make(types.EventNIDs, entries.Len())
+	for i := range entries {
+		nids[i] = entries[i].EventNID
 	}
 	stmt := sqlutil.TxStmt(txn, s.insertStateDataStmt)
 	err = stmt.QueryRowContext(
@@ -113,15 +112,15 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
 
 	results := make([][]types.EventNID, len(stateBlockNIDs))
 	i := 0
+	var stateBlockNID types.StateBlockNID
+	var result pq.Int64Array
 	for ; rows.Next(); i++ {
-		var stateBlockNID types.StateBlockNID
-		var result pq.Int64Array
 		if err = rows.Scan(&stateBlockNID, &result); err != nil {
 			return nil, err
 		}
-		r := []types.EventNID{}
-		for _, e := range result {
-			r = append(r, types.EventNID(e))
+		r := make([]types.EventNID, len(result))
+		for x := range result {
+			r[x] = types.EventNID(result[x])
 		}
 		results[i] = r
 	}
@@ -141,35 +140,3 @@ func stateBlockNIDsAsArray(stateBlockNIDs []types.StateBlockNID) pq.Int64Array {
 	}
 	return pq.Int64Array(nids)
 }
-
-type stateKeyTupleSorter []types.StateKeyTuple
-
-func (s stateKeyTupleSorter) Len() int           { return len(s) }
-func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
-func (s stateKeyTupleSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-
-// Check whether a tuple is in the list. Assumes that the list is sorted.
-func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool {
-	i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
-	return i < len(s) && s[i] == value
-}
-
-// List the unique eventTypeNIDs and eventStateKeyNIDs.
-// Assumes that the list is sorted.
-func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) {
-	eventTypeNIDs = make(pq.Int64Array, len(s))
-	eventStateKeyNIDs = make(pq.Int64Array, len(s))
-	for i := range s {
-		eventTypeNIDs[i] = int64(s[i].EventTypeNID)
-		eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
-	}
-	eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
-	eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
-	return
-}
-
-type int64Sorter []int64
-
-func (s int64Sorter) Len() int           { return len(s) }
-func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
-func (s int64Sorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
diff --git a/roomserver/storage/postgres/state_block_table_test.go b/roomserver/storage/postgres/state_block_table_test.go
deleted file mode 100644
index a0e2ec952..000000000
--- a/roomserver/storage/postgres/state_block_table_test.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"sort"
-	"testing"
-
-	"github.com/matrix-org/dendrite/roomserver/types"
-)
-
-func TestStateKeyTupleSorter(t *testing.T) {
-	input := stateKeyTupleSorter{
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-	}
-	want := []types.StateKeyTuple{
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-	}
-	doNotWant := []types.StateKeyTuple{
-		{EventTypeNID: 0, EventStateKeyNID: 0},
-		{EventTypeNID: 1, EventStateKeyNID: 3},
-		{EventTypeNID: 2, EventStateKeyNID: 1},
-		{EventTypeNID: 3, EventStateKeyNID: 1},
-	}
-	wantTypeNIDs := []int64{1, 2}
-	wantStateKeyNIDs := []int64{1, 2, 4}
-
-	// Sort the input and check it's in the right order.
-	sort.Sort(input)
-	gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays()
-
-	for i := range want {
-		if input[i] != want[i] {
-			t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
-		}
-
-		if !input.contains(want[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
-		}
-	}
-
-	for i := range doNotWant {
-		if input.contains(doNotWant[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
-		}
-	}
-
-	if len(wantTypeNIDs) != len(gotTypeNIDs) {
-		t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-	}
-
-	for i := range wantTypeNIDs {
-		if wantTypeNIDs[i] != gotTypeNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-
-	if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
-		t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
-	}
-
-	for i := range wantStateKeyNIDs {
-		if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-}
diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go
index 8ed886030..a24b7f3f0 100644
--- a/roomserver/storage/postgres/state_snapshot_table.go
+++ b/roomserver/storage/postgres/state_snapshot_table.go
@@ -77,12 +77,12 @@ type stateSnapshotStatements struct {
 	bulkSelectStateBlockNIDsStmt *sql.Stmt
 }
 
-func createStateSnapshotTable(db *sql.DB) error {
+func CreateStateSnapshotTable(db *sql.DB) error {
 	_, err := db.Exec(stateSnapshotSchema)
 	return err
 }
 
-func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
+func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
 	s := &stateSnapshotStatements{}
 
 	return s, sqlutil.StatementList{
@@ -95,12 +95,10 @@ func (s *stateSnapshotStatements) InsertState(
 	ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, nids types.StateBlockNIDs,
 ) (stateNID types.StateSnapshotNID, err error) {
 	nids = nids[:util.SortAndUnique(nids)]
-	var id int64
-	err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&id)
+	err = sqlutil.TxStmt(txn, s.insertStateStmt).QueryRowContext(ctx, nids.Hash(), int64(roomNID), stateBlockNIDsAsArray(nids)).Scan(&stateNID)
 	if err != nil {
 		return 0, err
 	}
-	stateNID = types.StateSnapshotNID(id)
 	return
 }
 
@@ -119,9 +117,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 	defer rows.Close() // nolint: errcheck
 	results := make([]types.StateBlockNIDList, len(stateNIDs))
 	i := 0
+	var stateBlockNIDs pq.Int64Array
 	for ; rows.Next(); i++ {
 		result := &results[i]
-		var stateBlockNIDs pq.Int64Array
 		if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index 88df72009..70ea4d8ba 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -80,19 +80,19 @@ func (d *Database) create(db *sql.DB) error {
 	if err := CreateEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomsTable(db); err != nil {
+	if err := CreateRoomsTable(db); err != nil {
 		return err
 	}
-	if err := createStateBlockTable(db); err != nil {
+	if err := CreateStateBlockTable(db); err != nil {
 		return err
 	}
-	if err := createStateSnapshotTable(db); err != nil {
+	if err := CreateStateSnapshotTable(db); err != nil {
 		return err
 	}
 	if err := CreatePrevEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomAliasesTable(db); err != nil {
+	if err := CreateRoomAliasesTable(db); err != nil {
 		return err
 	}
 	if err := CreateInvitesTable(db); err != nil {
@@ -128,15 +128,15 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	rooms, err := prepareRoomsTable(db)
+	rooms, err := PrepareRoomsTable(db)
 	if err != nil {
 		return err
 	}
-	stateBlock, err := prepareStateBlockTable(db)
+	stateBlock, err := PrepareStateBlockTable(db)
 	if err != nil {
 		return err
 	}
-	stateSnapshot, err := prepareStateSnapshotTable(db)
+	stateSnapshot, err := PrepareStateSnapshotTable(db)
 	if err != nil {
 		return err
 	}
@@ -144,7 +144,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	roomAliases, err := prepareRoomAliasesTable(db)
+	roomAliases, err := PrepareRoomAliasesTable(db)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 252e94c7e..cc4a9fff5 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1216,7 +1216,7 @@ func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString strin
 
 // GetKnownRooms returns a list of all rooms we know about.
 func (d *Database) GetKnownRooms(ctx context.Context) ([]string, error) {
-	return d.RoomsTable.SelectRoomIDs(ctx, nil)
+	return d.RoomsTable.SelectRoomIDsWithEvents(ctx, nil)
 }
 
 // ForgetRoom sets a users room to forgotten
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index feb06150a..1dda34c36 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -247,9 +247,9 @@ func (s *eventStatements) BulkSelectStateEventByNID(
 	ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID,
 	stateKeyTuples []types.StateKeyTuple,
 ) ([]types.StateEntry, error) {
-	tuples := stateKeyTupleSorter(stateKeyTuples)
+	tuples := types.StateKeyTupleSorter(stateKeyTuples)
 	sort.Sort(tuples)
-	eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays()
+	eventTypeNIDArray, eventStateKeyNIDArray := tuples.TypesAndStateKeysAsArrays()
 	params := make([]interface{}, 0, len(eventNIDs)+len(eventTypeNIDArray)+len(eventStateKeyNIDArray))
 	selectOrig := strings.Replace(bulkSelectStateEventByNIDSQL, "($1)", sqlutil.QueryVariadic(len(eventNIDs)), 1)
 	for _, v := range eventNIDs {
diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go
index 7c7bead95..3bdbbaa35 100644
--- a/roomserver/storage/sqlite3/room_aliases_table.go
+++ b/roomserver/storage/sqlite3/room_aliases_table.go
@@ -63,12 +63,12 @@ type roomAliasesStatements struct {
 	deleteRoomAliasStmt          *sql.Stmt
 }
 
-func createRoomAliasesTable(db *sql.DB) error {
+func CreateRoomAliasesTable(db *sql.DB) error {
 	_, err := db.Exec(roomAliasesSchema)
 	return err
 }
 
-func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
+func PrepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) {
 	s := &roomAliasesStatements{
 		db: db,
 	}
@@ -113,8 +113,8 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID(
 
 	defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed")
 
+	var alias string
 	for rows.Next() {
-		var alias string
 		if err = rows.Scan(&alias); err != nil {
 			return
 		}
diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go
index cd60c6785..03ad4b3d0 100644
--- a/roomserver/storage/sqlite3/rooms_table.go
+++ b/roomserver/storage/sqlite3/rooms_table.go
@@ -86,12 +86,12 @@ type roomStatements struct {
 	selectRoomIDsStmt  *sql.Stmt
 }
 
-func createRoomsTable(db *sql.DB) error {
+func CreateRoomsTable(db *sql.DB) error {
 	_, err := db.Exec(roomsSchema)
 	return err
 }
 
-func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
+func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	s := &roomStatements{
 		db: db,
 	}
@@ -108,7 +108,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) {
 	}.Prepare(db)
 }
 
-func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error) {
+func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt)
 	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
@@ -116,8 +116,8 @@ func (s *roomStatements) SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]stri
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -241,9 +241,9 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomVersionsForRoomNIDsStmt: rows.close() failed")
 	result := make(map[types.RoomNID]gomatrixserverlib.RoomVersion)
+	var roomNID types.RoomNID
+	var roomVersion gomatrixserverlib.RoomVersion
 	for rows.Next() {
-		var roomNID types.RoomNID
-		var roomVersion gomatrixserverlib.RoomVersion
 		if err = rows.Scan(&roomNID, &roomVersion); err != nil {
 			return nil, err
 		}
@@ -270,8 +270,8 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomIDsStmt: rows.close() failed")
 	var roomIDs []string
+	var roomID string
 	for rows.Next() {
-		var roomID string
 		if err = rows.Scan(&roomID); err != nil {
 			return nil, err
 		}
@@ -298,8 +298,8 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectRoomNIDsStmt: rows.close() failed")
 	var roomNIDs []types.RoomNID
+	var roomNID types.RoomNID
 	for rows.Next() {
-		var roomNID types.RoomNID
 		if err = rows.Scan(&roomNID); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go
index 3c829cdcd..4e67d4da1 100644
--- a/roomserver/storage/sqlite3/state_block_table.go
+++ b/roomserver/storage/sqlite3/state_block_table.go
@@ -20,7 +20,6 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"sort"
 	"strings"
 
 	"github.com/matrix-org/dendrite/internal"
@@ -64,12 +63,12 @@ type stateBlockStatements struct {
 	bulkSelectStateBlockEntriesStmt *sql.Stmt
 }
 
-func createStateBlockTable(db *sql.DB) error {
+func CreateStateBlockTable(db *sql.DB) error {
 	_, err := db.Exec(stateDataSchema)
 	return err
 }
 
-func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
+func PrepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) {
 	s := &stateBlockStatements{
 		db: db,
 	}
@@ -85,9 +84,9 @@ func (s *stateBlockStatements) BulkInsertStateData(
 	entries types.StateEntries,
 ) (id types.StateBlockNID, err error) {
 	entries = entries[:util.SortAndUnique(entries)]
-	nids := types.EventNIDs{} // zero slice to not store 'null' in the DB
-	for _, e := range entries {
-		nids = append(nids, e.EventNID)
+	nids := make(types.EventNIDs, entries.Len())
+	for i := range entries {
+		nids[i] = entries[i].EventNID
 	}
 	js, err := json.Marshal(nids)
 	if err != nil {
@@ -122,13 +121,13 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
 
 	results := make([][]types.EventNID, len(stateBlockNIDs))
 	i := 0
+	var stateBlockNID types.StateBlockNID
+	var result json.RawMessage
 	for ; rows.Next(); i++ {
-		var stateBlockNID types.StateBlockNID
-		var result json.RawMessage
 		if err = rows.Scan(&stateBlockNID, &result); err != nil {
 			return nil, err
 		}
-		r := []types.EventNID{}
+		var r []types.EventNID
 		if err = json.Unmarshal(result, &r); err != nil {
 			return nil, fmt.Errorf("json.Unmarshal: %w", err)
 		}
@@ -142,35 +141,3 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries(
 	}
 	return results, err
 }
-
-type stateKeyTupleSorter []types.StateKeyTuple
-
-func (s stateKeyTupleSorter) Len() int           { return len(s) }
-func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
-func (s stateKeyTupleSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-
-// Check whether a tuple is in the list. Assumes that the list is sorted.
-func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool {
-	i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
-	return i < len(s) && s[i] == value
-}
-
-// List the unique eventTypeNIDs and eventStateKeyNIDs.
-// Assumes that the list is sorted.
-func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) {
-	eventTypeNIDs = make([]int64, len(s))
-	eventStateKeyNIDs = make([]int64, len(s))
-	for i := range s {
-		eventTypeNIDs[i] = int64(s[i].EventTypeNID)
-		eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
-	}
-	eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
-	eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
-	return
-}
-
-type int64Sorter []int64
-
-func (s int64Sorter) Len() int           { return len(s) }
-func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
-func (s int64Sorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
diff --git a/roomserver/storage/sqlite3/state_block_table_test.go b/roomserver/storage/sqlite3/state_block_table_test.go
deleted file mode 100644
index 98439f5c0..000000000
--- a/roomserver/storage/sqlite3/state_block_table_test.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright 2017-2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"sort"
-	"testing"
-
-	"github.com/matrix-org/dendrite/roomserver/types"
-)
-
-func TestStateKeyTupleSorter(t *testing.T) {
-	input := stateKeyTupleSorter{
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-	}
-	want := []types.StateKeyTuple{
-		{EventTypeNID: 1, EventStateKeyNID: 1},
-		{EventTypeNID: 1, EventStateKeyNID: 2},
-		{EventTypeNID: 1, EventStateKeyNID: 4},
-		{EventTypeNID: 2, EventStateKeyNID: 2},
-	}
-	doNotWant := []types.StateKeyTuple{
-		{EventTypeNID: 0, EventStateKeyNID: 0},
-		{EventTypeNID: 1, EventStateKeyNID: 3},
-		{EventTypeNID: 2, EventStateKeyNID: 1},
-		{EventTypeNID: 3, EventStateKeyNID: 1},
-	}
-	wantTypeNIDs := []int64{1, 2}
-	wantStateKeyNIDs := []int64{1, 2, 4}
-
-	// Sort the input and check it's in the right order.
-	sort.Sort(input)
-	gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays()
-
-	for i := range want {
-		if input[i] != want[i] {
-			t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
-		}
-
-		if !input.contains(want[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
-		}
-	}
-
-	for i := range doNotWant {
-		if input.contains(doNotWant[i]) {
-			t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
-		}
-	}
-
-	if len(wantTypeNIDs) != len(gotTypeNIDs) {
-		t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-	}
-
-	for i := range wantTypeNIDs {
-		if wantTypeNIDs[i] != gotTypeNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-
-	if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
-		t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
-	}
-
-	for i := range wantStateKeyNIDs {
-		if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
-			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
-		}
-	}
-}
diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go
index 1f5e9ee3b..b8136b758 100644
--- a/roomserver/storage/sqlite3/state_snapshot_table.go
+++ b/roomserver/storage/sqlite3/state_snapshot_table.go
@@ -68,12 +68,12 @@ type stateSnapshotStatements struct {
 	bulkSelectStateBlockNIDsStmt *sql.Stmt
 }
 
-func createStateSnapshotTable(db *sql.DB) error {
+func CreateStateSnapshotTable(db *sql.DB) error {
 	_, err := db.Exec(stateSnapshotSchema)
 	return err
 }
 
-func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
+func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
 	s := &stateSnapshotStatements{
 		db: db,
 	}
@@ -96,12 +96,10 @@ func (s *stateSnapshotStatements) InsertState(
 		return
 	}
 	insertStmt := sqlutil.TxStmt(txn, s.insertStateStmt)
-	var id int64
-	err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&id)
+	err = insertStmt.QueryRowContext(ctx, stateBlockNIDs.Hash(), int64(roomNID), string(stateBlockNIDsJSON)).Scan(&stateNID)
 	if err != nil {
 		return 0, err
 	}
-	stateNID = types.StateSnapshotNID(id)
 	return
 }
 
@@ -127,9 +125,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed")
 	results := make([]types.StateBlockNIDList, len(stateNIDs))
 	i := 0
+	var stateBlockNIDsJSON string
 	for ; rows.Next(); i++ {
 		result := &results[i]
-		var stateBlockNIDsJSON string
 		if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil {
 			return nil, err
 		}
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index a4e32d528..8325fdad5 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -89,19 +89,19 @@ func (d *Database) create(db *sql.DB) error {
 	if err := CreateEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomsTable(db); err != nil {
+	if err := CreateRoomsTable(db); err != nil {
 		return err
 	}
-	if err := createStateBlockTable(db); err != nil {
+	if err := CreateStateBlockTable(db); err != nil {
 		return err
 	}
-	if err := createStateSnapshotTable(db); err != nil {
+	if err := CreateStateSnapshotTable(db); err != nil {
 		return err
 	}
 	if err := CreatePrevEventsTable(db); err != nil {
 		return err
 	}
-	if err := createRoomAliasesTable(db); err != nil {
+	if err := CreateRoomAliasesTable(db); err != nil {
 		return err
 	}
 	if err := CreateInvitesTable(db); err != nil {
@@ -137,15 +137,15 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	rooms, err := prepareRoomsTable(db)
+	rooms, err := PrepareRoomsTable(db)
 	if err != nil {
 		return err
 	}
-	stateBlock, err := prepareStateBlockTable(db)
+	stateBlock, err := PrepareStateBlockTable(db)
 	if err != nil {
 		return err
 	}
-	stateSnapshot, err := prepareStateSnapshotTable(db)
+	stateSnapshot, err := PrepareStateSnapshotTable(db)
 	if err != nil {
 		return err
 	}
@@ -153,7 +153,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room
 	if err != nil {
 		return err
 	}
-	roomAliases, err := prepareRoomAliasesTable(db)
+	roomAliases, err := PrepareRoomAliasesTable(db)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 95609787a..116e11c4e 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -72,7 +72,7 @@ type Rooms interface {
 	UpdateLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventNIDs []types.EventNID, lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID) error
 	SelectRoomVersionsForRoomNIDs(ctx context.Context, txn *sql.Tx, roomNID []types.RoomNID) (map[types.RoomNID]gomatrixserverlib.RoomVersion, error)
 	SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error)
-	SelectRoomIDs(ctx context.Context, txn *sql.Tx) ([]string, error)
+	SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error)
 	BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error)
 	BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error)
 }
diff --git a/roomserver/storage/tables/room_aliases_table_test.go b/roomserver/storage/tables/room_aliases_table_test.go
new file mode 100644
index 000000000..8fb57d5a4
--- /dev/null
+++ b/roomserver/storage/tables/room_aliases_table_test.go
@@ -0,0 +1,96 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateRoomAliasesTable(t *testing.T, dbType test.DBType) (tab tables.RoomAliases, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateRoomAliasesTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareRoomAliasesTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateRoomAliasesTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareRoomAliasesTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestRoomAliasesTable(t *testing.T) {
+	alice := test.NewUser()
+	room := test.NewRoom(t, alice)
+	room2 := test.NewRoom(t, alice)
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateRoomAliasesTable(t, dbType)
+		defer close()
+		alias, alias2, alias3 := "#alias:localhost", "#alias2:localhost", "#alias3:localhost"
+		// insert aliases
+		err := tab.InsertRoomAlias(ctx, nil, alias, room.ID, alice.ID)
+		assert.NoError(t, err)
+
+		err = tab.InsertRoomAlias(ctx, nil, alias2, room.ID, alice.ID)
+		assert.NoError(t, err)
+
+		err = tab.InsertRoomAlias(ctx, nil, alias3, room2.ID, alice.ID)
+		assert.NoError(t, err)
+
+		// verify we can get the roomID for the alias
+		roomID, err := tab.SelectRoomIDFromAlias(ctx, nil, alias)
+		assert.NoError(t, err)
+		assert.Equal(t, room.ID, roomID)
+
+		// .. and the creator
+		creator, err := tab.SelectCreatorIDFromAlias(ctx, nil, alias)
+		assert.NoError(t, err)
+		assert.Equal(t, alice.ID, creator)
+
+		creator, err = tab.SelectCreatorIDFromAlias(ctx, nil, "#doesntexist:localhost")
+		assert.NoError(t, err)
+		assert.Equal(t, "", creator)
+
+		roomID, err = tab.SelectRoomIDFromAlias(ctx, nil, "#doesntexist:localhost")
+		assert.NoError(t, err)
+		assert.Equal(t, "", roomID)
+
+		// get all aliases for a room
+		aliases, err := tab.SelectAliasesFromRoomID(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{alias, alias2}, aliases)
+
+		// delete an alias and verify it's deleted
+		err = tab.DeleteRoomAlias(ctx, nil, alias2)
+		assert.NoError(t, err)
+
+		aliases, err = tab.SelectAliasesFromRoomID(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, []string{alias}, aliases)
+
+		// deleting the same alias should be a no-op
+		err = tab.DeleteRoomAlias(ctx, nil, alias2)
+		assert.NoError(t, err)
+
+		// Delete non-existent alias should be a no-op
+		err = tab.DeleteRoomAlias(ctx, nil, "#doesntexist:localhost")
+		assert.NoError(t, err)
+	})
+}
diff --git a/roomserver/storage/tables/rooms_table_test.go b/roomserver/storage/tables/rooms_table_test.go
new file mode 100644
index 000000000..9872fb800
--- /dev/null
+++ b/roomserver/storage/tables/rooms_table_test.go
@@ -0,0 +1,128 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateRoomsTable(t *testing.T, dbType test.DBType) (tab tables.Rooms, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateRoomsTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareRoomsTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateRoomsTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareRoomsTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestRoomsTable(t *testing.T) {
+	alice := test.NewUser()
+	room := test.NewRoom(t, alice)
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateRoomsTable(t, dbType)
+		defer close()
+
+		wantRoomNID, err := tab.InsertRoomNID(ctx, nil, room.ID, room.Version)
+		assert.NoError(t, err)
+
+		// Create dummy room
+		_, err = tab.InsertRoomNID(ctx, nil, util.RandomString(16), room.Version)
+		assert.NoError(t, err)
+
+		gotRoomNID, err := tab.SelectRoomNID(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, wantRoomNID, gotRoomNID)
+
+		// Ensure non existent roomNID errors
+		roomNID, err := tab.SelectRoomNID(ctx, nil, "!doesnotexist:localhost")
+		assert.Error(t, err)
+		assert.Equal(t, types.RoomNID(0), roomNID)
+
+		roomInfo, err := tab.SelectRoomInfo(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, &types.RoomInfo{
+			RoomNID:          wantRoomNID,
+			RoomVersion:      room.Version,
+			StateSnapshotNID: 0,
+			IsStub:           true, // there are no latestEventNIDs
+		}, roomInfo)
+
+		roomInfo, err = tab.SelectRoomInfo(ctx, nil, "!doesnotexist:localhost")
+		assert.NoError(t, err)
+		assert.Nil(t, roomInfo)
+
+		// There are no rooms with latestEventNIDs yet
+		roomIDs, err := tab.SelectRoomIDsWithEvents(ctx, nil)
+		assert.NoError(t, err)
+		assert.Equal(t, 0, len(roomIDs))
+
+		roomVersions, err := tab.SelectRoomVersionsForRoomNIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
+		assert.NoError(t, err)
+		assert.Equal(t, roomVersions[wantRoomNID], room.Version)
+		// Room does not exist
+		_, ok := roomVersions[1337]
+		assert.False(t, ok)
+
+		roomIDs, err = tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337})
+		assert.NoError(t, err)
+		assert.Equal(t, []string{room.ID}, roomIDs)
+
+		roomNIDs, err := tab.BulkSelectRoomNIDs(ctx, nil, []string{room.ID, "!doesnotexist:localhost"})
+		assert.NoError(t, err)
+		assert.Equal(t, []types.RoomNID{wantRoomNID}, roomNIDs)
+
+		wantEventNIDs := []types.EventNID{1, 2, 3}
+		lastEventSentNID := types.EventNID(3)
+		stateSnapshotNID := types.StateSnapshotNID(1)
+		// make the room "usable"
+		err = tab.UpdateLatestEventNIDs(ctx, nil, wantRoomNID, wantEventNIDs, lastEventSentNID, stateSnapshotNID)
+		assert.NoError(t, err)
+
+		roomInfo, err = tab.SelectRoomInfo(ctx, nil, room.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, &types.RoomInfo{
+			RoomNID:          wantRoomNID,
+			RoomVersion:      room.Version,
+			StateSnapshotNID: 1,
+			IsStub:           false,
+		}, roomInfo)
+
+		eventNIDs, snapshotNID, err := tab.SelectLatestEventNIDs(ctx, nil, wantRoomNID)
+		assert.NoError(t, err)
+		assert.Equal(t, wantEventNIDs, eventNIDs)
+		assert.Equal(t, types.StateSnapshotNID(1), snapshotNID)
+
+		// Again, doesn't exist
+		_, _, err = tab.SelectLatestEventNIDs(ctx, nil, 1337)
+		assert.Error(t, err)
+
+		eventNIDs, eventNID, snapshotNID, err := tab.SelectLatestEventsNIDsForUpdate(ctx, nil, wantRoomNID)
+		assert.NoError(t, err)
+		assert.Equal(t, wantEventNIDs, eventNIDs)
+		assert.Equal(t, types.EventNID(3), eventNID)
+		assert.Equal(t, types.StateSnapshotNID(1), snapshotNID)
+	})
+}
diff --git a/roomserver/storage/tables/state_block_table_test.go b/roomserver/storage/tables/state_block_table_test.go
new file mode 100644
index 000000000..de0b420bc
--- /dev/null
+++ b/roomserver/storage/tables/state_block_table_test.go
@@ -0,0 +1,92 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateStateBlockTable(t *testing.T, dbType test.DBType) (tab tables.StateBlock, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateStateBlockTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareStateBlockTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateStateBlockTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareStateBlockTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestStateBlockTable(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateStateBlockTable(t, dbType)
+		defer close()
+
+		// generate some dummy data
+		var entries types.StateEntries
+		for i := 0; i < 100; i++ {
+			entry := types.StateEntry{
+				EventNID: types.EventNID(i),
+			}
+			entries = append(entries, entry)
+		}
+		stateBlockNID, err := tab.BulkInsertStateData(ctx, nil, entries)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateBlockNID(1), stateBlockNID)
+
+		// generate a different hash, to get a new StateBlockNID
+		var entries2 types.StateEntries
+		for i := 100; i < 300; i++ {
+			entry := types.StateEntry{
+				EventNID: types.EventNID(i),
+			}
+			entries2 = append(entries2, entry)
+		}
+		stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateBlockNID(2), stateBlockNID)
+
+		eventNIDs, err := tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 2})
+		assert.NoError(t, err)
+		assert.Equal(t, len(entries), len(eventNIDs[0]))
+		assert.Equal(t, len(entries2), len(eventNIDs[1]))
+
+		// try to get a StateBlockNID which does not exist
+		_, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{5})
+		assert.Error(t, err)
+
+		// This should return an error, since we can only retrieve 1 StateBlock
+		_, err = tab.BulkSelectStateBlockEntries(ctx, nil, types.StateBlockNIDs{1, 5})
+		assert.Error(t, err)
+
+		for i := 0; i < 65555; i++ {
+			entry := types.StateEntry{
+				EventNID: types.EventNID(i),
+			}
+			entries2 = append(entries2, entry)
+		}
+		stateBlockNID, err = tab.BulkInsertStateData(ctx, nil, entries2)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateBlockNID(3), stateBlockNID)
+	})
+}
diff --git a/roomserver/storage/tables/state_snapshot_table_test.go b/roomserver/storage/tables/state_snapshot_table_test.go
new file mode 100644
index 000000000..dcdb5d8f1
--- /dev/null
+++ b/roomserver/storage/tables/state_snapshot_table_test.go
@@ -0,0 +1,86 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateStateSnapshotTable(t *testing.T, dbType test.DBType) (tab tables.StateSnapshot, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	switch dbType {
+	case test.DBTypePostgres:
+		err = postgres.CreateStateSnapshotTable(db)
+		assert.NoError(t, err)
+		tab, err = postgres.PrepareStateSnapshotTable(db)
+	case test.DBTypeSQLite:
+		err = sqlite3.CreateStateSnapshotTable(db)
+		assert.NoError(t, err)
+		tab, err = sqlite3.PrepareStateSnapshotTable(db)
+	}
+	assert.NoError(t, err)
+
+	return tab, close
+}
+
+func TestStateSnapshotTable(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, close := mustCreateStateSnapshotTable(t, dbType)
+		defer close()
+
+		// generate some dummy data
+		var stateBlockNIDs types.StateBlockNIDs
+		for i := 0; i < 100; i++ {
+			stateBlockNIDs = append(stateBlockNIDs, types.StateBlockNID(i))
+		}
+		stateNID, err := tab.InsertState(ctx, nil, 1, stateBlockNIDs)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateSnapshotNID(1), stateNID)
+
+		// verify ON CONFLICT; Note: this updates the sequence!
+		stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs)
+		assert.NoError(t, err)
+		assert.Equal(t, types.StateSnapshotNID(1), stateNID)
+
+		// create a second snapshot
+		var stateBlockNIDs2 types.StateBlockNIDs
+		for i := 100; i < 150; i++ {
+			stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i))
+		}
+
+		stateNID, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2)
+		assert.NoError(t, err)
+		// StateSnapshotNID is now 3, since the DO UPDATE SET statement incremented the sequence
+		assert.Equal(t, types.StateSnapshotNID(3), stateNID)
+
+		nidLists, err := tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{1, 3})
+		assert.NoError(t, err)
+		assert.Equal(t, stateBlockNIDs, types.StateBlockNIDs(nidLists[0].StateBlockNIDs))
+		assert.Equal(t, stateBlockNIDs2, types.StateBlockNIDs(nidLists[1].StateBlockNIDs))
+
+		// check we get an error if the state snapshot does not exist
+		_, err = tab.BulkSelectStateBlockNIDs(ctx, nil, []types.StateSnapshotNID{2})
+		assert.Error(t, err)
+
+		// create a second snapshot
+		for i := 0; i < 65555; i++ {
+			stateBlockNIDs2 = append(stateBlockNIDs2, types.StateBlockNID(i))
+		}
+		_, err = tab.InsertState(ctx, nil, 1, stateBlockNIDs2)
+		assert.NoError(t, err)
+	})
+}
diff --git a/roomserver/types/types.go b/roomserver/types/types.go
index ce4e5fd1e..62695aaee 100644
--- a/roomserver/types/types.go
+++ b/roomserver/types/types.go
@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 	"golang.org/x/crypto/blake2b"
 )
 
@@ -97,6 +98,38 @@ func (a StateKeyTuple) LessThan(b StateKeyTuple) bool {
 	return a.EventStateKeyNID < b.EventStateKeyNID
 }
 
+type StateKeyTupleSorter []StateKeyTuple
+
+func (s StateKeyTupleSorter) Len() int           { return len(s) }
+func (s StateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) }
+func (s StateKeyTupleSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+// Check whether a tuple is in the list. Assumes that the list is sorted.
+func (s StateKeyTupleSorter) contains(value StateKeyTuple) bool {
+	i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) })
+	return i < len(s) && s[i] == value
+}
+
+// List the unique eventTypeNIDs and eventStateKeyNIDs.
+// Assumes that the list is sorted.
+func (s StateKeyTupleSorter) TypesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) {
+	eventTypeNIDs = make([]int64, len(s))
+	eventStateKeyNIDs = make([]int64, len(s))
+	for i := range s {
+		eventTypeNIDs[i] = int64(s[i].EventTypeNID)
+		eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID)
+	}
+	eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))]
+	eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))]
+	return
+}
+
+type int64Sorter []int64
+
+func (s int64Sorter) Len() int           { return len(s) }
+func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] }
+func (s int64Sorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
 // A StateEntry is an entry in the room state of a matrix room.
 type StateEntry struct {
 	StateKeyTuple
diff --git a/roomserver/types/types_test.go b/roomserver/types/types_test.go
index b1e84b821..a26b80f74 100644
--- a/roomserver/types/types_test.go
+++ b/roomserver/types/types_test.go
@@ -1,6 +1,7 @@
 package types
 
 import (
+	"sort"
 	"testing"
 )
 
@@ -24,3 +25,66 @@ func TestDeduplicateStateEntries(t *testing.T) {
 		}
 	}
 }
+
+func TestStateKeyTupleSorter(t *testing.T) {
+	input := StateKeyTupleSorter{
+		{EventTypeNID: 1, EventStateKeyNID: 2},
+		{EventTypeNID: 1, EventStateKeyNID: 4},
+		{EventTypeNID: 2, EventStateKeyNID: 2},
+		{EventTypeNID: 1, EventStateKeyNID: 1},
+	}
+	want := []StateKeyTuple{
+		{EventTypeNID: 1, EventStateKeyNID: 1},
+		{EventTypeNID: 1, EventStateKeyNID: 2},
+		{EventTypeNID: 1, EventStateKeyNID: 4},
+		{EventTypeNID: 2, EventStateKeyNID: 2},
+	}
+	doNotWant := []StateKeyTuple{
+		{EventTypeNID: 0, EventStateKeyNID: 0},
+		{EventTypeNID: 1, EventStateKeyNID: 3},
+		{EventTypeNID: 2, EventStateKeyNID: 1},
+		{EventTypeNID: 3, EventStateKeyNID: 1},
+	}
+	wantTypeNIDs := []int64{1, 2}
+	wantStateKeyNIDs := []int64{1, 2, 4}
+
+	// Sort the input and check it's in the right order.
+	sort.Sort(input)
+	gotTypeNIDs, gotStateKeyNIDs := input.TypesAndStateKeysAsArrays()
+
+	for i := range want {
+		if input[i] != want[i] {
+			t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i])
+		}
+
+		if !input.contains(want[i]) {
+			t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i])
+		}
+	}
+
+	for i := range doNotWant {
+		if input.contains(doNotWant[i]) {
+			t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i])
+		}
+	}
+
+	if len(wantTypeNIDs) != len(gotTypeNIDs) {
+		t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
+	}
+
+	for i := range wantTypeNIDs {
+		if wantTypeNIDs[i] != gotTypeNIDs[i] {
+			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
+		}
+	}
+
+	if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) {
+		t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs)
+	}
+
+	for i := range wantStateKeyNIDs {
+		if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] {
+			t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs)
+		}
+	}
+}

From cd82460513d5abf04e56c01667d56499d4c354be Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Tue, 17 May 2022 10:45:50 +0100
Subject: [PATCH 117/391] Add docs which explain how to calculate coverage
 (#2468)

---
 docs/coverage.md | 84 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 docs/coverage.md

diff --git a/docs/coverage.md b/docs/coverage.md
new file mode 100644
index 000000000..7a3b7cb9e
--- /dev/null
+++ b/docs/coverage.md
@@ -0,0 +1,84 @@
+---
+title: Coverage
+parent: Development
+permalink: /development/coverage
+---
+
+To generate a test coverage report for Sytest, a small patch needs to be applied to the Sytest repository to compile and use the instrumented binary:
+```patch
+diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm
+index 8f0e209c..ad057e52 100644
+--- a/lib/SyTest/Homeserver/Dendrite.pm
++++ b/lib/SyTest/Homeserver/Dendrite.pm
+@@ -337,7 +337,7 @@ sub _start_monolith
+ 
+    $output->diag( "Starting monolith server" );
+    my @command = (
+-      $self->{bindir} . '/dendrite-monolith-server',
++      $self->{bindir} . '/dendrite-monolith-server', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL",
+       '--config', $self->{paths}{config},
+       '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port,
+       '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port,
+diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
+index f009332b..7ea79869 100755
+--- a/scripts/dendrite_sytest.sh
++++ b/scripts/dendrite_sytest.sh
+@@ -34,7 +34,8 @@ export GOBIN=/tmp/bin
+ echo >&2 "--- Building dendrite from source"
+ cd /src
+ mkdir -p $GOBIN
+-go install -v ./cmd/dendrite-monolith-server
++# go install -v ./cmd/dendrite-monolith-server
++go test -c -cover -covermode=atomic -o $GOBIN/dendrite-monolith-server -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server
+ go install -v ./cmd/generate-keys
+ cd -
+ ```
+
+ Then run Sytest. This will generate a new file `integrationcover.log` in each server's directory e.g `server-0/integrationcover.log`. To parse it,
+ ensure your working directory is under the Dendrite repository then run:
+ ```bash
+ go tool cover -func=/path/to/server-0/integrationcover.log
+ ```
+ which will produce an output like:
+ ```
+ ...
+ github.com/matrix-org/util/json.go:83:											NewJSONRequestHandler				100.0%
+github.com/matrix-org/util/json.go:90:											Protect						57.1%
+github.com/matrix-org/util/json.go:110:											RequestWithLogging				100.0%
+github.com/matrix-org/util/json.go:132:											MakeJSONAPI					70.0%
+github.com/matrix-org/util/json.go:151:											respond						61.5%
+github.com/matrix-org/util/json.go:180:											WithCORSOptions					0.0%
+github.com/matrix-org/util/json.go:191:											SetCORSHeaders					100.0%
+github.com/matrix-org/util/json.go:202:											RandomString					100.0%
+github.com/matrix-org/util/json.go:210:											init						100.0%
+github.com/matrix-org/util/unique.go:13:										Unique						91.7%
+github.com/matrix-org/util/unique.go:48:										SortAndUnique					100.0%
+github.com/matrix-org/util/unique.go:55:										UniqueStrings					100.0%
+total:															(statements)					53.7%
+```
+The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations,
+which will never be tested in a single test run (e.g sqlite or postgres, monolith or polylith). To get a more accurate value, additional processing is required
+to remove packages which will never be tested and extension MSCs:
+```bash
+# These commands are all similar but change which package paths are _removed_ from the output.
+
+# For Postgres (monolith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt
+
+# For Postgres (polylith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'sqlite|setup/mscs|api_trace' > coverage.txt
+
+# For SQLite (monolith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt
+
+# For SQLite (polylith)
+go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'postgres|setup/mscs|api_trace' > coverage.txt
+```
+
+A total value can then be calculated using:
+```bash
+cat coverage.txt | awk -F '\t+' '{x = x + $3} END {print x/NR}'
+```
+
+
+We currently do not have a way to combine Sytest/Complement/Unit Tests into a single coverage report.
\ No newline at end of file

From 6de29c1cd23d218f04d2e570932db8967d6adc4f Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Tue, 17 May 2022 13:23:35 +0100
Subject: [PATCH 118/391] bugfix: E2EE device keys could sometimes not be sent
 to remote servers (#2466)

* Fix flakey sytest 'Local device key changes get to remote servers'

* Debug logs

* Remove internal/test and use /test only

Remove a lot of ancient code too.

* Use FederationRoomserverAPI in more places

* Use more interfaces in federationapi; begin adding regression test

* Linting

* Add regression test

* Unbreak tests

* ALL THE LOGS

* Fix a race condition which could cause events to not be sent to servers

If a new room event which rewrites state arrives, we remove all joined hosts
then re-calculate them. This wasn't done in a transaction so for a brief period
we would have no joined hosts. During this interim, key change events which arrive
would not be sent to destination servers. This would sporadically fail on sytest.

* Unbreak new tests

* Linting
---
 cmd/generate-keys/main.go                     |   2 +-
 federationapi/api/api.go                      |  40 ++-
 federationapi/consumers/keychange.go          |   8 +-
 federationapi/consumers/roomserver.go         |  29 +--
 federationapi/federationapi.go                |   4 +-
 federationapi/federationapi_test.go           | 236 +++++++++++++++++-
 federationapi/internal/api.go                 |   8 +-
 federationapi/internal/perform.go             |  18 +-
 federationapi/queue/destinationqueue.go       |  31 +--
 federationapi/queue/queue.go                  |   9 +-
 federationapi/routing/query.go                |   2 +-
 federationapi/routing/routing.go              |   2 +-
 federationapi/routing/send.go                 |   2 +-
 federationapi/routing/send_test.go            |   2 +-
 federationapi/routing/threepid.go             |  16 +-
 federationapi/storage/interface.go            |   3 +-
 .../storage/postgres/joined_hosts_table.go    |   5 +
 federationapi/storage/shared/storage.go       |  26 +-
 internal/caching/cache_typing_test.go         |   2 +-
 internal/test/client.go                       | 158 ------------
 internal/test/kafka.go                        |  76 ------
 internal/test/server.go                       | 152 -----------
 keyserver/internal/device_list_update.go      |   4 +-
 keyserver/internal/internal.go                |   2 +-
 keyserver/keyserver.go                        |   2 +-
 roomserver/api/api.go                         |   1 +
 roomserver/internal/input/input_test.go       |   4 +-
 roomserver/internal/query/query_test.go       |   2 +-
 .../storage/tables/events_table_test.go       |   2 +-
 .../tables/previous_events_table_test.go      |   2 +-
 .../storage/tables/published_table_test.go    |   2 +-
 .../storage/tables/room_aliases_table_test.go |   2 +-
 roomserver/storage/tables/rooms_table_test.go |   2 +-
 syncapi/storage/storage_test.go               |   6 +-
 .../storage/tables/output_room_events_test.go |   2 +-
 syncapi/storage/tables/topology_test.go       |   2 +-
 syncapi/syncapi_test.go                       |  15 +-
 test/event.go                                 |  18 ++
 test/http.go                                  |  47 ++++
 {internal/test => test}/keyring.go            |   0
 internal/test/config.go => test/keys.go       |  84 -------
 test/room.go                                  |  66 +++--
 {internal/test => test}/slice.go              |   0
 test/{ => testrig}/base.go                    |  11 +-
 test/{ => testrig}/jetstream.go               |   2 +-
 test/user.go                                  |  52 +++-
 userapi/storage/storage_test.go               |  20 +-
 userapi/userapi_test.go                       |   3 +-
 48 files changed, 566 insertions(+), 618 deletions(-)
 delete mode 100644 internal/test/client.go
 delete mode 100644 internal/test/kafka.go
 delete mode 100644 internal/test/server.go
 rename {internal/test => test}/keyring.go (100%)
 rename internal/test/config.go => test/keys.go (61%)
 rename {internal/test => test}/slice.go (100%)
 rename test/{ => testrig}/base.go (92%)
 rename test/{ => testrig}/jetstream.go (98%)

diff --git a/cmd/generate-keys/main.go b/cmd/generate-keys/main.go
index bddf219dc..8acd28be0 100644
--- a/cmd/generate-keys/main.go
+++ b/cmd/generate-keys/main.go
@@ -20,7 +20,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/dendrite/test"
 )
 
 const usage = `Usage: %s
diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index fc25194e0..53d4701f3 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -12,12 +12,16 @@ import (
 
 // FederationInternalAPI is used to query information from the federation sender.
 type FederationInternalAPI interface {
-	FederationClient
+	gomatrixserverlib.FederatedStateClient
+	KeyserverFederationAPI
 	gomatrixserverlib.KeyDatabase
 	ClientFederationAPI
 	RoomserverFederationAPI
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
+	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
+	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
+	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
 
 	// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
 	PerformBroadcastEDU(
@@ -60,17 +64,43 @@ type RoomserverFederationAPI interface {
 	LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
-// FederationClient is a subset of gomatrixserverlib.FederationClient functions which the fedsender
+// KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver
 // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
 // this interface are of type FederationClientError
-type FederationClient interface {
-	gomatrixserverlib.FederatedStateClient
+type KeyserverFederationAPI interface {
 	GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
 	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
 	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
+}
+
+// an interface for gmsl.FederationClient - contains functions called by federationapi only.
+type FederationClient interface {
+	gomatrixserverlib.KeyClient
+	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
+
+	// Perform operations
+	LookupRoomAlias(ctx context.Context, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
+	Peek(ctx context.Context, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
+	MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
+	SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
+	MakeLeave(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
+	SendLeave(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
+	SendInviteV2(ctx context.Context, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
+
+	GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
+
+	GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
+	GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
+	ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
+	QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
+	Backfill(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
 	MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
 	MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
-	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
+
+	ExchangeThirdPartyInvite(ctx context.Context, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
+	LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
+	LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
+	LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
 // FederationClientError is returned from FederationClient methods in the event of a problem.
diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go
index 0ece18e97..95c9a7fdd 100644
--- a/federationapi/consumers/keychange.go
+++ b/federationapi/consumers/keychange.go
@@ -39,7 +39,7 @@ type KeyChangeConsumer struct {
 	db         storage.Database
 	queues     *queue.OutgoingQueues
 	serverName gomatrixserverlib.ServerName
-	rsAPI      roomserverAPI.RoomserverInternalAPI
+	rsAPI      roomserverAPI.FederationRoomserverAPI
 	topic      string
 }
 
@@ -50,7 +50,7 @@ func NewKeyChangeConsumer(
 	js nats.JetStreamContext,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
 ) *KeyChangeConsumer {
 	return &KeyChangeConsumer{
 		ctx:        process.Context(),
@@ -120,6 +120,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
+	logrus.Infof("DEBUG: %v joined rooms for user %v", queryRes.RoomIDs, m.UserID)
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
@@ -128,6 +129,9 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	}
 
 	if len(destinations) == 0 {
+		logger.WithField("num_rooms", len(queryRes.RoomIDs)).Debug("user is in no federated rooms")
+		destinations, err = t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, false)
+		logrus.Infof("GetJoinedHostsForRooms exclude self=false -> %v %v", destinations, err)
 		return true
 	}
 	// Pack the EDU and marshal it
diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index 80317ee69..7a0816ff2 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/federationapi/queue"
@@ -36,7 +37,7 @@ import (
 type OutputRoomEventConsumer struct {
 	ctx       context.Context
 	cfg       *config.FederationAPI
-	rsAPI     api.RoomserverInternalAPI
+	rsAPI     api.FederationRoomserverAPI
 	jetstream nats.JetStreamContext
 	durable   string
 	db        storage.Database
@@ -51,7 +52,7 @@ func NewOutputRoomEventConsumer(
 	js nats.JetStreamContext,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
-	rsAPI api.RoomserverInternalAPI,
+	rsAPI api.FederationRoomserverAPI,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
 		ctx:       process.Context(),
@@ -89,15 +90,7 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 	switch output.Type {
 	case api.OutputTypeNewRoomEvent:
 		ev := output.NewRoomEvent.Event
-
-		if output.NewRoomEvent.RewritesState {
-			if err := s.db.PurgeRoomState(s.ctx, ev.RoomID()); err != nil {
-				log.WithError(err).Errorf("roomserver output log: purge room state failure")
-				return false
-			}
-		}
-
-		if err := s.processMessage(*output.NewRoomEvent); err != nil {
+		if err := s.processMessage(*output.NewRoomEvent, output.NewRoomEvent.RewritesState); err != nil {
 			// panic rather than continue with an inconsistent database
 			log.WithFields(log.Fields{
 				"event_id":   ev.EventID(),
@@ -145,7 +138,7 @@ func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPee
 
 // processMessage updates the list of currently joined hosts in the room
 // and then sends the event to the hosts that were joined before the event.
-func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error {
+func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rewritesState bool) error {
 	addsStateEvents, missingEventIDs := ore.NeededStateEventIDs()
 
 	// Ask the roomserver and add in the rest of the results into the set.
@@ -164,7 +157,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err
 		addsStateEvents = append(addsStateEvents, eventsRes.Events...)
 	}
 
-	addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents))
+	addsJoinedHosts, err := JoinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(addsStateEvents))
 	if err != nil {
 		return err
 	}
@@ -173,13 +166,13 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err
 	// expressed as a delta against the current state.
 	// TODO(#290): handle EventIDMismatchError and recover the current state by
 	// talking to the roomserver
+	logrus.Infof("room %s adds joined hosts: %v removes %v", ore.Event.RoomID(), addsJoinedHosts, ore.RemovesStateEventIDs)
 	oldJoinedHosts, err := s.db.UpdateRoom(
 		s.ctx,
 		ore.Event.RoomID(),
-		ore.LastSentEventID,
-		ore.Event.EventID(),
 		addsJoinedHosts,
 		ore.RemovesStateEventIDs,
+		rewritesState, // if we're re-writing state, nuke all joined hosts before adding
 	)
 	if err != nil {
 		return err
@@ -238,7 +231,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 		return nil, err
 	}
 
-	combinedAddsJoinedHosts, err := joinedHostsFromEvents(combinedAddsEvents)
+	combinedAddsJoinedHosts, err := JoinedHostsFromEvents(combinedAddsEvents)
 	if err != nil {
 		return nil, err
 	}
@@ -284,10 +277,10 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 	return result, nil
 }
 
-// joinedHostsFromEvents turns a list of state events into a list of joined hosts.
+// JoinedHostsFromEvents turns a list of state events into a list of joined hosts.
 // This errors if one of the events was invalid.
 // It should be impossible for an invalid event to get this far in the pipeline.
-func joinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) {
+func JoinedHostsFromEvents(evs []*gomatrixserverlib.Event) ([]types.JoinedHost, error) {
 	var joinedHosts []types.JoinedHost
 	for _, ev := range evs {
 		if ev.Type() != "m.room.member" || ev.StateKey() == nil {
diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index bec9ac777..ff159beea 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -93,8 +93,8 @@ func AddPublicRoutes(
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite,
-	federation *gomatrixserverlib.FederationClient,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
+	federation api.FederationClient,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
 	caches *caching.Caches,
 	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index eedebc6cd..ae244c566 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -3,18 +3,250 @@ package federationapi_test
 import (
 	"context"
 	"crypto/ed25519"
+	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/internal"
-	"github.com/matrix-org/dendrite/internal/test"
+	keyapi "github.com/matrix-org/dendrite/keyserver/api"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
 )
 
+type fedRoomserverAPI struct {
+	rsapi.FederationRoomserverAPI
+	inputRoomEvents   func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse)
+	queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error
+}
+
+// PerformJoin will call this function
+func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+	if f.inputRoomEvents == nil {
+		return
+	}
+	f.inputRoomEvents(ctx, req, res)
+}
+
+// keychange consumer calls this
+func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
+	if f.queryRoomsForUser == nil {
+		return nil
+	}
+	return f.queryRoomsForUser(ctx, req, res)
+}
+
+// TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
+type fedClient struct {
+	api.FederationClient
+	allowJoins []*test.Room
+	keys       map[gomatrixserverlib.ServerName]struct {
+		key   ed25519.PrivateKey
+		keyID gomatrixserverlib.KeyID
+	}
+	t       *testing.T
+	sentTxn bool
+}
+
+func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) {
+	fmt.Println("GetServerKeys:", matrixServer)
+	var keys gomatrixserverlib.ServerKeys
+	var keyID gomatrixserverlib.KeyID
+	var pkey ed25519.PrivateKey
+	for srv, data := range f.keys {
+		if srv == matrixServer {
+			pkey = data.key
+			keyID = data.keyID
+			break
+		}
+	}
+	if pkey == nil {
+		return keys, nil
+	}
+
+	keys.ServerName = matrixServer
+	keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(10 * time.Hour))
+	publicKey := pkey.Public().(ed25519.PublicKey)
+	keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
+		keyID: {
+			Key: gomatrixserverlib.Base64Bytes(publicKey),
+		},
+	}
+	toSign, err := json.Marshal(keys.ServerKeyFields)
+	if err != nil {
+		return keys, err
+	}
+
+	keys.Raw, err = gomatrixserverlib.SignJSON(
+		string(matrixServer), keyID, pkey, toSign,
+	)
+	if err != nil {
+		return keys, err
+	}
+
+	return keys, nil
+}
+
+func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) {
+	for _, r := range f.allowJoins {
+		if r.ID == roomID {
+			res.RoomVersion = r.Version
+			res.JoinEvent = gomatrixserverlib.EventBuilder{
+				Sender:     userID,
+				RoomID:     roomID,
+				Type:       "m.room.member",
+				StateKey:   &userID,
+				Content:    gomatrixserverlib.RawJSON([]byte(`{"membership":"join"}`)),
+				PrevEvents: r.ForwardExtremities(),
+			}
+			var needed gomatrixserverlib.StateNeeded
+			needed, err = gomatrixserverlib.StateNeededForEventBuilder(&res.JoinEvent)
+			if err != nil {
+				f.t.Errorf("StateNeededForEventBuilder: %v", err)
+				return
+			}
+			res.JoinEvent.AuthEvents = r.MustGetAuthEventRefsForEvent(f.t, needed)
+			return
+		}
+	}
+	return
+}
+func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
+	for _, r := range f.allowJoins {
+		if r.ID == event.RoomID() {
+			r.InsertEvent(f.t, event.Headered(r.Version))
+			f.t.Logf("Join event: %v", event.EventID())
+			res.StateEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.CurrentState())
+			res.AuthEvents = gomatrixserverlib.NewEventJSONsFromHeaderedEvents(r.Events())
+		}
+	}
+	return
+}
+
+func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
+	for _, edu := range t.EDUs {
+		if edu.Type == gomatrixserverlib.MDeviceListUpdate {
+			f.sentTxn = true
+		}
+	}
+	f.t.Logf("got /send")
+	return
+}
+
+// Regression test to make sure that /send_join is updating the destination hosts synchronously and
+// isn't relying on the roomserver.
+func TestFederationAPIJoinThenKeyUpdate(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testFederationAPIJoinThenKeyUpdate(t, dbType)
+	})
+}
+
+func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
+	base, close := testrig.CreateBaseDendrite(t, dbType)
+	base.Cfg.FederationAPI.PreferDirectFetch = true
+	defer close()
+	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+
+	serverA := gomatrixserverlib.ServerName("server.a")
+	serverAKeyID := gomatrixserverlib.KeyID("ed25519:servera")
+	serverAPrivKey := test.PrivateKeyA
+	creator := test.NewUser(t, test.WithSigningServer(serverA, serverAKeyID, serverAPrivKey))
+
+	myServer := base.Cfg.Global.ServerName
+	myServerKeyID := base.Cfg.Global.KeyID
+	myServerPrivKey := base.Cfg.Global.PrivateKey
+	joiningUser := test.NewUser(t, test.WithSigningServer(myServer, myServerKeyID, myServerPrivKey))
+	fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID)
+	room := test.NewRoom(t, creator)
+
+	rsapi := &fedRoomserverAPI{
+		inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+			if req.Asynchronous {
+				t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous")
+			}
+		},
+		queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error {
+			if req.UserID == joiningUser.ID && req.WantMembership == "join" {
+				res.RoomIDs = []string{room.ID}
+				return nil
+			}
+			return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req)
+		},
+	}
+	fc := &fedClient{
+		allowJoins: []*test.Room{room},
+		t:          t,
+		keys: map[gomatrixserverlib.ServerName]struct {
+			key   ed25519.PrivateKey
+			keyID gomatrixserverlib.KeyID
+		}{
+			serverA: {
+				key:   serverAPrivKey,
+				keyID: serverAKeyID,
+			},
+			myServer: {
+				key:   myServerPrivKey,
+				keyID: myServerKeyID,
+			},
+		},
+	}
+	fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false)
+
+	var resp api.PerformJoinResponse
+	fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{
+		RoomID:      room.ID,
+		UserID:      joiningUser.ID,
+		ServerNames: []gomatrixserverlib.ServerName{serverA},
+	}, &resp)
+	if resp.JoinedVia != serverA {
+		t.Errorf("PerformJoin: joined via %v want %v", resp.JoinedVia, serverA)
+	}
+	if resp.LastError != nil {
+		t.Fatalf("PerformJoin: returned error: %+v", *resp.LastError)
+	}
+
+	// Inject a keyserver key change event and ensure we try to send it out. If we don't, then the
+	// federationapi is incorrectly waiting for an output room event to arrive to update the joined
+	// hosts table.
+	key := keyapi.DeviceMessage{
+		Type: keyapi.TypeDeviceKeyUpdate,
+		DeviceKeys: &keyapi.DeviceKeys{
+			UserID:      joiningUser.ID,
+			DeviceID:    "MY_DEVICE",
+			DisplayName: "BLARGLE",
+			KeyJSON:     []byte(`{}`),
+		},
+	}
+	b, err := json.Marshal(key)
+	if err != nil {
+		t.Fatalf("Failed to marshal device message: %s", err)
+	}
+
+	msg := &nats.Msg{
+		Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
+		Header:  nats.Header{},
+		Data:    b,
+	}
+	msg.Header.Set(jetstream.UserID, key.UserID)
+
+	testrig.MustPublishMsgs(t, jsctx, msg)
+	time.Sleep(500 * time.Millisecond)
+	if !fc.sentTxn {
+		t.Fatalf("did not send device list update")
+	}
+}
+
 // Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404.
 // Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated.
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
@@ -86,7 +318,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 		}
 		gerr, ok := err.(gomatrix.HTTPError)
 		if !ok {
-			t.Errorf("failed to cast response error as gomatrix.HTTPError")
+			t.Errorf("failed to cast response error as gomatrix.HTTPError: %s", err)
 			continue
 		}
 		t.Logf("Error: %+v", gerr)
diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go
index 4e9fa8410..14056eafc 100644
--- a/federationapi/internal/api.go
+++ b/federationapi/internal/api.go
@@ -25,8 +25,8 @@ type FederationInternalAPI struct {
 	db         storage.Database
 	cfg        *config.FederationAPI
 	statistics *statistics.Statistics
-	rsAPI      roomserverAPI.RoomserverInternalAPI
-	federation *gomatrixserverlib.FederationClient
+	rsAPI      roomserverAPI.FederationRoomserverAPI
+	federation api.FederationClient
 	keyRing    *gomatrixserverlib.KeyRing
 	queues     *queue.OutgoingQueues
 	joins      sync.Map // joins currently in progress
@@ -34,8 +34,8 @@ type FederationInternalAPI struct {
 
 func NewFederationInternalAPI(
 	db storage.Database, cfg *config.FederationAPI,
-	rsAPI roomserverAPI.RoomserverInternalAPI,
-	federation *gomatrixserverlib.FederationClient,
+	rsAPI roomserverAPI.FederationRoomserverAPI,
+	federation api.FederationClient,
 	statistics *statistics.Statistics,
 	caches *caching.Caches,
 	queues *queue.OutgoingQueues,
diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 577cb70e0..7ccd68ef0 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/consumers"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrix"
@@ -235,6 +236,21 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		return fmt.Errorf("respSendJoin.Check: %w", err)
 	}
 
+	// We need to immediately update our list of joined hosts for this room now as we are technically
+	// joined. We must do this synchronously: we cannot rely on the roomserver output events as they
+	// will happen asyncly. If we don't update this table, you can end up with bad failure modes like
+	// joining a room, waiting for 200 OK then changing device keys and have those keys not be sent
+	// to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers")
+	// The events are trusted now as we performed auth checks above.
+	joinedHosts, err := consumers.JoinedHostsFromEvents(respState.StateEvents.TrustedEvents(respMakeJoin.RoomVersion, false))
+	if err != nil {
+		return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
+	}
+	logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts")
+	if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil {
+		return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err)
+	}
+
 	// If we successfully performed a send_join above then the other
 	// server now thinks we're a part of the room. Send the newly
 	// returned state to the roomserver to update our local view.
@@ -650,7 +666,7 @@ func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder
 
 // FederatedAuthProvider is an auth chain provider which fetches events from the server provided
 func federatedAuthProvider(
-	ctx context.Context, federation *gomatrixserverlib.FederationClient,
+	ctx context.Context, federation api.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier, server gomatrixserverlib.ServerName,
 ) gomatrixserverlib.AuthChainProvider {
 	// A list of events that we have retried, if they were not included in
diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go
index 747940403..b6edec5da 100644
--- a/federationapi/queue/destinationqueue.go
+++ b/federationapi/queue/destinationqueue.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
@@ -49,21 +50,21 @@ type destinationQueue struct {
 	db                 storage.Database
 	process            *process.ProcessContext
 	signing            *SigningInfo
-	rsAPI              api.RoomserverInternalAPI
-	client             *gomatrixserverlib.FederationClient // federation client
-	origin             gomatrixserverlib.ServerName        // origin of requests
-	destination        gomatrixserverlib.ServerName        // destination of requests
-	running            atomic.Bool                         // is the queue worker running?
-	backingOff         atomic.Bool                         // true if we're backing off
-	overflowed         atomic.Bool                         // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more
-	statistics         *statistics.ServerStatistics        // statistics about this remote server
-	transactionIDMutex sync.Mutex                          // protects transactionID
-	transactionID      gomatrixserverlib.TransactionID     // last transaction ID if retrying, or "" if last txn was successful
-	notify             chan struct{}                       // interrupts idle wait pending PDUs/EDUs
-	pendingPDUs        []*queuedPDU                        // PDUs waiting to be sent
-	pendingEDUs        []*queuedEDU                        // EDUs waiting to be sent
-	pendingMutex       sync.RWMutex                        // protects pendingPDUs and pendingEDUs
-	interruptBackoff   chan bool                           // interrupts backoff
+	rsAPI              api.FederationRoomserverAPI
+	client             fedapi.FederationClient         // federation client
+	origin             gomatrixserverlib.ServerName    // origin of requests
+	destination        gomatrixserverlib.ServerName    // destination of requests
+	running            atomic.Bool                     // is the queue worker running?
+	backingOff         atomic.Bool                     // true if we're backing off
+	overflowed         atomic.Bool                     // the queues exceed maxPDUsInMemory/maxEDUsInMemory, so we should consult the database for more
+	statistics         *statistics.ServerStatistics    // statistics about this remote server
+	transactionIDMutex sync.Mutex                      // protects transactionID
+	transactionID      gomatrixserverlib.TransactionID // last transaction ID if retrying, or "" if last txn was successful
+	notify             chan struct{}                   // interrupts idle wait pending PDUs/EDUs
+	pendingPDUs        []*queuedPDU                    // PDUs waiting to be sent
+	pendingEDUs        []*queuedEDU                    // EDUs waiting to be sent
+	pendingMutex       sync.RWMutex                    // protects pendingPDUs and pendingEDUs
+	interruptBackoff   chan bool                       // interrupts backoff
 }
 
 // Send event adds the event to the pending queue for the destination.
diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go
index d152886f5..4c25c4ce6 100644
--- a/federationapi/queue/queue.go
+++ b/federationapi/queue/queue.go
@@ -26,6 +26,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 
+	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
@@ -39,9 +40,9 @@ type OutgoingQueues struct {
 	db          storage.Database
 	process     *process.ProcessContext
 	disabled    bool
-	rsAPI       api.RoomserverInternalAPI
+	rsAPI       api.FederationRoomserverAPI
 	origin      gomatrixserverlib.ServerName
-	client      *gomatrixserverlib.FederationClient
+	client      fedapi.FederationClient
 	statistics  *statistics.Statistics
 	signing     *SigningInfo
 	queuesMutex sync.Mutex // protects the below
@@ -85,8 +86,8 @@ func NewOutgoingQueues(
 	process *process.ProcessContext,
 	disabled bool,
 	origin gomatrixserverlib.ServerName,
-	client *gomatrixserverlib.FederationClient,
-	rsAPI api.RoomserverInternalAPI,
+	client fedapi.FederationClient,
+	rsAPI api.FederationRoomserverAPI,
 	statistics *statistics.Statistics,
 	signing *SigningInfo,
 ) *OutgoingQueues {
diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go
index 707b7b019..316c61a14 100644
--- a/federationapi/routing/query.go
+++ b/federationapi/routing/query.go
@@ -30,7 +30,7 @@ import (
 // RoomAliasToID converts the queried alias into a room ID and returns it
 func RoomAliasToID(
 	httpReq *http.Request,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	senderAPI federationAPI.FederationInternalAPI,
diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index 9f95ed07e..e25f9866e 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -54,7 +54,7 @@ func Setup(
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fsAPI *fedInternal.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	userAPI userapi.FederationUserAPI,
 	keyAPI keyserverAPI.FederationKeyAPI,
 	mscCfg *config.MSCs,
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 55a113675..c25dabce9 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -85,7 +85,7 @@ func Send(
 	rsAPI api.FederationRoomserverAPI,
 	keyAPI keyapi.FederationKeyAPI,
 	keys gomatrixserverlib.JSONVerifier,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	mu *internal.MutexByRoom,
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,
diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go
index 011d4e342..a111580c7 100644
--- a/federationapi/routing/send_test.go
+++ b/federationapi/routing/send_test.go
@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go
index 16f245cee..ccde9168e 100644
--- a/federationapi/routing/threepid.go
+++ b/federationapi/routing/threepid.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -57,7 +58,7 @@ var (
 func CreateInvitesFrom3PIDInvites(
 	req *http.Request, rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 	userAPI userapi.FederationUserAPI,
 ) util.JSONResponse {
 	var body invites
@@ -107,7 +108,7 @@ func ExchangeThirdPartyInvite(
 	roomID string,
 	rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
-	federation *gomatrixserverlib.FederationClient,
+	federation federationAPI.FederationClient,
 ) util.JSONResponse {
 	var builder gomatrixserverlib.EventBuilder
 	if err := json.Unmarshal(request.Content(), &builder); err != nil {
@@ -165,7 +166,12 @@ func ExchangeThirdPartyInvite(
 
 	// Ask the requesting server to sign the newly created event so we know it
 	// acknowledged it
-	signedEvent, err := federation.SendInvite(httpReq.Context(), request.Origin(), event)
+	inviteReq, err := gomatrixserverlib.NewInviteV2Request(event.Headered(verRes.RoomVersion), nil)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("failed to make invite v2 request")
+		return jsonerror.InternalServerError()
+	}
+	signedEvent, err := federation.SendInviteV2(httpReq.Context(), request.Origin(), inviteReq)
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed")
 		return jsonerror.InternalServerError()
@@ -205,7 +211,7 @@ func ExchangeThirdPartyInvite(
 func createInviteFrom3PIDInvite(
 	ctx context.Context, rsAPI api.FederationRoomserverAPI,
 	cfg *config.FederationAPI,
-	inv invite, federation *gomatrixserverlib.FederationClient,
+	inv invite, federation federationAPI.FederationClient,
 	userAPI userapi.FederationUserAPI,
 ) (*gomatrixserverlib.Event, error) {
 	verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID}
@@ -335,7 +341,7 @@ func buildMembershipEvent(
 // them responded with an error.
 func sendToRemoteServer(
 	ctx context.Context, inv invite,
-	federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI,
+	federation federationAPI.FederationClient, _ *config.FederationAPI,
 	builder gomatrixserverlib.EventBuilder,
 ) (err error) {
 	remoteServers := make([]gomatrixserverlib.ServerName, 2)
diff --git a/federationapi/storage/interface.go b/federationapi/storage/interface.go
index e3038651b..29254948b 100644
--- a/federationapi/storage/interface.go
+++ b/federationapi/storage/interface.go
@@ -25,13 +25,12 @@ import (
 type Database interface {
 	gomatrixserverlib.KeyDatabase
 
-	UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error)
+	UpdateRoom(ctx context.Context, roomID string, addHosts []types.JoinedHost, removeHosts []string, purgeRoomFirst bool) (joinedHosts []types.JoinedHost, err error)
 
 	GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error)
 	GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
 	// GetJoinedHostsForRooms returns the complete set of servers in the rooms given.
 	GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, excludeSelf bool) ([]gomatrixserverlib.ServerName, error)
-	PurgeRoomState(ctx context.Context, roomID string) error
 
 	StoreJSON(ctx context.Context, js string) (*shared.Receipt, error)
 
diff --git a/federationapi/storage/postgres/joined_hosts_table.go b/federationapi/storage/postgres/joined_hosts_table.go
index 5c95b72a8..bb6f6bfa3 100644
--- a/federationapi/storage/postgres/joined_hosts_table.go
+++ b/federationapi/storage/postgres/joined_hosts_table.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
 )
 
 const joinedHostsSchema = `
@@ -111,6 +112,7 @@ func (s *joinedHostsStatements) InsertJoinedHosts(
 	roomID, eventID string,
 	serverName gomatrixserverlib.ServerName,
 ) error {
+	logrus.Debugf("FederationJoinedHosts: INSERT %v %v %v", roomID, eventID, serverName)
 	stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt)
 	_, err := stmt.ExecContext(ctx, roomID, eventID, serverName)
 	return err
@@ -119,6 +121,7 @@ func (s *joinedHostsStatements) InsertJoinedHosts(
 func (s *joinedHostsStatements) DeleteJoinedHosts(
 	ctx context.Context, txn *sql.Tx, eventIDs []string,
 ) error {
+	logrus.Debugf("FederationJoinedHosts: DELETE WITH EVENTS %v", eventIDs)
 	stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt)
 	_, err := stmt.ExecContext(ctx, pq.StringArray(eventIDs))
 	return err
@@ -127,6 +130,7 @@ func (s *joinedHostsStatements) DeleteJoinedHosts(
 func (s *joinedHostsStatements) DeleteJoinedHostsForRoom(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) error {
+	logrus.Debugf("FederationJoinedHosts: DELETE ALL IN ROOM %v", roomID)
 	stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsForRoomStmt)
 	_, err := stmt.ExecContext(ctx, roomID)
 	return err
@@ -207,6 +211,7 @@ func joinedHostsFromStmt(
 			ServerName:    gomatrixserverlib.ServerName(serverName),
 		})
 	}
+	logrus.Debugf("FederationJoinedHosts: SELECT %v => %+v", roomID, result)
 
 	return result, rows.Err()
 }
diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go
index 160c7f6fa..a00d782f1 100644
--- a/federationapi/storage/shared/storage.go
+++ b/federationapi/storage/shared/storage.go
@@ -63,11 +63,21 @@ func (r *Receipt) String() string {
 // this isn't a duplicate message.
 func (d *Database) UpdateRoom(
 	ctx context.Context,
-	roomID, oldEventID, newEventID string,
+	roomID string,
 	addHosts []types.JoinedHost,
 	removeHosts []string,
+	purgeRoomFirst bool,
 ) (joinedHosts []types.JoinedHost, err error) {
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		if purgeRoomFirst {
+			// If the event is a create event then we'll delete all of the existing
+			// data for the room. The only reason that a create event would be replayed
+			// to us in this way is if we're about to receive the entire room state.
+			if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
+				return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
+			}
+		}
+
 		joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID)
 		if err != nil {
 			return err
@@ -138,20 +148,6 @@ func (d *Database) StoreJSON(
 	}, nil
 }
 
-func (d *Database) PurgeRoomState(
-	ctx context.Context, roomID string,
-) error {
-	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		// If the event is a create event then we'll delete all of the existing
-		// data for the room. The only reason that a create event would be replayed
-		// to us in this way is if we're about to receive the entire room state.
-		if err := d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
-			return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
-		}
-		return nil
-	})
-}
-
 func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationBlacklist.InsertBlacklist(context.TODO(), txn, serverName)
diff --git a/internal/caching/cache_typing_test.go b/internal/caching/cache_typing_test.go
index c03d89bc3..2cef32d3e 100644
--- a/internal/caching/cache_typing_test.go
+++ b/internal/caching/cache_typing_test.go
@@ -20,7 +20,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/dendrite/test"
 )
 
 func TestEDUCache(t *testing.T) {
diff --git a/internal/test/client.go b/internal/test/client.go
deleted file mode 100644
index a38540ac9..000000000
--- a/internal/test/client.go
+++ /dev/null
@@ -1,158 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// Request contains the information necessary to issue a request and test its result
-type Request struct {
-	Req              *http.Request
-	WantedBody       string
-	WantedStatusCode int
-	LastErr          *LastRequestErr
-}
-
-// LastRequestErr is a synchronised error wrapper
-// Useful for obtaining the last error from a set of requests
-type LastRequestErr struct {
-	sync.Mutex
-	Err error
-}
-
-// Set sets the error
-func (r *LastRequestErr) Set(err error) {
-	r.Lock()
-	defer r.Unlock()
-	r.Err = err
-}
-
-// Get gets the error
-func (r *LastRequestErr) Get() error {
-	r.Lock()
-	defer r.Unlock()
-	return r.Err
-}
-
-// CanonicalJSONInput canonicalises a slice of JSON strings
-// Useful for test input
-func CanonicalJSONInput(jsonData []string) []string {
-	for i := range jsonData {
-		jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i]))
-		if err != nil && err != io.EOF {
-			panic(err)
-		}
-		jsonData[i] = string(jsonBytes)
-	}
-	return jsonData
-}
-
-// Do issues a request and checks the status code and body of the response
-func (r *Request) Do() (err error) {
-	client := &http.Client{
-		Timeout: 5 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-		},
-	}
-	res, err := client.Do(r.Req)
-	if err != nil {
-		return err
-	}
-	defer (func() { err = res.Body.Close() })()
-
-	if res.StatusCode != r.WantedStatusCode {
-		return fmt.Errorf("incorrect status code. Expected: %d  Got: %d", r.WantedStatusCode, res.StatusCode)
-	}
-
-	if r.WantedBody != "" {
-		resBytes, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return err
-		}
-		jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes)
-		if err != nil {
-			return err
-		}
-		if string(jsonBytes) != r.WantedBody {
-			return fmt.Errorf("returned wrong bytes. Expected:\n%s\n\nGot:\n%s", r.WantedBody, string(jsonBytes))
-		}
-	}
-
-	return nil
-}
-
-// DoUntilSuccess blocks and repeats the same request until the response returns the desired status code and body.
-// It then closes the given channel and returns.
-func (r *Request) DoUntilSuccess(done chan error) {
-	r.LastErr = &LastRequestErr{}
-	for {
-		if err := r.Do(); err != nil {
-			r.LastErr.Set(err)
-			time.Sleep(1 * time.Second) // don't tightloop
-			continue
-		}
-		close(done)
-		return
-	}
-}
-
-// Run repeatedly issues a request until success, error or a timeout is reached
-func (r *Request) Run(label string, timeout time.Duration, serverCmdChan chan error) {
-	fmt.Printf("==TESTING== %v (timeout: %v)\n", label, timeout)
-	done := make(chan error, 1)
-
-	// We need to wait for the server to:
-	// - have connected to the database
-	// - have created the tables
-	// - be listening on the given port
-	go r.DoUntilSuccess(done)
-
-	// wait for one of:
-	// - the test to pass (done channel is closed)
-	// - the server to exit with an error (error sent on serverCmdChan)
-	// - our test timeout to expire
-	// We don't need to clean up since the main() function handles that in the event we panic
-	select {
-	case <-time.After(timeout):
-		fmt.Printf("==TESTING== %v TIMEOUT\n", label)
-		if reqErr := r.LastErr.Get(); reqErr != nil {
-			fmt.Println("Last /sync request error:")
-			fmt.Println(reqErr)
-		}
-		panic(fmt.Sprintf("%v server timed out", label))
-	case err := <-serverCmdChan:
-		if err != nil {
-			fmt.Println("=============================================================================================")
-			fmt.Printf("%v server failed to run. If failing with 'pq: password authentication failed for user' try:", label)
-			fmt.Println("    export PGHOST=/var/run/postgresql")
-			fmt.Println("=============================================================================================")
-			panic(err)
-		}
-	case <-done:
-		fmt.Printf("==TESTING== %v PASSED\n", label)
-	}
-}
diff --git a/internal/test/kafka.go b/internal/test/kafka.go
deleted file mode 100644
index cbf246304..000000000
--- a/internal/test/kafka.go
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"io"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
-
-// KafkaExecutor executes kafka scripts.
-type KafkaExecutor struct {
-	// The location of Zookeeper. Typically this is `localhost:2181`.
-	ZookeeperURI string
-	// The directory where Kafka is installed to. Used to locate kafka scripts.
-	KafkaDirectory string
-	// The location of the Kafka logs. Typically this is `localhost:9092`.
-	KafkaURI string
-	// Where stdout and stderr should be written to. Typically this is `os.Stderr`.
-	OutputWriter io.Writer
-}
-
-// CreateTopic creates a new kafka topic. This is created with a single partition.
-func (e *KafkaExecutor) CreateTopic(topic string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
-		"--create",
-		"--zookeeper", e.ZookeeperURI,
-		"--replication-factor", "1",
-		"--partitions", "1",
-		"--topic", topic,
-	)
-	cmd.Stdout = e.OutputWriter
-	cmd.Stderr = e.OutputWriter
-	return cmd.Run()
-}
-
-// WriteToTopic writes data to a kafka topic.
-func (e *KafkaExecutor) WriteToTopic(topic string, data []string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-console-producer.sh"),
-		"--broker-list", e.KafkaURI,
-		"--topic", topic,
-	)
-	cmd.Stdout = e.OutputWriter
-	cmd.Stderr = e.OutputWriter
-	cmd.Stdin = strings.NewReader(strings.Join(data, "\n"))
-	return cmd.Run()
-}
-
-// DeleteTopic deletes a given kafka topic if it exists.
-func (e *KafkaExecutor) DeleteTopic(topic string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
-		"--delete",
-		"--if-exists",
-		"--zookeeper", e.ZookeeperURI,
-		"--topic", topic,
-	)
-	cmd.Stderr = e.OutputWriter
-	cmd.Stdout = e.OutputWriter
-	return cmd.Run()
-}
diff --git a/internal/test/server.go b/internal/test/server.go
deleted file mode 100644
index ca14ea1bf..000000000
--- a/internal/test/server.go
+++ /dev/null
@@ -1,152 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-// Defaulting allows assignment of string variables with a fallback default value
-// Useful for use with os.Getenv() for example
-func Defaulting(value, defaultValue string) string {
-	if value == "" {
-		value = defaultValue
-	}
-	return value
-}
-
-// CreateDatabase creates a new database, dropping it first if it exists
-func CreateDatabase(command string, args []string, database string) error {
-	cmd := exec.Command(command, args...)
-	cmd.Stdin = strings.NewReader(
-		fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
-	)
-	// Send stdout and stderr to our stderr so that we see error messages from
-	// the psql process
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
-
-// CreateBackgroundCommand creates an executable command
-// The Cmd being executed is returned. A channel is also returned,
-// which will have any termination errors sent down it, followed immediately by the channel being closed.
-func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan error) {
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stderr
-
-	if err := cmd.Start(); err != nil {
-		panic("failed to start server: " + err.Error())
-	}
-	cmdChan := make(chan error, 1)
-	go func() {
-		cmdChan <- cmd.Wait()
-		close(cmdChan)
-	}()
-	return cmd, cmdChan
-}
-
-// InitDatabase creates the database and config file needed for the server to run
-func InitDatabase(postgresDatabase, postgresContainerName string, databases []string) {
-	if len(databases) > 0 {
-		var dbCmd string
-		var dbArgs []string
-		if postgresContainerName == "" {
-			dbCmd = "psql"
-			dbArgs = []string{postgresDatabase}
-		} else {
-			dbCmd = "docker"
-			dbArgs = []string{
-				"exec", "-i", postgresContainerName, "psql", "-U", "postgres", postgresDatabase,
-			}
-		}
-		for _, database := range databases {
-			if err := CreateDatabase(dbCmd, dbArgs, database); err != nil {
-				panic(err)
-			}
-		}
-	}
-}
-
-// StartProxy creates a reverse proxy
-func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
-	proxyArgs := []string{
-		"--bind-address", bindAddr,
-		"--sync-api-server-url", "http://" + string(cfg.SyncAPI.InternalAPI.Connect),
-		"--client-api-server-url", "http://" + string(cfg.ClientAPI.InternalAPI.Connect),
-		"--media-api-server-url", "http://" + string(cfg.MediaAPI.InternalAPI.Connect),
-		"--tls-cert", "server.crt",
-		"--tls-key", "server.key",
-	}
-	return CreateBackgroundCommand(
-		filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),
-		proxyArgs,
-	)
-}
-
-// ListenAndServe will listen on a random high-numbered port and attach the given router.
-// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
-func ListenAndServe(t *testing.T, router http.Handler, useTLS bool) (apiURL string, cancel func()) {
-	listener, err := net.Listen("tcp", ":0")
-	if err != nil {
-		t.Fatalf("failed to listen: %s", err)
-	}
-	port := listener.Addr().(*net.TCPAddr).Port
-	srv := http.Server{}
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		srv.Handler = router
-		var err error
-		if useTLS {
-			certFile := filepath.Join(os.TempDir(), "dendrite.cert")
-			keyFile := filepath.Join(os.TempDir(), "dendrite.key")
-			err = NewTLSKey(keyFile, certFile)
-			if err != nil {
-				t.Logf("failed to generate tls key/cert: %s", err)
-				return
-			}
-			err = srv.ServeTLS(listener, certFile, keyFile)
-		} else {
-			err = srv.Serve(listener)
-		}
-		if err != nil && err != http.ErrServerClosed {
-			t.Logf("Listen failed: %s", err)
-		}
-	}()
-
-	secure := ""
-	if useTLS {
-		secure = "s"
-	}
-	return fmt.Sprintf("http%s://localhost:%d", secure, port), func() {
-		_ = srv.Shutdown(context.Background())
-		wg.Wait()
-	}
-}
diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 561c9a163..23f3e1a67 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -84,7 +84,7 @@ type DeviceListUpdater struct {
 	db          DeviceListUpdaterDatabase
 	api         DeviceListUpdaterAPI
 	producer    KeyChangeProducer
-	fedClient   fedsenderapi.FederationClient
+	fedClient   fedsenderapi.KeyserverFederationAPI
 	workerChans []chan gomatrixserverlib.ServerName
 
 	// When device lists are stale for a user, they get inserted into this map with a channel which `Update` will
@@ -127,7 +127,7 @@ type KeyChangeProducer interface {
 // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale.
 func NewDeviceListUpdater(
 	db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer,
-	fedClient fedsenderapi.FederationClient, numWorkers int,
+	fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int,
 ) *DeviceListUpdater {
 	return &DeviceListUpdater{
 		userIDToMutex:  make(map[string]*sync.Mutex),
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index be71e5750..f8d0d69c3 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -37,7 +37,7 @@ import (
 type KeyInternalAPI struct {
 	DB         storage.Database
 	ThisServer gomatrixserverlib.ServerName
-	FedClient  fedsenderapi.FederationClient
+	FedClient  fedsenderapi.KeyserverFederationAPI
 	UserAPI    userapi.KeyserverUserAPI
 	Producer   *producers.KeyChange
 	Updater    *DeviceListUpdater
diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go
index 47d7f57f9..3ffd3ba1e 100644
--- a/keyserver/keyserver.go
+++ b/keyserver/keyserver.go
@@ -37,7 +37,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) {
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient,
+	base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.KeyserverFederationAPI,
 ) api.KeyInternalAPI {
 	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index cbb4cebca..80e7aed64 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -183,6 +183,7 @@ type FederationRoomserverAPI interface {
 	QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error
 	// Query whether a server is allowed to see an event
 	QueryServerAllowedToSeeEvent(ctx context.Context, req *QueryServerAllowedToSeeEventRequest, res *QueryServerAllowedToSeeEventResponse) error
+	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
 	PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
 	// Query a given amount (or less) of events prior to a given set of events.
diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go
index a95c13550..7c65f9eac 100644
--- a/roomserver/internal/input/input_test.go
+++ b/roomserver/internal/input/input_test.go
@@ -12,7 +12,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 )
@@ -22,7 +22,7 @@ var jc *nats.Conn
 
 func TestMain(m *testing.M) {
 	var b *base.BaseDendrite
-	b, js, jc = test.Base(nil)
+	b, js, jc = testrig.Base(nil)
 	code := m.Run()
 	b.ShutdownDendrite()
 	b.WaitForComponentsToFinish()
diff --git a/roomserver/internal/query/query_test.go b/roomserver/internal/query/query_test.go
index ba5bb9f55..03627ea97 100644
--- a/roomserver/internal/query/query_test.go
+++ b/roomserver/internal/query/query_test.go
@@ -19,8 +19,8 @@ import (
 	"encoding/json"
 	"testing"
 
-	"github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
diff --git a/roomserver/storage/tables/events_table_test.go b/roomserver/storage/tables/events_table_test.go
index d5d699c4c..6f72a59b5 100644
--- a/roomserver/storage/tables/events_table_test.go
+++ b/roomserver/storage/tables/events_table_test.go
@@ -39,7 +39,7 @@ func mustCreateEventsTable(t *testing.T, dbType test.DBType) (tables.Events, fun
 }
 
 func Test_EventsTable(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
diff --git a/roomserver/storage/tables/previous_events_table_test.go b/roomserver/storage/tables/previous_events_table_test.go
index 96d7bfed0..63d540696 100644
--- a/roomserver/storage/tables/previous_events_table_test.go
+++ b/roomserver/storage/tables/previous_events_table_test.go
@@ -38,7 +38,7 @@ func mustCreatePreviousEventsTable(t *testing.T, dbType test.DBType) (tab tables
 
 func TestPreviousEventsTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, close := mustCreatePreviousEventsTable(t, dbType)
diff --git a/roomserver/storage/tables/published_table_test.go b/roomserver/storage/tables/published_table_test.go
index 87662ed4c..fff6dc186 100644
--- a/roomserver/storage/tables/published_table_test.go
+++ b/roomserver/storage/tables/published_table_test.go
@@ -38,7 +38,7 @@ func mustCreatePublishedTable(t *testing.T, dbType test.DBType) (tab tables.Publ
 
 func TestPublishedTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, close := mustCreatePublishedTable(t, dbType)
diff --git a/roomserver/storage/tables/room_aliases_table_test.go b/roomserver/storage/tables/room_aliases_table_test.go
index 8fb57d5a4..624d92ae6 100644
--- a/roomserver/storage/tables/room_aliases_table_test.go
+++ b/roomserver/storage/tables/room_aliases_table_test.go
@@ -36,7 +36,7 @@ func mustCreateRoomAliasesTable(t *testing.T, dbType test.DBType) (tab tables.Ro
 }
 
 func TestRoomAliasesTable(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	room2 := test.NewRoom(t, alice)
 	ctx := context.Background()
diff --git a/roomserver/storage/tables/rooms_table_test.go b/roomserver/storage/tables/rooms_table_test.go
index 9872fb800..0a02369a1 100644
--- a/roomserver/storage/tables/rooms_table_test.go
+++ b/roomserver/storage/tables/rooms_table_test.go
@@ -38,7 +38,7 @@ func mustCreateRoomsTable(t *testing.T, dbType test.DBType) (tab tables.Rooms, c
 }
 
 func TestRoomsTable(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	ctx := context.Background()
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index 1150c2f3d..563c92e34 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -47,7 +47,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserver
 
 func TestWriteEvents(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		r := test.NewRoom(t, alice)
 		db, close := MustCreateDatabase(t, dbType)
 		defer close()
@@ -60,7 +60,7 @@ func TestRecentEventsPDU(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		db, close := MustCreateDatabase(t, dbType)
 		defer close()
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		// dummy room to make sure SQL queries are filtering on room ID
 		MustWriteEvents(t, db, test.NewRoom(t, alice).Events())
 
@@ -163,7 +163,7 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		db, close := MustCreateDatabase(t, dbType)
 		defer close()
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		r := test.NewRoom(t, alice)
 		for i := 0; i < 10; i++ {
 			r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("hi %d", i)})
diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go
index 8bbf879d4..69bbd04c9 100644
--- a/syncapi/storage/tables/output_room_events_test.go
+++ b/syncapi/storage/tables/output_room_events_test.go
@@ -45,7 +45,7 @@ func newOutputRoomEventsTable(t *testing.T, dbType test.DBType) (tables.Events,
 
 func TestOutputRoomEventsTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, db, close := newOutputRoomEventsTable(t, dbType)
diff --git a/syncapi/storage/tables/topology_test.go b/syncapi/storage/tables/topology_test.go
index 2334aae2e..f4f75bdf3 100644
--- a/syncapi/storage/tables/topology_test.go
+++ b/syncapi/storage/tables/topology_test.go
@@ -40,7 +40,7 @@ func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.D
 
 func TestTopologyTable(t *testing.T) {
 	ctx := context.Background()
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		tab, db, close := newTopologyTable(t, dbType)
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index d3d898394..5ecfd8772 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
@@ -86,7 +87,7 @@ func TestSyncAPIAccessTokens(t *testing.T) {
 }
 
 func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
-	user := test.NewUser()
+	user := test.NewUser(t)
 	room := test.NewRoom(t, user)
 	alice := userapi.Device{
 		ID:          "ALICEID",
@@ -96,14 +97,14 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
 		AccountType: userapi.AccountTypeUser,
 	}
 
-	base, close := test.CreateBaseDendrite(t, dbType)
+	base, close := testrig.CreateBaseDendrite(t, dbType)
 	defer close()
 
 	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
 	msgs := toNATSMsgs(t, base, room.Events())
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
-	test.MustPublishMsgs(t, jsctx, msgs...)
+	testrig.MustPublishMsgs(t, jsctx, msgs...)
 
 	testCases := []struct {
 		name            string
@@ -173,7 +174,7 @@ func TestSyncAPICreateRoomSyncEarly(t *testing.T) {
 }
 
 func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
-	user := test.NewUser()
+	user := test.NewUser(t)
 	room := test.NewRoom(t, user)
 	alice := userapi.Device{
 		ID:          "ALICEID",
@@ -183,7 +184,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 		AccountType: userapi.AccountTypeUser,
 	}
 
-	base, close := test.CreateBaseDendrite(t, dbType)
+	base, close := testrig.CreateBaseDendrite(t, dbType)
 	defer close()
 
 	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
@@ -198,7 +199,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 	sinceTokens := make([]string, len(msgs))
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
 	for i, msg := range msgs {
-		test.MustPublishMsgs(t, jsctx, msg)
+		testrig.MustPublishMsgs(t, jsctx, msg)
 		time.Sleep(100 * time.Millisecond)
 		w := httptest.NewRecorder()
 		base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
@@ -262,7 +263,7 @@ func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverli
 		if ev.StateKey() != nil {
 			addsStateIDs = append(addsStateIDs, ev.EventID())
 		}
-		result[i] = test.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{
+		result[i] = testrig.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{
 			Type: rsapi.OutputTypeNewRoomEvent,
 			NewRoomEvent: &rsapi.OutputNewRoomEvent{
 				Event:             ev,
diff --git a/test/event.go b/test/event.go
index 40cb8f0e1..73fc656bd 100644
--- a/test/event.go
+++ b/test/event.go
@@ -52,6 +52,24 @@ func WithUnsigned(unsigned interface{}) eventModifier {
 	}
 }
 
+func WithKeyID(keyID gomatrixserverlib.KeyID) eventModifier {
+	return func(e *eventMods) {
+		e.keyID = keyID
+	}
+}
+
+func WithPrivateKey(pkey ed25519.PrivateKey) eventModifier {
+	return func(e *eventMods) {
+		e.privKey = pkey
+	}
+}
+
+func WithOrigin(origin gomatrixserverlib.ServerName) eventModifier {
+	return func(e *eventMods) {
+		e.origin = origin
+	}
+}
+
 // Reverse a list of events
 func Reversed(in []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
 	out := make([]*gomatrixserverlib.HeaderedEvent, len(in))
diff --git a/test/http.go b/test/http.go
index a458a3385..37b3648f8 100644
--- a/test/http.go
+++ b/test/http.go
@@ -2,10 +2,15 @@ package test
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
+	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
+	"path/filepath"
+	"sync"
 	"testing"
 )
 
@@ -43,3 +48,45 @@ func NewRequest(t *testing.T, method, path string, opts ...HTTPRequestOpt) *http
 	}
 	return req
 }
+
+// ListenAndServe will listen on a random high-numbered port and attach the given router.
+// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
+func ListenAndServe(t *testing.T, router http.Handler, withTLS bool) (apiURL string, cancel func()) {
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatalf("failed to listen: %s", err)
+	}
+	port := listener.Addr().(*net.TCPAddr).Port
+	srv := http.Server{}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		srv.Handler = router
+		var err error
+		if withTLS {
+			certFile := filepath.Join(t.TempDir(), "dendrite.cert")
+			keyFile := filepath.Join(t.TempDir(), "dendrite.key")
+			err = NewTLSKey(keyFile, certFile)
+			if err != nil {
+				t.Errorf("failed to make TLS key: %s", err)
+				return
+			}
+			err = srv.ServeTLS(listener, certFile, keyFile)
+		} else {
+			err = srv.Serve(listener)
+		}
+		if err != nil && err != http.ErrServerClosed {
+			t.Logf("Listen failed: %s", err)
+		}
+	}()
+	s := ""
+	if withTLS {
+		s = "s"
+	}
+	return fmt.Sprintf("http%s://localhost:%d", s, port), func() {
+		_ = srv.Shutdown(context.Background())
+		wg.Wait()
+	}
+}
diff --git a/internal/test/keyring.go b/test/keyring.go
similarity index 100%
rename from internal/test/keyring.go
rename to test/keyring.go
diff --git a/internal/test/config.go b/test/keys.go
similarity index 61%
rename from internal/test/config.go
rename to test/keys.go
index d8e0c4531..75e3800e0 100644
--- a/internal/test/config.go
+++ b/test/keys.go
@@ -25,103 +25,19 @@ import (
 	"io/ioutil"
 	"math/big"
 	"os"
-	"path/filepath"
 	"strings"
 	"time"
-
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"gopkg.in/yaml.v2"
 )
 
 const (
-	// ConfigFile is the name of the config file for a server.
-	ConfigFile = "dendrite.yaml"
 	// ServerKeyFile is the name of the file holding the matrix server private key.
 	ServerKeyFile = "server_key.pem"
 	// TLSCertFile is the name of the file holding the TLS certificate used for federation.
 	TLSCertFile = "tls_cert.pem"
 	// TLSKeyFile is the name of the file holding the TLS key used for federation.
 	TLSKeyFile = "tls_key.pem"
-	// MediaDir is the name of the directory used to store media.
-	MediaDir = "media"
 )
 
-// MakeConfig makes a config suitable for running integration tests.
-// Generates new matrix and TLS keys for the server.
-func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
-	var cfg config.Dendrite
-	cfg.Defaults(true)
-
-	port := startPort
-	assignAddress := func() config.HTTPAddress {
-		result := config.HTTPAddress(fmt.Sprintf("http://%s:%d", host, port))
-		port++
-		return result
-	}
-
-	serverKeyPath := filepath.Join(configDir, ServerKeyFile)
-	tlsCertPath := filepath.Join(configDir, TLSKeyFile)
-	tlsKeyPath := filepath.Join(configDir, TLSCertFile)
-	mediaBasePath := filepath.Join(configDir, MediaDir)
-
-	if err := NewMatrixKey(serverKeyPath); err != nil {
-		return nil, 0, err
-	}
-
-	if err := NewTLSKey(tlsKeyPath, tlsCertPath); err != nil {
-		return nil, 0, err
-	}
-
-	cfg.Version = config.Version
-
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress())
-	cfg.Global.PrivateKeyPath = config.Path(serverKeyPath)
-
-	cfg.MediaAPI.BasePath = config.Path(mediaBasePath)
-
-	cfg.Global.JetStream.Addresses = []string{kafkaURI}
-
-	// TODO: Use different databases for the different schemas.
-	// Using the same database for every schema currently works because
-	// the table names are globally unique. But we might not want to
-	// rely on that in the future.
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(database)
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(database)
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database)
-
-	cfg.AppServiceAPI.InternalAPI.Listen = assignAddress()
-	cfg.FederationAPI.InternalAPI.Listen = assignAddress()
-	cfg.KeyServer.InternalAPI.Listen = assignAddress()
-	cfg.MediaAPI.InternalAPI.Listen = assignAddress()
-	cfg.RoomServer.InternalAPI.Listen = assignAddress()
-	cfg.SyncAPI.InternalAPI.Listen = assignAddress()
-	cfg.UserAPI.InternalAPI.Listen = assignAddress()
-
-	cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen
-	cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen
-	cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen
-	cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen
-	cfg.RoomServer.InternalAPI.Connect = cfg.RoomServer.InternalAPI.Listen
-	cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen
-	cfg.UserAPI.InternalAPI.Connect = cfg.UserAPI.InternalAPI.Listen
-
-	return &cfg, port, nil
-}
-
-// WriteConfig writes the config file to the directory.
-func WriteConfig(cfg *config.Dendrite, configDir string) error {
-	data, err := yaml.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666)
-}
-
 // NewMatrixKey generates a new ed25519 matrix server key and writes it to a file.
 func NewMatrixKey(matrixKeyPath string) (err error) {
 	var data [35]byte
diff --git a/test/room.go b/test/room.go
index 619cb5c9a..6ae403b3f 100644
--- a/test/room.go
+++ b/test/room.go
@@ -15,7 +15,6 @@
 package test
 
 import (
-	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
 	"sync/atomic"
@@ -35,12 +34,6 @@ var (
 	PresetTrustedPrivateChat Preset = 3
 
 	roomIDCounter = int64(0)
-
-	testKeyID      = gomatrixserverlib.KeyID("ed25519:test")
-	testPrivateKey = ed25519.NewKeyFromSeed([]byte{
-		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
-	})
 )
 
 type Room struct {
@@ -49,22 +42,25 @@ type Room struct {
 	preset  Preset
 	creator *User
 
-	authEvents gomatrixserverlib.AuthEvents
-	events     []*gomatrixserverlib.HeaderedEvent
+	authEvents   gomatrixserverlib.AuthEvents
+	currentState map[string]*gomatrixserverlib.HeaderedEvent
+	events       []*gomatrixserverlib.HeaderedEvent
 }
 
 // Create a new test room. Automatically creates the initial create events.
 func NewRoom(t *testing.T, creator *User, modifiers ...roomModifier) *Room {
 	t.Helper()
 	counter := atomic.AddInt64(&roomIDCounter, 1)
-
-	// set defaults then let roomModifiers override
+	if creator.srvName == "" {
+		t.Fatalf("NewRoom: creator doesn't belong to a server: %+v", *creator)
+	}
 	r := &Room{
-		ID:         fmt.Sprintf("!%d:localhost", counter),
-		creator:    creator,
-		authEvents: gomatrixserverlib.NewAuthEvents(nil),
-		preset:     PresetPublicChat,
-		Version:    gomatrixserverlib.RoomVersionV9,
+		ID:           fmt.Sprintf("!%d:%s", counter, creator.srvName),
+		creator:      creator,
+		authEvents:   gomatrixserverlib.NewAuthEvents(nil),
+		preset:       PresetPublicChat,
+		Version:      gomatrixserverlib.RoomVersionV9,
+		currentState: make(map[string]*gomatrixserverlib.HeaderedEvent),
 	}
 	for _, m := range modifiers {
 		m(t, r)
@@ -73,6 +69,24 @@ func NewRoom(t *testing.T, creator *User, modifiers ...roomModifier) *Room {
 	return r
 }
 
+func (r *Room) MustGetAuthEventRefsForEvent(t *testing.T, needed gomatrixserverlib.StateNeeded) []gomatrixserverlib.EventReference {
+	t.Helper()
+	a, err := needed.AuthEventReferences(&r.authEvents)
+	if err != nil {
+		t.Fatalf("MustGetAuthEvents: %v", err)
+	}
+	return a
+}
+
+func (r *Room) ForwardExtremities() []string {
+	if len(r.events) == 0 {
+		return nil
+	}
+	return []string{
+		r.events[len(r.events)-1].EventID(),
+	}
+}
+
 func (r *Room) insertCreateEvents(t *testing.T) {
 	t.Helper()
 	var joinRule gomatrixserverlib.JoinRuleContent
@@ -88,6 +102,7 @@ func (r *Room) insertCreateEvents(t *testing.T) {
 		joinRule.JoinRule = "public"
 		hisVis.HistoryVisibility = "shared"
 	}
+
 	r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomCreate, map[string]interface{}{
 		"creator":      r.creator.ID,
 		"room_version": r.Version,
@@ -112,16 +127,16 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten
 	}
 
 	if mod.privKey == nil {
-		mod.privKey = testPrivateKey
+		mod.privKey = creator.privKey
 	}
 	if mod.keyID == "" {
-		mod.keyID = testKeyID
+		mod.keyID = creator.keyID
 	}
 	if mod.originServerTS.IsZero() {
 		mod.originServerTS = time.Now()
 	}
 	if mod.origin == "" {
-		mod.origin = gomatrixserverlib.ServerName("localhost")
+		mod.origin = creator.srvName
 	}
 
 	var unsigned gomatrixserverlib.RawJSON
@@ -174,13 +189,14 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten
 // Add a new event to this room DAG. Not thread-safe.
 func (r *Room) InsertEvent(t *testing.T, he *gomatrixserverlib.HeaderedEvent) {
 	t.Helper()
-	// Add the event to the list of auth events
+	// Add the event to the list of auth/state events
 	r.events = append(r.events, he)
 	if he.StateKey() != nil {
 		err := r.authEvents.AddEvent(he.Unwrap())
 		if err != nil {
 			t.Fatalf("InsertEvent: failed to add event to auth events: %s", err)
 		}
+		r.currentState[he.Type()+" "+*he.StateKey()] = he
 	}
 }
 
@@ -188,6 +204,16 @@ func (r *Room) Events() []*gomatrixserverlib.HeaderedEvent {
 	return r.events
 }
 
+func (r *Room) CurrentState() []*gomatrixserverlib.HeaderedEvent {
+	events := make([]*gomatrixserverlib.HeaderedEvent, len(r.currentState))
+	i := 0
+	for _, e := range r.currentState {
+		events[i] = e
+		i++
+	}
+	return events
+}
+
 func (r *Room) CreateAndInsert(t *testing.T, creator *User, eventType string, content interface{}, mods ...eventModifier) *gomatrixserverlib.HeaderedEvent {
 	t.Helper()
 	he := r.CreateEvent(t, creator, eventType, content, mods...)
diff --git a/internal/test/slice.go b/test/slice.go
similarity index 100%
rename from internal/test/slice.go
rename to test/slice.go
diff --git a/test/base.go b/test/testrig/base.go
similarity index 92%
rename from test/base.go
rename to test/testrig/base.go
index 664442c03..facb49f3e 100644
--- a/test/base.go
+++ b/test/testrig/base.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package test
+package testrig
 
 import (
 	"errors"
@@ -24,22 +24,23 @@ import (
 
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/nats-io/nats.go"
 )
 
-func CreateBaseDendrite(t *testing.T, dbType DBType) (*base.BaseDendrite, func()) {
+func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) {
 	var cfg config.Dendrite
 	cfg.Defaults(false)
 	cfg.Global.JetStream.InMemory = true
 
 	switch dbType {
-	case DBTypePostgres:
+	case test.DBTypePostgres:
 		cfg.Global.Defaults(true)   // autogen a signing key
 		cfg.MediaAPI.Defaults(true) // autogen a media path
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
 		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
-		connStr, close := PrepareDBConnectionString(t, dbType)
+		connStr, close := test.PrepareDBConnectionString(t, dbType)
 		cfg.Global.DatabaseOptions = config.DatabaseOptions{
 			ConnectionString:       config.DataSource(connStr),
 			MaxOpenConnections:     10,
@@ -47,7 +48,7 @@ func CreateBaseDendrite(t *testing.T, dbType DBType) (*base.BaseDendrite, func()
 			ConnMaxLifetimeSeconds: 60,
 		}
 		return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), close
-	case DBTypeSQLite:
+	case test.DBTypeSQLite:
 		cfg.Defaults(true) // sets a sqlite db per component
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
diff --git a/test/jetstream.go b/test/testrig/jetstream.go
similarity index 98%
rename from test/jetstream.go
rename to test/testrig/jetstream.go
index 488c22beb..74cf95062 100644
--- a/test/jetstream.go
+++ b/test/testrig/jetstream.go
@@ -1,4 +1,4 @@
-package test
+package testrig
 
 import (
 	"encoding/json"
diff --git a/test/user.go b/test/user.go
index 41a66e1c4..0020098a5 100644
--- a/test/user.go
+++ b/test/user.go
@@ -15,22 +15,64 @@
 package test
 
 import (
+	"crypto/ed25519"
 	"fmt"
 	"sync/atomic"
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 var (
 	userIDCounter = int64(0)
+
+	serverName = gomatrixserverlib.ServerName("test")
+	keyID      = gomatrixserverlib.KeyID("ed25519:test")
+	privateKey = ed25519.NewKeyFromSeed([]byte{
+		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
+	})
+
+	// private keys that tests can use
+	PrivateKeyA = ed25519.NewKeyFromSeed([]byte{
+		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 77,
+	})
+	PrivateKeyB = ed25519.NewKeyFromSeed([]byte{
+		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 66,
+	})
 )
 
 type User struct {
 	ID string
+	// key ID and private key of the server who has this user, if known.
+	keyID   gomatrixserverlib.KeyID
+	privKey ed25519.PrivateKey
+	srvName gomatrixserverlib.ServerName
 }
 
-func NewUser() *User {
-	counter := atomic.AddInt64(&userIDCounter, 1)
-	u := &User{
-		ID: fmt.Sprintf("@%d:localhost", counter),
+type UserOpt func(*User)
+
+func WithSigningServer(srvName gomatrixserverlib.ServerName, keyID gomatrixserverlib.KeyID, privKey ed25519.PrivateKey) UserOpt {
+	return func(u *User) {
+		u.keyID = keyID
+		u.privKey = privKey
+		u.srvName = srvName
 	}
-	return u
+}
+
+func NewUser(t *testing.T, opts ...UserOpt) *User {
+	counter := atomic.AddInt64(&userIDCounter, 1)
+	var u User
+	for _, opt := range opts {
+		opt(&u)
+	}
+	if u.keyID == "" || u.srvName == "" || u.privKey == nil {
+		t.Logf("NewUser: missing signing server credentials; using default.")
+		WithSigningServer(serverName, keyID, privateKey)(&u)
+	}
+	u.ID = fmt.Sprintf("@%d:%s", counter, u.srvName)
+	t.Logf("NewUser: created user %s", u.ID)
+	return &u
 }
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index 5683fe067..5bee880d3 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -43,7 +43,7 @@ func Test_AccountData(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		db, close := mustCreateDatabase(t, dbType)
 		defer close()
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
 		assert.NoError(t, err)
 
@@ -74,7 +74,7 @@ func Test_Accounts(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		db, close := mustCreateDatabase(t, dbType)
 		defer close()
-		alice := test.NewUser()
+		alice := test.NewUser(t)
 		aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
 		assert.NoError(t, err)
 
@@ -128,7 +128,7 @@ func Test_Accounts(t *testing.T) {
 }
 
 func Test_Devices(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	localpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
 	assert.NoError(t, err)
 	deviceID := util.RandomString(8)
@@ -212,7 +212,7 @@ func Test_Devices(t *testing.T) {
 }
 
 func Test_KeyBackup(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	room := test.NewRoom(t, alice)
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
@@ -291,7 +291,7 @@ func Test_KeyBackup(t *testing.T) {
 }
 
 func Test_LoginToken(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		db, close := mustCreateDatabase(t, dbType)
 		defer close()
@@ -321,7 +321,7 @@ func Test_LoginToken(t *testing.T) {
 }
 
 func Test_OpenID(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	token := util.RandomString(24)
 
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
@@ -341,7 +341,7 @@ func Test_OpenID(t *testing.T) {
 }
 
 func Test_Profile(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
 	assert.NoError(t, err)
 
@@ -379,7 +379,7 @@ func Test_Profile(t *testing.T) {
 }
 
 func Test_Pusher(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
 	assert.NoError(t, err)
 
@@ -430,7 +430,7 @@ func Test_Pusher(t *testing.T) {
 }
 
 func Test_ThreePID(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
 	assert.NoError(t, err)
 
@@ -467,7 +467,7 @@ func Test_ThreePID(t *testing.T) {
 }
 
 func Test_Notification(t *testing.T) {
-	alice := test.NewUser()
+	alice := test.NewUser(t)
 	aliceLocalpart, _, err := gomatrixserverlib.SplitID('@', alice.ID)
 	assert.NoError(t, err)
 	room := test.NewRoom(t, alice)
diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index e614765a2..40e37c5d6 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -24,7 +24,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	internalTest "github.com/matrix-org/dendrite/internal/test"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/dendrite/userapi/inthttp"
@@ -135,7 +134,7 @@ func TestQueryProfile(t *testing.T) {
 	t.Run("HTTP API", func(t *testing.T) {
 		router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
 		userapi.AddInternalRoutes(router, userAPI)
-		apiURL, cancel := internalTest.ListenAndServe(t, router, false)
+		apiURL, cancel := test.ListenAndServe(t, router, false)
 		defer cancel()
 		httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{})
 		if err != nil {

From ac92e047728efc3d50d6dddbe392ca44afd63a38 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 17 May 2022 13:31:48 +0100
Subject: [PATCH 119/391] Remove debug logging

---
 federationapi/consumers/keychange.go                 | 5 +----
 federationapi/consumers/roomserver.go                | 2 --
 federationapi/storage/postgres/joined_hosts_table.go | 5 -----
 3 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go
index 95c9a7fdd..6d3cf0e46 100644
--- a/federationapi/consumers/keychange.go
+++ b/federationapi/consumers/keychange.go
@@ -120,7 +120,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
-	logrus.Infof("DEBUG: %v joined rooms for user %v", queryRes.RoomIDs, m.UserID)
+
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
@@ -129,9 +129,6 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	}
 
 	if len(destinations) == 0 {
-		logger.WithField("num_rooms", len(queryRes.RoomIDs)).Debug("user is in no federated rooms")
-		destinations, err = t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, false)
-		logrus.Infof("GetJoinedHostsForRooms exclude self=false -> %v %v", destinations, err)
 		return true
 	}
 	// Pack the EDU and marshal it
diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index 7a0816ff2..e50ec66ad 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -21,7 +21,6 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
-	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/federationapi/queue"
@@ -166,7 +165,6 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	// expressed as a delta against the current state.
 	// TODO(#290): handle EventIDMismatchError and recover the current state by
 	// talking to the roomserver
-	logrus.Infof("room %s adds joined hosts: %v removes %v", ore.Event.RoomID(), addsJoinedHosts, ore.RemovesStateEventIDs)
 	oldJoinedHosts, err := s.db.UpdateRoom(
 		s.ctx,
 		ore.Event.RoomID(),
diff --git a/federationapi/storage/postgres/joined_hosts_table.go b/federationapi/storage/postgres/joined_hosts_table.go
index bb6f6bfa3..5c95b72a8 100644
--- a/federationapi/storage/postgres/joined_hosts_table.go
+++ b/federationapi/storage/postgres/joined_hosts_table.go
@@ -24,7 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 )
 
 const joinedHostsSchema = `
@@ -112,7 +111,6 @@ func (s *joinedHostsStatements) InsertJoinedHosts(
 	roomID, eventID string,
 	serverName gomatrixserverlib.ServerName,
 ) error {
-	logrus.Debugf("FederationJoinedHosts: INSERT %v %v %v", roomID, eventID, serverName)
 	stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt)
 	_, err := stmt.ExecContext(ctx, roomID, eventID, serverName)
 	return err
@@ -121,7 +119,6 @@ func (s *joinedHostsStatements) InsertJoinedHosts(
 func (s *joinedHostsStatements) DeleteJoinedHosts(
 	ctx context.Context, txn *sql.Tx, eventIDs []string,
 ) error {
-	logrus.Debugf("FederationJoinedHosts: DELETE WITH EVENTS %v", eventIDs)
 	stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt)
 	_, err := stmt.ExecContext(ctx, pq.StringArray(eventIDs))
 	return err
@@ -130,7 +127,6 @@ func (s *joinedHostsStatements) DeleteJoinedHosts(
 func (s *joinedHostsStatements) DeleteJoinedHostsForRoom(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) error {
-	logrus.Debugf("FederationJoinedHosts: DELETE ALL IN ROOM %v", roomID)
 	stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsForRoomStmt)
 	_, err := stmt.ExecContext(ctx, roomID)
 	return err
@@ -211,7 +207,6 @@ func joinedHostsFromStmt(
 			ServerName:    gomatrixserverlib.ServerName(serverName),
 		})
 	}
-	logrus.Debugf("FederationJoinedHosts: SELECT %v => %+v", roomID, result)
 
 	return result, rows.Err()
 }

From b3162755a9053bbb30a83f00928ff0a0852ad32e Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Tue, 17 May 2022 15:53:08 +0100
Subject: [PATCH 120/391] bugfix: fix race condition when updating presence via
 /sync (#2470)

* bugfix: fix race condition when updating presence via /sync

Previously when presence is updated via /sync, we would send the presence update
asyncly via NATS. This created a race condition:
 - If the presence update is processed quickly, the /sync which triggered the presence
   update would see an online presence.
 - If the presence update was processed slowly, the /sync which triggered the presence
   update would see an offline presence.

This is the root cause behind the flakey sytest: 'User sees their own presence in a sync'.

The fix is to ensure we update the database/advance the stream position synchronously
for local users.

* Bugfix for test
---
 syncapi/consumers/presence.go    | 25 +++++++++------
 syncapi/sync/requestpool.go      | 15 ++++++++-
 syncapi/sync/requestpool_test.go |  8 +++++
 syncapi/syncapi.go               | 20 ++++++------
 syncapi/syncapi_test.go          | 55 ++++++++++++++++++++++++++++++++
 5 files changed, 103 insertions(+), 20 deletions(-)

diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go
index 388c08ff4..bfd72d604 100644
--- a/syncapi/consumers/presence.go
+++ b/syncapi/consumers/presence.go
@@ -138,9 +138,12 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 	presence := msg.Header.Get("presence")
 	timestamp := msg.Header.Get("last_active_ts")
 	fromSync, _ := strconv.ParseBool(msg.Header.Get("from_sync"))
-
 	logrus.Debugf("syncAPI received presence event: %+v", msg.Header)
 
+	if fromSync { // do not process local presence changes; we already did this synchronously.
+		return true
+	}
+
 	ts, err := strconv.Atoi(timestamp)
 	if err != nil {
 		return true
@@ -151,15 +154,19 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 		newMsg := msg.Header.Get("status_msg")
 		statusMsg = &newMsg
 	}
-	// OK is already checked, so no need to do it again
+	// already checked, so no need to check error
 	p, _ := types.PresenceFromString(presence)
-	pos, err := s.db.UpdatePresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
-	if err != nil {
-		return true
-	}
-
-	s.stream.Advance(pos)
-	s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID)
 
+	s.EmitPresence(ctx, userID, p, statusMsg, ts, fromSync)
 	return true
 }
+
+func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
+	pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
+	if err != nil {
+		logrus.WithError(err).WithField("user", userID).WithField("presence", presence).Warn("failed to updated presence for user")
+		return
+	}
+	s.stream.Advance(pos)
+	s.notifier.OnNewPresence(types.StreamingToken{PresencePosition: pos}, userID)
+}
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index fdf46cdde..ad151f70b 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -53,19 +53,24 @@ type RequestPool struct {
 	streams  *streams.Streams
 	Notifier *notifier.Notifier
 	producer PresencePublisher
+	consumer PresenceConsumer
 }
 
 type PresencePublisher interface {
 	SendPresence(userID string, presence types.Presence, statusMsg *string) error
 }
 
+type PresenceConsumer interface {
+	EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool)
+}
+
 // NewRequestPool makes a new RequestPool
 func NewRequestPool(
 	db storage.Database, cfg *config.SyncAPI,
 	userAPI userapi.SyncUserAPI, keyAPI keyapi.SyncKeyAPI,
 	rsAPI roomserverAPI.SyncRoomserverAPI,
 	streams *streams.Streams, notifier *notifier.Notifier,
-	producer PresencePublisher, enableMetrics bool,
+	producer PresencePublisher, consumer PresenceConsumer, enableMetrics bool,
 ) *RequestPool {
 	if enableMetrics {
 		prometheus.MustRegister(
@@ -83,6 +88,7 @@ func NewRequestPool(
 		streams:  streams,
 		Notifier: notifier,
 		producer: producer,
+		consumer: consumer,
 	}
 	go rp.cleanLastSeen()
 	go rp.cleanPresence(db, time.Minute*5)
@@ -160,6 +166,13 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
 		logrus.WithError(err).Error("Unable to publish presence message from sync")
 		return
 	}
+
+	// now synchronously update our view of the world. It's critical we do this before calculating
+	// the /sync response else we may not return presence: online immediately.
+	rp.consumer.EmitPresence(
+		context.Background(), userID, presenceID, newPresence.ClientFields.StatusMsg,
+		int(gomatrixserverlib.AsTimestamp(time.Now())), true,
+	)
 }
 
 func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) {
diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go
index 5e52bc7c9..0c7209521 100644
--- a/syncapi/sync/requestpool_test.go
+++ b/syncapi/sync/requestpool_test.go
@@ -38,6 +38,12 @@ func (d dummyDB) MaxStreamPositionForPresence(ctx context.Context) (types.Stream
 	return 0, nil
 }
 
+type dummyConsumer struct{}
+
+func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
+
+}
+
 func TestRequestPool_updatePresence(t *testing.T) {
 	type args struct {
 		presence string
@@ -45,6 +51,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
 		sleep    time.Duration
 	}
 	publisher := &dummyPublisher{}
+	consumer := &dummyConsumer{}
 	syncMap := sync.Map{}
 
 	tests := []struct {
@@ -101,6 +108,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
 	rp := &RequestPool{
 		presence: &syncMap,
 		producer: publisher,
+		consumer: consumer,
 		cfg: &config.SyncAPI{
 			Matrix: &config.Global{
 				JetStream: config.JetStream{
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index d8bacb2da..92db18d56 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -64,8 +64,17 @@ func AddPublicRoutes(
 		Topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
 		JetStream: js,
 	}
+	presenceConsumer := consumers.NewPresenceConsumer(
+		base.ProcessContext, cfg, js, natsClient, syncDB,
+		notifier, streams.PresenceStreamProvider,
+		userAPI,
+	)
 
-	requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, base.EnableMetrics)
+	requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, base.EnableMetrics)
+
+	if err = presenceConsumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start presence consumer")
+	}
 
 	userAPIStreamEventProducer := &producers.UserAPIStreamEventProducer{
 		JetStream: js,
@@ -131,15 +140,6 @@ func AddPublicRoutes(
 		logrus.WithError(err).Panicf("failed to start receipts consumer")
 	}
 
-	presenceConsumer := consumers.NewPresenceConsumer(
-		base.ProcessContext, cfg, js, natsClient, syncDB,
-		notifier, streams.PresenceStreamProvider,
-		userAPI,
-	)
-	if err = presenceConsumer.Start(); err != nil {
-		logrus.WithError(err).Panicf("failed to start presence consumer")
-	}
-
 	routing.Setup(
 		base.PublicClientAPIMux, requestPool, syncDB, userAPI,
 		rsAPI, cfg, base.Caches,
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 5ecfd8772..3ce7c64b7 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -19,6 +19,7 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
+	"github.com/tidwall/gjson"
 )
 
 type syncRoomserverAPI struct {
@@ -256,6 +257,60 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 	}
 }
 
+// Test that if we hit /sync we get back presence: online, regardless of whether messages get delivered
+// via NATS. Regression test for a flakey test "User sees their own presence in a sync"
+func TestSyncAPIUpdatePresenceImmediately(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testSyncAPIUpdatePresenceImmediately(t, dbType)
+	})
+}
+
+func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) {
+	user := test.NewUser(t)
+	alice := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      user.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "Alice",
+		AccountType: userapi.AccountTypeUser,
+	}
+
+	base, close := testrig.CreateBaseDendrite(t, dbType)
+	base.Cfg.Global.Presence.EnableOutbound = true
+	base.Cfg.Global.Presence.EnableInbound = true
+	defer close()
+
+	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, &syncKeyAPI{})
+	w := httptest.NewRecorder()
+	base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+		"access_token": alice.AccessToken,
+		"timeout":      "0",
+		"set_presence": "online",
+	})))
+	if w.Code != 200 {
+		t.Fatalf("got HTTP %d want %d", w.Code, 200)
+	}
+	var res types.Response
+	if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+		t.Errorf("failed to decode response body: %s", err)
+	}
+	if len(res.Presence.Events) != 1 {
+		t.Fatalf("expected 1 presence events, got: %+v", res.Presence.Events)
+	}
+	if res.Presence.Events[0].Sender != alice.UserID {
+		t.Errorf("sender: got %v want %v", res.Presence.Events[0].Sender, alice.UserID)
+	}
+	if res.Presence.Events[0].Type != "m.presence" {
+		t.Errorf("type: got %v want %v", res.Presence.Events[0].Type, "m.presence")
+	}
+	if gjson.ParseBytes(res.Presence.Events[0].Content).Get("presence").Str != "online" {
+		t.Errorf("content: not online,  got %v", res.Presence.Events[0].Content)
+	}
+
+}
+
 func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
 	result := make([]*nats.Msg, len(input))
 	for i, ev := range input {

From f321a7d55ea75e6a5276cd88eddcbbc82ceeaaeb Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 18 May 2022 15:17:23 +0200
Subject: [PATCH 121/391] Really SKIP_NODB (#2472)

* Really SKIP_NODB

* Use fatalError in createLocalDB

* Check if createdb exists

* Revert change

* Remove !Quiet
---
 test/db.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/db.go b/test/db.go
index a1754cd08..c7cb919f6 100644
--- a/test/db.go
+++ b/test/db.go
@@ -44,8 +44,9 @@ func fatalError(t *testing.T, format string, args ...interface{}) {
 }
 
 func createLocalDB(t *testing.T, dbName string) {
-	if !Quiet {
-		t.Log("Note: tests require a postgres install accessible to the current user")
+	if _, err := exec.LookPath("createdb"); err != nil {
+		fatalError(t, "Note: tests require a postgres install accessible to the current user")
+		return
 	}
 	createDB := exec.Command("createdb", dbName)
 	if !Quiet {
@@ -63,6 +64,9 @@ func createRemoteDB(t *testing.T, dbName, user, connStr string) {
 	if err != nil {
 		fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err)
 	}
+	if err = db.Ping(); err != nil {
+		fatalError(t, "failed to open postgres conn with connstr=%s : %s", connStr, err)
+	}
 	_, err = db.Exec(fmt.Sprintf(`CREATE DATABASE %s;`, dbName))
 	if err != nil {
 		pqErr, ok := err.(*pq.Error)

From 21dd5a7176e52d018b91854db273424e4430af7b Mon Sep 17 00:00:00 2001
From: kegsay <kegan@matrix.org>
Date: Thu, 19 May 2022 09:00:56 +0100
Subject: [PATCH 122/391] syncapi: don't return early for no-op incremental
 syncs (#2473)

* syncapi: don't return early for no-op incremental syncs

Comments explain why, but basically it's an inefficient use
of bandwidth and some sytests rely on /sync to block.

* Honour timeouts

* Actually return a response with timeout=0
---
 syncapi/sync/requestpool.go | 246 ++++++++++++++++++++----------------
 syncapi/types/types.go      |  13 ++
 2 files changed, 149 insertions(+), 110 deletions(-)

diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index ad151f70b..7b9526b53 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -251,125 +251,151 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 	waitingSyncRequests.Inc()
 	defer waitingSyncRequests.Dec()
 
-	currentPos := rp.Notifier.CurrentPosition()
+	// loop until we get some data
+	for {
+		startTime := time.Now()
+		currentPos := rp.Notifier.CurrentPosition()
 
-	if !rp.shouldReturnImmediately(syncReq, currentPos) {
-		timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above
-		defer timer.Stop()
+		// if the since token matches the current positions, wait via the notifier
+		if !rp.shouldReturnImmediately(syncReq, currentPos) {
+			timer := time.NewTimer(syncReq.Timeout) // case of timeout=0 is handled above
+			defer timer.Stop()
 
-		userStreamListener := rp.Notifier.GetListener(*syncReq)
-		defer userStreamListener.Close()
+			userStreamListener := rp.Notifier.GetListener(*syncReq)
+			defer userStreamListener.Close()
 
-		giveup := func() util.JSONResponse {
-			syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached")
-			syncReq.Response.NextBatch = syncReq.Since
-			// We should always try to include OTKs in sync responses, otherwise clients might upload keys
-			// even if that's not required. See also:
-			// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
-			// Only try to get OTKs if the context isn't already done.
-			if syncReq.Context.Err() == nil {
-				err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
-				if err != nil && err != context.Canceled {
-					syncReq.Log.WithError(err).Warn("failed to get OTK counts")
+			giveup := func() util.JSONResponse {
+				syncReq.Log.Debugln("Responding to sync since client gave up or timeout was reached")
+				syncReq.Response.NextBatch = syncReq.Since
+				// We should always try to include OTKs in sync responses, otherwise clients might upload keys
+				// even if that's not required. See also:
+				// https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281
+				// Only try to get OTKs if the context isn't already done.
+				if syncReq.Context.Err() == nil {
+					err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response)
+					if err != nil && err != context.Canceled {
+						syncReq.Log.WithError(err).Warn("failed to get OTK counts")
+					}
+				}
+				return util.JSONResponse{
+					Code: http.StatusOK,
+					JSON: syncReq.Response,
 				}
 			}
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: syncReq.Response,
+
+			select {
+			case <-syncReq.Context.Done(): // Caller gave up
+				return giveup()
+
+			case <-timer.C: // Timeout reached
+				return giveup()
+
+			case <-userStreamListener.GetNotifyChannel(syncReq.Since):
+				syncReq.Log.Debugln("Responding to sync after wake-up")
+				currentPos.ApplyUpdates(userStreamListener.GetSyncPosition())
+			}
+		} else {
+			syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
+		}
+
+		if syncReq.Since.IsEmpty() {
+			// Complete sync
+			syncReq.Response.NextBatch = types.StreamingToken{
+				PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+				PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
+			}
+		} else {
+			// Incremental sync
+			syncReq.Response.NextBatch = types.StreamingToken{
+				PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.PDUPosition, currentPos.PDUPosition,
+				),
+				TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.TypingPosition, currentPos.TypingPosition,
+				),
+				ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
+				),
+				InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.InvitePosition, currentPos.InvitePosition,
+				),
+				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
+				),
+				AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
+				),
+				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
+				),
+				DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
+				),
+				PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
+					syncReq.Context, syncReq,
+					syncReq.Since.PresencePosition, currentPos.PresencePosition,
+				),
+			}
+			// it's possible for there to be no updates for this user even though since < current pos,
+			// e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op
+			// response immediately, so let's try this again but pretend they bumped their since token.
+			// If the incremental sync was processed very quickly then we expect the next loop to block
+			// with a notifier, but if things are slow it's entirely possible that currentPos is no
+			// longer the current position so we will hit this code path again. We need to do this and
+			// not return a no-op response because:
+			// - It's an inefficient use of bandwidth.
+			// - Some sytests which test 'waking up' sync rely on some sync requests to block, which
+			//   they weren't always doing, resulting in flakey tests.
+			if !syncReq.Response.HasUpdates() {
+				syncReq.Since = currentPos
+				// do not loop again if the ?timeout= is 0 as that means "return immediately"
+				if syncReq.Timeout > 0 {
+					syncReq.Timeout = syncReq.Timeout - time.Since(startTime)
+					if syncReq.Timeout < 0 {
+						syncReq.Timeout = 0
+					}
+					continue
+				}
 			}
 		}
 
-		select {
-		case <-syncReq.Context.Done(): // Caller gave up
-			return giveup()
-
-		case <-timer.C: // Timeout reached
-			return giveup()
-
-		case <-userStreamListener.GetNotifyChannel(syncReq.Since):
-			syncReq.Log.Debugln("Responding to sync after wake-up")
-			currentPos.ApplyUpdates(userStreamListener.GetSyncPosition())
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: syncReq.Response,
 		}
-	} else {
-		syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
-	}
-
-	if syncReq.Since.IsEmpty() {
-		// Complete sync
-		syncReq.Response.NextBatch = types.StreamingToken{
-			PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-			PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
-				syncReq.Context, syncReq,
-			),
-		}
-	} else {
-		// Incremental sync
-		syncReq.Response.NextBatch = types.StreamingToken{
-			PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.PDUPosition, currentPos.PDUPosition,
-			),
-			TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.TypingPosition, currentPos.TypingPosition,
-			),
-			ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
-			),
-			InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.InvitePosition, currentPos.InvitePosition,
-			),
-			SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
-			),
-			AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
-			),
-			NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
-			),
-			DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
-			),
-			PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
-				syncReq.Context, syncReq,
-				syncReq.Since.PresencePosition, currentPos.PresencePosition,
-			),
-		}
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: syncReq.Response,
 	}
 }
 
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index ba6b4f8cd..159fa08b6 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -350,6 +350,19 @@ type Response struct {
 	DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
 }
 
+func (r *Response) HasUpdates() bool {
+	// purposefully exclude DeviceListsOTKCount as we always include them
+	return (len(r.AccountData.Events) > 0 ||
+		len(r.Presence.Events) > 0 ||
+		len(r.Rooms.Invite) > 0 ||
+		len(r.Rooms.Join) > 0 ||
+		len(r.Rooms.Leave) > 0 ||
+		len(r.Rooms.Peek) > 0 ||
+		len(r.ToDevice.Events) > 0 ||
+		len(r.DeviceLists.Changed) > 0 ||
+		len(r.DeviceLists.Left) > 0)
+}
+
 // NewResponse creates an empty response with initialised maps.
 func NewResponse() *Response {
 	res := Response{}

From a53c9300aa501ccf31658fe832f5e4565441f8f4 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 20 May 2022 13:27:11 +0200
Subject: [PATCH 123/391] Fix `/deactivate` (#2474)

* Fix /deactivate

* Update test to correctly check the expected response
---
 clientapi/auth/user_interactive.go      |  7 ++---
 clientapi/auth/user_interactive_test.go | 35 +++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go
index 6caf7dcdc..82ecf674c 100644
--- a/clientapi/auth/user_interactive.go
+++ b/clientapi/auth/user_interactive.go
@@ -102,8 +102,7 @@ type userInteractiveFlow struct {
 // the user already has a valid access token, but we want to double-check
 // that it isn't stolen by re-authenticating them.
 type UserInteractive struct {
-	Completed []string
-	Flows     []userInteractiveFlow
+	Flows []userInteractiveFlow
 	// Map of login type to implementation
 	Types map[string]Type
 	// Map of session ID to completed login types, will need to be extended in future
@@ -116,7 +115,6 @@ func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI)
 		Config:               cfg,
 	}
 	return &UserInteractive{
-		Completed: []string{},
 		Flows: []userInteractiveFlow{
 			{
 				Stages: []string{typePassword.Name()},
@@ -140,7 +138,6 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
 
 func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
 	// TODO: Handle multi-stage flows
-	u.Completed = append(u.Completed, authType)
 	delete(u.Sessions, sessionID)
 }
 
@@ -157,7 +154,7 @@ func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse {
 	return &util.JSONResponse{
 		Code: 401,
 		JSON: Challenge{
-			Completed: u.Completed,
+			Completed: u.Sessions[sessionID],
 			Flows:     u.Flows,
 			Session:   sessionID,
 			Params:    make(map[string]interface{}),
diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go
index 262e48103..001b1a6d4 100644
--- a/clientapi/auth/user_interactive_test.go
+++ b/clientapi/auth/user_interactive_test.go
@@ -187,3 +187,38 @@ func TestUserInteractivePasswordBadLogin(t *testing.T) {
 		}
 	}
 }
+
+func TestUserInteractive_AddCompletedStage(t *testing.T) {
+	tests := []struct {
+		name      string
+		sessionID string
+	}{
+		{
+			name:      "first user",
+			sessionID: util.RandomString(8),
+		},
+		{
+			name:      "second user",
+			sessionID: util.RandomString(8),
+		},
+		{
+			name:      "third user",
+			sessionID: util.RandomString(8),
+		},
+	}
+	u := setup()
+	ctx := context.Background()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, resp := u.Verify(ctx, []byte("{}"), nil)
+			challenge, ok := resp.JSON.(Challenge)
+			if !ok {
+				t.Fatalf("expected a Challenge, got %T", resp.JSON)
+			}
+			if len(challenge.Completed) > 0 {
+				t.Fatalf("expected 0 completed stages, got %d", len(challenge.Completed))
+			}
+			u.AddCompletedStage(tt.sessionID, "")
+		})
+	}
+}

From 028963af1ca44c3b532b8e775ff5d8ca08f0c79d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 23 May 2022 14:26:56 +0100
Subject: [PATCH 124/391] Update to matrix-org/gomatrixserverlib#311 (#2479)

* Update to matrix-org/gomatrixserverlib#311

* Update to squash commit
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index ecdeb77fa..815aee9a3 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220523132611-f7ea99978d39
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 145c2a04c..812aac610 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433 h1:nwAlThHGPI2EAAJklXvgMcdhXF6ZiHp60+fmaYMoaDA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220513103617-eee8fd528433/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220523132611-f7ea99978d39 h1:6hYFlWmy8GENIvYyKmjZioCawgwMjjOvY2HZZGHR4aY=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220523132611-f7ea99978d39/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From b5a497a0c038c713d08834dc8fe453f77c412e9e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 23 May 2022 14:54:43 +0100
Subject: [PATCH 125/391] Allow defers to run in `TestMain` in federation API
 tests

---
 federationapi/federationapi_keys_test.go | 106 ++++++++++++-----------
 1 file changed, 54 insertions(+), 52 deletions(-)

diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go
index 31e9a4c73..73f737717 100644
--- a/federationapi/federationapi_keys_test.go
+++ b/federationapi/federationapi_keys_test.go
@@ -55,60 +55,62 @@ var servers = map[string]*server{
 func TestMain(m *testing.M) {
 	// Set up the server key API for each "server" that we
 	// will use in our tests.
-	for _, s := range servers {
-		// Generate a new key.
-		_, testPriv, err := ed25519.GenerateKey(nil)
-		if err != nil {
-			panic("can't generate identity key: " + err.Error())
+	os.Exit(func() int {
+		for _, s := range servers {
+			// Generate a new key.
+			_, testPriv, err := ed25519.GenerateKey(nil)
+			if err != nil {
+				panic("can't generate identity key: " + err.Error())
+			}
+
+			// Create a new cache but don't enable prometheus!
+			s.cache, err = caching.NewInMemoryLRUCache(false)
+			if err != nil {
+				panic("can't create cache: " + err.Error())
+			}
+
+			// Create a temporary directory for JetStream.
+			d, err := ioutil.TempDir("./", "jetstream*")
+			if err != nil {
+				panic(err)
+			}
+			defer os.RemoveAll(d)
+
+			// Draw up just enough Dendrite config for the server key
+			// API to work.
+			cfg := &config.Dendrite{}
+			cfg.Defaults(true)
+			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
+			cfg.Global.PrivateKey = testPriv
+			cfg.Global.JetStream.InMemory = true
+			cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
+			cfg.Global.JetStream.StoragePath = config.Path(d)
+			cfg.Global.KeyID = serverKeyID
+			cfg.Global.KeyValidityPeriod = s.validity
+			cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
+			s.config = &cfg.FederationAPI
+
+			// Create a transport which redirects federation requests to
+			// the mock round tripper. Since we're not *really* listening for
+			// federation requests then this will return the key instead.
+			transport := &http.Transport{}
+			transport.RegisterProtocol("matrix", &MockRoundTripper{})
+
+			// Create the federation client.
+			s.fedclient = gomatrixserverlib.NewFederationClient(
+				s.config.Matrix.ServerName, serverKeyID, testPriv,
+				gomatrixserverlib.WithTransport(transport),
+			)
+
+			// Finally, build the server key APIs.
+			sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
+			s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
 		}
 
-		// Create a new cache but don't enable prometheus!
-		s.cache, err = caching.NewInMemoryLRUCache(false)
-		if err != nil {
-			panic("can't create cache: " + err.Error())
-		}
-
-		// Create a temporary directory for JetStream.
-		d, err := ioutil.TempDir("./", "jetstream*")
-		if err != nil {
-			panic(err)
-		}
-		defer os.RemoveAll(d)
-
-		// Draw up just enough Dendrite config for the server key
-		// API to work.
-		cfg := &config.Dendrite{}
-		cfg.Defaults(true)
-		cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
-		cfg.Global.PrivateKey = testPriv
-		cfg.Global.JetStream.InMemory = true
-		cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
-		cfg.Global.JetStream.StoragePath = config.Path(d)
-		cfg.Global.KeyID = serverKeyID
-		cfg.Global.KeyValidityPeriod = s.validity
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
-		s.config = &cfg.FederationAPI
-
-		// Create a transport which redirects federation requests to
-		// the mock round tripper. Since we're not *really* listening for
-		// federation requests then this will return the key instead.
-		transport := &http.Transport{}
-		transport.RegisterProtocol("matrix", &MockRoundTripper{})
-
-		// Create the federation client.
-		s.fedclient = gomatrixserverlib.NewFederationClient(
-			s.config.Matrix.ServerName, serverKeyID, testPriv,
-			gomatrixserverlib.WithTransport(transport),
-		)
-
-		// Finally, build the server key APIs.
-		sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
-		s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
-	}
-
-	// Now that we have built our server key APIs, start the
-	// rest of the tests.
-	os.Exit(m.Run())
+		// Now that we have built our server key APIs, start the
+		// rest of the tests.
+		return m.Run()
+	}())
 }
 
 type MockRoundTripper struct{}

From 8a7567c652fb490f8d6d4e25fc2b282fd4d8c76c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 23 May 2022 17:49:48 +0100
Subject: [PATCH 126/391] Fix control flow related to joining restricted v8/v9
 rooms (update to matrix-org/gomatrixserverlib#312)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 815aee9a3..d00a0723e 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220523132611-f7ea99978d39
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220523164922-dfd9bd7c03fb
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 812aac610..45ba13c56 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220523132611-f7ea99978d39 h1:6hYFlWmy8GENIvYyKmjZioCawgwMjjOvY2HZZGHR4aY=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220523132611-f7ea99978d39/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220523164922-dfd9bd7c03fb h1:MRhaR1IKYdizM2vECOcJlleExLEwy4KAvQ6v73QQRnA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220523164922-dfd9bd7c03fb/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 447226790153afc47b62df4d81f66ec7129dca90 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 23 May 2022 17:54:01 +0100
Subject: [PATCH 127/391] Preserve content from `/make_join` as well as
 supplied content in the request (#2481)

---
 federationapi/internal/perform.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 7ccd68ef0..c176a6ea8 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -166,7 +166,8 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	if content == nil {
 		content = map[string]interface{}{}
 	}
-	content["membership"] = "join"
+	_ = json.Unmarshal(respMakeJoin.JoinEvent.Content, &content)
+	content["membership"] = gomatrixserverlib.Join
 	if err = respMakeJoin.JoinEvent.SetContent(content); err != nil {
 		return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err)
 	}

From 20844942a8923752c7ec2ca1cbcd37b5c36de875 Mon Sep 17 00:00:00 2001
From: Aaron Ogle <geekgonecrazy@users.noreply.github.com>
Date: Tue, 24 May 2022 03:07:33 -0500
Subject: [PATCH 128/391] Handle invite in appservice roomserver consumer
 (#2476)

Signed-off-by: Aaron Ogle <aaron@geekgonecrazy.com>
---
 appservice/consumers/roomserver.go | 49 ++++++++++++++++++------------
 1 file changed, 29 insertions(+), 20 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index e406e88a7..37d4ef9c2 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -83,29 +83,38 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 		return true
 	}
 
-	if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
-		return true
-	}
+	log.WithFields(log.Fields{
+		"type": output.Type,
+	}).Debug("Got a message in OutputRoomEventConsumer")
 
-	newEventID := output.NewRoomEvent.Event.EventID()
-	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
-	events = append(events, output.NewRoomEvent.Event)
-	if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
-		eventsReq := &api.QueryEventsByIDRequest{
-			EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
-		}
-		eventsRes := &api.QueryEventsByIDResponse{}
-		for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
-			if eventID != newEventID {
-				eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+	events := []*gomatrixserverlib.HeaderedEvent{}
+	if output.Type == api.OutputTypeNewRoomEvent && output.NewRoomEvent != nil {
+		newEventID := output.NewRoomEvent.Event.EventID()
+		events = append(events, output.NewRoomEvent.Event)
+		if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
+			eventsReq := &api.QueryEventsByIDRequest{
+				EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
+			}
+			eventsRes := &api.QueryEventsByIDResponse{}
+			for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
+				if eventID != newEventID {
+					eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+				}
+			}
+			if len(eventsReq.EventIDs) > 0 {
+				if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+					return false
+				}
+				events = append(events, eventsRes.Events...)
 			}
 		}
-		if len(eventsReq.EventIDs) > 0 {
-			if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
-				return false
-			}
-			events = append(events, eventsRes.Events...)
-		}
+	} else if output.Type == api.OutputTypeNewInviteEvent && output.NewInviteEvent != nil {
+		events = append(events, output.NewInviteEvent.Event)
+	} else {
+		log.WithFields(log.Fields{
+			"type": output.Type,
+		}).Debug("appservice OutputRoomEventConsumer ignoring event", string(msg.Data))
+		return true
 	}
 
 	// Send event to any relevant application services

From c83837e684347563c068c5ca4849090695bc97d0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 24 May 2022 09:11:23 +0100
Subject: [PATCH 129/391] Strip `join_authorised_by_users_server` when updating
 membership events using `/rooms/{roomID}/state` (#2482)

---
 clientapi/routing/sendevent.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index 5f84739d0..b8a7fe98f 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -104,6 +104,13 @@ func SendEvent(
 		return *resErr
 	}
 
+	// If we're sending a membership update, make sure to strip the authorised
+	// via key if it is present, otherwise other servers won't be able to auth
+	// the event if the room is set to the "restricted" join rule.
+	if eventType == gomatrixserverlib.MRoomMember {
+		delete(r, "join_authorised_by_users_server")
+	}
+
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{

From 7379b02b703ff6cd56a0eb806aeb5249b09232c0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 24 May 2022 10:10:25 +0100
Subject: [PATCH 130/391] Give PL100 to invited users in `trusted_private_chat`
 preset (#2485)

---
 clientapi/routing/createroom.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index d40d84a79..3f92b7ba6 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -245,7 +245,9 @@ func createRoom(
 	case presetTrustedPrivateChat:
 		joinRuleContent.JoinRule = gomatrixserverlib.Invite
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
-		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
+		for _, invitee := range r.Invite {
+			powerLevelContent.Users[invitee] = 100
+		}
 	case presetPublicChat:
 		joinRuleContent.JoinRule = gomatrixserverlib.Public
 		historyVisibilityContent.HistoryVisibility = historyVisibilityShared

From ead0112aa15d6988aaab71aa8cd5aab0e70f7bf0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 24 May 2022 10:22:26 +0100
Subject: [PATCH 131/391] Fix `join_authorised_via_users_server` key name in
 `SendEvent`

---
 clientapi/routing/sendevent.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index b8a7fe98f..70bf72f88 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -108,7 +108,7 @@ func SendEvent(
 	// via key if it is present, otherwise other servers won't be able to auth
 	// the event if the room is set to the "restricted" join rule.
 	if eventType == gomatrixserverlib.MRoomMember {
-		delete(r, "join_authorised_by_users_server")
+		delete(r, "join_authorised_via_users_server")
 	}
 
 	evTime, err := httputil.ParseTSParam(req)

From d621dd2986fa0b8cce9d164a7249456d0be47c81 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 24 May 2022 11:08:28 +0100
Subject: [PATCH 132/391] Allow authing invited user in rooms with
 `"restricted"` join rule (update to matrix-org/gomatrixserverlib#313)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index d00a0723e..89ff04f63 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220523164922-dfd9bd7c03fb
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 45ba13c56..3ffd52182 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220523164922-dfd9bd7c03fb h1:MRhaR1IKYdizM2vECOcJlleExLEwy4KAvQ6v73QQRnA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220523164922-dfd9bd7c03fb/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c h1:J9krMtVgo4mV/G+mRA1u3GL6nNxdNnuPcs891uIQGic=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 81843e8836e6f8f334c7b5fd1433c427c10a9443 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 25 May 2022 10:05:30 +0100
Subject: [PATCH 133/391] Restricted join support on `/make_join`, `/send_join`
 (#2478)

* Add `QueryRestrictedJoinAllowed`

* Add `Resident` flag to `QueryRestrictedJoinAllowedResponse`

* Check restricted joins on federation API

* Return `Restricted` to determine if the room was restricted or not

* Populate `AuthorisedVia` properly

* Sign the event on `/send_join`, return it in the `/send_join` response in the `"event"` key

* Kick back joins with invalid authorising user IDs, use event from `"event"` key if returned in `RespSendJoin`

* Use invite helper in `QueryRestrictedJoinAllowed`

* Only use users with the power to invite, change error bubbling a bit

* Placate the almighty linter

One day I will nuke `gocyclo` from orbit and everything in the world will be much better for it.

* Review comments
---
 clientapi/jsonerror/jsonerror.go            |   6 +
 federationapi/internal/perform.go           |  30 ++++-
 federationapi/routing/join.go               | 122 ++++++++++++++++++-
 roomserver/api/api.go                       |   1 +
 roomserver/api/api_trace.go                 |  10 ++
 roomserver/api/query.go                     |  20 ++++
 roomserver/internal/perform/perform_join.go |  34 ++++++
 roomserver/internal/query/query.go          | 126 ++++++++++++++++++++
 roomserver/inthttp/client.go                |  11 ++
 roomserver/inthttp/server.go                |  13 ++
 10 files changed, 369 insertions(+), 4 deletions(-)

diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go
index 1fc1c0c01..70bac61dc 100644
--- a/clientapi/jsonerror/jsonerror.go
+++ b/clientapi/jsonerror/jsonerror.go
@@ -154,6 +154,12 @@ func MissingParam(msg string) *MatrixError {
 	return &MatrixError{"M_MISSING_PARAM", msg}
 }
 
+// UnableToAuthoriseJoin is an error that is returned when a server can't
+// determine whether to allow a restricted join or not.
+func UnableToAuthoriseJoin(msg string) *MatrixError {
+	return &MatrixError{"M_UNABLE_TO_AUTHORISE_JOIN", msg}
+}
+
 // LeaveServerNoticeError is an error returned when trying to reject an invite
 // for a server notice room.
 func LeaveServerNoticeError() *MatrixError {
diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index c176a6ea8..970a64653 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -210,10 +210,18 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	}
 	r.statistics.ForServer(serverName).Success()
 
-	authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
+	// If the remote server returned an event in the "event" key of
+	// the send_join request then we should use that instead. It may
+	// contain signatures that we don't know about.
+	if respSendJoin.Event != nil && isWellFormedMembershipEvent(
+		respSendJoin.Event, roomID, userID, r.cfg.Matrix.ServerName,
+	) {
+		event = respSendJoin.Event
+	}
 
 	// Sanity-check the join response to ensure that it has a create
 	// event, that the room version is known, etc.
+	authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
 	if err = sanityCheckAuthChain(authEvents); err != nil {
 		return fmt.Errorf("sanityCheckAuthChain: %w", err)
 	}
@@ -271,6 +279,26 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	return nil
 }
 
+// isWellFormedMembershipEvent returns true if the event looks like a legitimate
+// membership event.
+func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string, origin gomatrixserverlib.ServerName) bool {
+	if membership, err := event.Membership(); err != nil {
+		return false
+	} else if membership != gomatrixserverlib.Join {
+		return false
+	}
+	if event.RoomID() != roomID {
+		return false
+	}
+	if event.Origin() != origin {
+		return false
+	}
+	if !event.StateKeyEquals(userID) {
+		return false
+	}
+	return true
+}
+
 // PerformOutboundPeekRequest implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformOutboundPeek(
 	ctx context.Context,
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 767699728..9f8d94105 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"sort"
@@ -103,6 +104,16 @@ func MakeJoin(
 		}
 	}
 
+	// Check if the restricted join is allowed. If the room doesn't
+	// support restricted joins then this is effectively a no-op.
+	res, authorisedVia, err := checkRestrictedJoin(httpReq, rsAPI, verRes.RoomVersion, roomID, userID)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("checkRestrictedJoin failed")
+		return jsonerror.InternalServerError()
+	} else if res != nil {
+		return *res
+	}
+
 	// Try building an event for the server
 	builder := gomatrixserverlib.EventBuilder{
 		Sender:   userID,
@@ -110,8 +121,11 @@ func MakeJoin(
 		Type:     "m.room.member",
 		StateKey: &userID,
 	}
-	err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join})
-	if err != nil {
+	content := gomatrixserverlib.MemberContent{
+		Membership:    gomatrixserverlib.Join,
+		AuthorisedVia: authorisedVia,
+	}
+	if err = builder.SetContent(content); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("builder.SetContent failed")
 		return jsonerror.InternalServerError()
 	}
@@ -161,6 +175,7 @@ func MakeJoin(
 // SendJoin implements the /send_join API
 // The make-join send-join dance makes much more sense as a single
 // flow so the cyclomatic complexity is high:
+// nolint:gocyclo
 func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
@@ -314,6 +329,40 @@ func SendJoin(
 		}
 	}
 
+	// If the membership content contains a user ID for a server that is not
+	// ours then we should kick it back.
+	var memberContent gomatrixserverlib.MemberContent
+	if err := json.Unmarshal(event.Content(), &memberContent); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	}
+	if memberContent.AuthorisedVia != "" {
+		_, domain, err := gomatrixserverlib.SplitID('@', memberContent.AuthorisedVia)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q is invalid.", memberContent.AuthorisedVia)),
+			}
+		}
+		if domain != cfg.Matrix.ServerName {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(fmt.Sprintf("The authorising username %q does not belong to this server.", memberContent.AuthorisedVia)),
+			}
+		}
+	}
+
+	// Sign the membership event. This is required for restricted joins to work
+	// in the case that the authorised via user is one of our own users. It also
+	// doesn't hurt to do it even if it isn't a restricted join.
+	signed := event.Sign(
+		string(cfg.Matrix.ServerName),
+		cfg.Matrix.KeyID,
+		cfg.Matrix.PrivateKey,
+	)
+
 	// Send the events to the room server.
 	// We are responsible for notifying other servers that the user has joined
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
@@ -323,7 +372,7 @@ func SendJoin(
 			InputRoomEvents: []api.InputRoomEvent{
 				{
 					Kind:          api.KindNew,
-					Event:         event.Headered(stateAndAuthChainResponse.RoomVersion),
+					Event:         signed.Headered(stateAndAuthChainResponse.RoomVersion),
 					SendAsServer:  string(cfg.Matrix.ServerName),
 					TransactionID: nil,
 				},
@@ -354,10 +403,77 @@ func SendJoin(
 			StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents),
 			AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents),
 			Origin:      cfg.Matrix.ServerName,
+			Event:       &signed,
 		},
 	}
 }
 
+// checkRestrictedJoin finds out whether or not we can assist in processing
+// a restricted room join. If the room version does not support restricted
+// joins then this function returns with no side effects. This returns three
+// values:
+// * an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
+//   should always be sent back to the client if one is specified
+// * a user ID of an authorising user, typically a user that has power to
+//   issue invites in the room, if one has been found
+// * an error if there was a problem finding out if this was allowable,
+//   like if the room version isn't known or a problem happened talking to
+//   the roomserver
+func checkRestrictedJoin(
+	httpReq *http.Request,
+	rsAPI api.FederationRoomserverAPI,
+	roomVersion gomatrixserverlib.RoomVersion,
+	roomID, userID string,
+) (*util.JSONResponse, string, error) {
+	if allowRestricted, err := roomVersion.AllowRestrictedJoinsInEventAuth(); err != nil {
+		return nil, "", err
+	} else if !allowRestricted {
+		return nil, "", nil
+	}
+	req := &api.QueryRestrictedJoinAllowedRequest{
+		RoomID: roomID,
+		UserID: userID,
+	}
+	res := &api.QueryRestrictedJoinAllowedResponse{}
+	if err := rsAPI.QueryRestrictedJoinAllowed(httpReq.Context(), req, res); err != nil {
+		return nil, "", err
+	}
+
+	switch {
+	case !res.Restricted:
+		// The join rules for the room don't restrict membership.
+		return nil, "", nil
+
+	case !res.Resident:
+		// The join rules restrict membership but our server isn't currently
+		// joined to all of the allowed rooms, so we can't actually decide
+		// whether or not to allow the user to join. This error code should
+		// tell the joining server to try joining via another resident server
+		// instead.
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnableToAuthoriseJoin("This server cannot authorise the join."),
+		}, "", nil
+
+	case !res.Allowed:
+		// The join rules restrict membership, our server is in the relevant
+		// rooms and the user wasn't joined to join any of the allowed rooms
+		// and therefore can't join this room.
+		return &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You are not joined to any matching rooms."),
+		}, "", nil
+
+	default:
+		// The join rules restrict membership, our server is in the relevant
+		// rooms and the user was allowed to join because they belong to one
+		// of the allowed rooms. We now need to pick one of our own local users
+		// from within the room to use as the authorising user ID, so that it
+		// can be referred to from within the membership content.
+		return nil, res.AuthorisedVia, nil
+	}
+}
+
 type eventsByDepth []*gomatrixserverlib.HeaderedEvent
 
 func (e eventsByDepth) Len() int {
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 80e7aed64..f87ff2962 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -184,6 +184,7 @@ type FederationRoomserverAPI interface {
 	// Query whether a server is allowed to see an event
 	QueryServerAllowedToSeeEvent(ctx context.Context, req *QueryServerAllowedToSeeEventRequest, res *QueryServerAllowedToSeeEventResponse) error
 	QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
+	QueryRestrictedJoinAllowed(ctx context.Context, req *QueryRestrictedJoinAllowedRequest, res *QueryRestrictedJoinAllowedResponse) error
 	PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
 	// Query a given amount (or less) of events prior to a given set of events.
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index 711324644..92c5c1b1d 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -354,6 +354,16 @@ func (t *RoomserverInternalAPITrace) QueryAuthChain(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) QueryRestrictedJoinAllowed(
+	ctx context.Context,
+	request *QueryRestrictedJoinAllowedRequest,
+	response *QueryRestrictedJoinAllowedResponse,
+) error {
+	err := t.Impl.QueryRestrictedJoinAllowed(ctx, request, response)
+	util.GetLogger(ctx).WithError(err).Infof("QueryRestrictedJoinAllowed req=%+v res=%+v", js(request), js(response))
+	return err
+}
+
 func js(thing interface{}) string {
 	b, err := json.Marshal(thing)
 	if err != nil {
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index afafb87c3..f157a9025 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -348,6 +348,26 @@ type QueryServerBannedFromRoomResponse struct {
 	Banned bool `json:"banned"`
 }
 
+type QueryRestrictedJoinAllowedRequest struct {
+	UserID string `json:"user_id"`
+	RoomID string `json:"room_id"`
+}
+
+type QueryRestrictedJoinAllowedResponse struct {
+	// True if the room membership is restricted by the join rule being set to "restricted"
+	Restricted bool `json:"restricted"`
+	// True if our local server is joined to all of the allowed rooms specified in the "allow"
+	// key of the join rule, false if we are missing from some of them and therefore can't
+	// reliably decide whether or not we can satisfy the join
+	Resident bool `json:"resident"`
+	// True if the restricted join is allowed because we found the membership in one of the
+	// allowed rooms from the join rule, false if not
+	Allowed bool `json:"allowed"`
+	// Contains the user ID of the selected user ID that has power to issue invites, this will
+	// get populated into the "join_authorised_via_users_server" content in the membership
+	AuthorisedVia string `json:"authorised_via,omitempty"`
+}
+
 // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) {
 	se := make(map[string]string)
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 61a0206ef..c9e839198 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -24,6 +24,7 @@ import (
 	"github.com/getsentry/sentry-go"
 	fsAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	rsAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
@@ -160,6 +161,7 @@ func (r *Joiner) performJoinRoomByAlias(
 }
 
 // TODO: Break this function up a bit
+// nolint:gocyclo
 func (r *Joiner) performJoinRoomByID(
 	ctx context.Context,
 	req *rsAPI.PerformJoinRequest,
@@ -210,6 +212,11 @@ func (r *Joiner) performJoinRoomByID(
 		req.Content = map[string]interface{}{}
 	}
 	req.Content["membership"] = gomatrixserverlib.Join
+	if authorisedVia, aerr := r.populateAuthorisedViaUserForRestrictedJoin(ctx, req); aerr != nil {
+		return "", "", aerr
+	} else if authorisedVia != "" {
+		req.Content["join_authorised_via_users_server"] = authorisedVia
+	}
 	if err = eb.SetContent(req.Content); err != nil {
 		return "", "", fmt.Errorf("eb.SetContent: %w", err)
 	}
@@ -350,6 +357,33 @@ func (r *Joiner) performFederatedJoinRoomByID(
 	return fedRes.JoinedVia, nil
 }
 
+func (r *Joiner) populateAuthorisedViaUserForRestrictedJoin(
+	ctx context.Context,
+	joinReq *rsAPI.PerformJoinRequest,
+) (string, error) {
+	req := &api.QueryRestrictedJoinAllowedRequest{
+		UserID: joinReq.UserID,
+		RoomID: joinReq.RoomIDOrAlias,
+	}
+	res := &api.QueryRestrictedJoinAllowedResponse{}
+	if err := r.Queryer.QueryRestrictedJoinAllowed(ctx, req, res); err != nil {
+		return "", fmt.Errorf("r.Queryer.QueryRestrictedJoinAllowed: %w", err)
+	}
+	if !res.Restricted {
+		return "", nil
+	}
+	if !res.Resident {
+		return "", nil
+	}
+	if !res.Allowed {
+		return "", &rsAPI.PerformError{
+			Code: rsAPI.PerformErrorNotAllowed,
+			Msg:  fmt.Sprintf("The join to room %s was not allowed.", joinReq.RoomIDOrAlias),
+		}
+	}
+	return res.AuthorisedVia, nil
+}
+
 func buildEvent(
 	ctx context.Context, db storage.Database, cfg *config.Global, builder *gomatrixserverlib.EventBuilder,
 ) (*gomatrixserverlib.HeaderedEvent, *rsAPI.QueryLatestEventsAndStateResponse, error) {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index d25bdc378..70613baab 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -16,6 +16,7 @@ package query
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 
@@ -757,3 +758,128 @@ func (r *Queryer) QueryAuthChain(ctx context.Context, req *api.QueryAuthChainReq
 	res.AuthChain = hchain
 	return nil
 }
+
+// nolint:gocyclo
+func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse) error {
+	// Look up if we know anything about the room. If it doesn't exist
+	// or is a stub entry then we can't do anything.
+	roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
+	if err != nil {
+		return fmt.Errorf("r.DB.RoomInfo: %w", err)
+	}
+	if roomInfo == nil || roomInfo.IsStub {
+		return nil // fmt.Errorf("room %q doesn't exist or is stub room", req.RoomID)
+	}
+	// If the room version doesn't allow restricted joins then don't
+	// try to process any further.
+	allowRestrictedJoins, err := roomInfo.RoomVersion.AllowRestrictedJoinsInEventAuth()
+	if err != nil {
+		return fmt.Errorf("roomInfo.RoomVersion.AllowRestrictedJoinsInEventAuth: %w", err)
+	} else if !allowRestrictedJoins {
+		return nil
+	}
+	// Get the join rules to work out if the join rule is "restricted".
+	joinRulesEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomJoinRules, "")
+	if err != nil {
+		return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+	}
+	var joinRules gomatrixserverlib.JoinRuleContent
+	if err = json.Unmarshal(joinRulesEvent.Content(), &joinRules); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	// If the join rule isn't "restricted" then there's nothing more to do.
+	res.Restricted = joinRules.JoinRule == gomatrixserverlib.Restricted
+	if !res.Restricted {
+		return nil
+	}
+	// Start off by populating the "resident" flag in the response. If we
+	// come across any rooms in the request that are missing, we will unset
+	// the flag.
+	res.Resident = true
+	// If the user is already invited to the room then the join is allowed
+	// but we don't specify an authorised via user, since the event auth
+	// will allow the join anyway.
+	var pending bool
+	if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
+		return fmt.Errorf("helpers.IsInvitePending: %w", err)
+	} else if pending {
+		res.Allowed = true
+		return nil
+	}
+	// We need to get the power levels content so that we can determine which
+	// users in the room are entitled to issue invites. We need to use one of
+	// these users as the authorising user.
+	powerLevelsEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomPowerLevels, "")
+	if err != nil {
+		return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+	}
+	var powerLevels gomatrixserverlib.PowerLevelContent
+	if err = json.Unmarshal(powerLevelsEvent.Content(), &powerLevels); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	// Step through the join rules and see if the user matches any of them.
+	for _, rule := range joinRules.Allow {
+		// We only understand "m.room_membership" rules at this point in
+		// time, so skip any rule that doesn't match those.
+		if rule.Type != gomatrixserverlib.MRoomMembership {
+			continue
+		}
+		// See if the room exists. If it doesn't exist or if it's a stub
+		// room entry then we can't check memberships.
+		targetRoomInfo, err := r.DB.RoomInfo(ctx, rule.RoomID)
+		if err != nil || targetRoomInfo == nil || targetRoomInfo.IsStub {
+			res.Resident = false
+			continue
+		}
+		// First of all work out if *we* are still in the room, otherwise
+		// it's possible that the memberships will be out of date.
+		isIn, err := r.DB.GetLocalServerInRoom(ctx, targetRoomInfo.RoomNID)
+		if err != nil || !isIn {
+			// If we aren't in the room, we can no longer tell if the room
+			// memberships are up-to-date.
+			res.Resident = false
+			continue
+		}
+		// At this point we're happy that we are in the room, so now let's
+		// see if the target user is in the room.
+		_, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
+		if err != nil {
+			continue
+		}
+		// If the user is not in the room then we will skip them.
+		if !isIn {
+			continue
+		}
+		// The user is in the room, so now we will need to authorise the
+		// join using the user ID of one of our own users in the room. Pick
+		// one.
+		joinNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, targetRoomInfo.RoomNID, true, true)
+		if err != nil || len(joinNIDs) == 0 {
+			// There should always be more than one join NID at this point
+			// because we are gated behind GetLocalServerInRoom, but y'know,
+			// sometimes strange things happen.
+			continue
+		}
+		// For each of the joined users, let's see if we can get a valid
+		// membership event.
+		for _, joinNID := range joinNIDs {
+			events, err := r.DB.Events(ctx, []types.EventNID{joinNID})
+			if err != nil || len(events) != 1 {
+				continue
+			}
+			event := events[0]
+			if event.Type() != gomatrixserverlib.MRoomMember || event.StateKey() == nil {
+				continue // shouldn't happen
+			}
+			// Only users that have the power to invite should be chosen.
+			if powerLevels.UserLevel(*event.StateKey()) < powerLevels.Invite {
+				continue
+			}
+			res.Resident = true
+			res.Allowed = true
+			res.AuthorisedVia = *event.StateKey()
+			return nil
+		}
+	}
+	return nil
+}
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index 09358001b..7b10ae657 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -61,6 +61,7 @@ const (
 	RoomserverQueryKnownUsersPath              = "/roomserver/queryKnownUsers"
 	RoomserverQueryServerBannedFromRoomPath    = "/roomserver/queryServerBannedFromRoom"
 	RoomserverQueryAuthChainPath               = "/roomserver/queryAuthChain"
+	RoomserverQueryRestrictedJoinAllowed       = "/roomserver/queryRestrictedJoinAllowed"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -557,6 +558,16 @@ func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
 
+func (h *httpRoomserverInternalAPI) QueryRestrictedJoinAllowed(
+	ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRestrictedJoinAllowed")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryRestrictedJoinAllowed
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
 func (h *httpRoomserverInternalAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, res *api.PerformForgetResponse) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformForget")
 	defer span.Finish()
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index 9042e341b..ad4fdc469 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -472,4 +472,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverQueryRestrictedJoinAllowed,
+		httputil.MakeInternalAPI("queryRestrictedJoinAllowed", func(req *http.Request) util.JSONResponse {
+			request := api.QueryRestrictedJoinAllowedRequest{}
+			response := api.QueryRestrictedJoinAllowedResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := r.QueryRestrictedJoinAllowed(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

From 9ce3898d03445df97e5dba3701628100ef8b3051 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 25 May 2022 13:37:15 +0100
Subject: [PATCH 134/391] Fix the `resolve-state` tool

---
 cmd/resolve-state/main.go | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index c52fd6c42..3a9f3ce4f 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -4,13 +4,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"os"
 	"strconv"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -23,11 +24,17 @@ import (
 //   e.g. ./resolve-state --roomversion=5 1254 1235 1282
 
 var roomVersion = flag.String("roomversion", "5", "the room version to parse events as")
+var filterType = flag.String("filtertype", "", "the event types to filter on")
 
 func main() {
 	ctx := context.Background()
 	cfg := setup.ParseFlags(true)
-	args := os.Args[1:]
+	cfg.Logging = append(cfg.Logging[:0], config.LogrusHook{
+		Type:  "std",
+		Level: "error",
+	})
+	base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
+	args := flag.Args()
 
 	fmt.Println("Room version", *roomVersion)
 
@@ -45,7 +52,7 @@ func main() {
 		panic(err)
 	}
 
-	roomserverDB, err := storage.Open(nil, &cfg.RoomServer.Database, cache)
+	roomserverDB, err := storage.Open(base, &cfg.RoomServer.Database, cache)
 	if err != nil {
 		panic(err)
 	}
@@ -113,9 +120,18 @@ func main() {
 	}
 
 	fmt.Println("Resolved state contains", len(resolved), "events")
+	filteringEventType := *filterType
+	count := 0
 	for _, event := range resolved {
+		if filteringEventType != "" && event.Type() != filteringEventType {
+			continue
+		}
+		count++
 		fmt.Println()
 		fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey())
 		fmt.Printf("  %s\n", string(event.Content()))
 	}
+
+	fmt.Println()
+	fmt.Println("Returned", count, "state events after filtering")
 }

From ac4d0072cf5507d7de1504d16aa5c4ca8537eb26 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 25 May 2022 14:31:07 +0100
Subject: [PATCH 135/391] Fix using the remote provided event from `/send_join`
 (#2490)

---
 federationapi/internal/perform.go | 12 ++++++++----
 federationapi/routing/join.go     |  2 +-
 go.mod                            |  2 +-
 go.sum                            |  4 ++--
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 970a64653..9100c8f18 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -213,10 +213,14 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// If the remote server returned an event in the "event" key of
 	// the send_join request then we should use that instead. It may
 	// contain signatures that we don't know about.
-	if respSendJoin.Event != nil && isWellFormedMembershipEvent(
-		respSendJoin.Event, roomID, userID, r.cfg.Matrix.ServerName,
-	) {
-		event = respSendJoin.Event
+	if len(respSendJoin.Event) > 0 {
+		var remoteEvent *gomatrixserverlib.Event
+		remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion)
+		if err == nil && isWellFormedMembershipEvent(
+			remoteEvent, roomID, userID, r.cfg.Matrix.ServerName,
+		) {
+			event = remoteEvent
+		}
 	}
 
 	// Sanity-check the join response to ensure that it has a create
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 9f8d94105..abf9c0945 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -403,7 +403,7 @@ func SendJoin(
 			StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents),
 			AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents),
 			Origin:      cfg.Matrix.ServerName,
-			Event:       &signed,
+			Event:       signed.JSON(),
 		},
 	}
 }
diff --git a/go.mod b/go.mod
index 89ff04f63..2513ac87f 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220525125811-a75fa15a03a0
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 3ffd52182..03e71d4a5 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c h1:J9krMtVgo4mV/G+mRA1u3GL6nNxdNnuPcs891uIQGic=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220525125811-a75fa15a03a0 h1:wmxJJA5dczakTi7UsU/MWuWJY3gmC6RWzTN2S484tHo=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220525125811-a75fa15a03a0/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From ff533986355995cb6f7b9e52e83abbc89cd11d14 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 25 May 2022 16:39:51 +0100
Subject: [PATCH 136/391] Mark room versions 8 and 9 as stable

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 2513ac87f..98a6e9de9 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220525125811-a75fa15a03a0
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220525153921-7597e29b8320
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 03e71d4a5..29dba6bcd 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220525125811-a75fa15a03a0 h1:wmxJJA5dczakTi7UsU/MWuWJY3gmC6RWzTN2S484tHo=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220525125811-a75fa15a03a0/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220525153921-7597e29b8320 h1:J1R1R/JKI942vCVeLwPGu0xh/viAgvRrTh3ACP5xIBE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220525153921-7597e29b8320/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 6940c7c7dd9cefc3f7fa0bec00926e96ee669b56 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 25 May 2022 16:40:31 +0100
Subject: [PATCH 137/391] Try to spot state deletions when they happen (#2489)

---
 roomserver/internal/input/input_events.go     |  4 ----
 .../internal/input/input_latest_events.go     | 19 ++++++++++++-------
 .../storage/tables/events_table_test.go       |  1 -
 roomserver/types/types.go                     |  4 ----
 4 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 3ab9ba4f0..d437d7768 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -535,8 +535,6 @@ func (r *Inputer) calculateAndSetState(
 	roomState := state.NewStateResolution(updater, roomInfo)
 
 	if input.HasState {
-		stateAtEvent.Overwrite = true
-
 		// We've been told what the state at the event is so we don't need to calculate it.
 		// Check that those state events are in the database and store the state.
 		var entries []types.StateEntry
@@ -549,8 +547,6 @@ func (r *Inputer) calculateAndSetState(
 			return fmt.Errorf("updater.AddState: %w", err)
 		}
 	} else {
-		stateAtEvent.Overwrite = false
-
 		// We haven't been told what the state at the event is so we need to calculate it from the prev_events
 		if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, isRejected); err != nil {
 			return fmt.Errorf("roomState.CalculateAndStoreStateBeforeEvent: %w", err)
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index e4c138d58..4e5b73420 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -27,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // updateLatestEvents updates the list of latest events for this room in the database and writes the
@@ -258,13 +259,6 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.CalculateAndStoreStateAfterEvents: %w", err)
 	}
 
-	// If we are overwriting the state then we should make sure that we
-	// don't send anything out over federation again, it will very likely
-	// be a repeat.
-	if u.stateAtEvent.Overwrite {
-		u.sendAsServer = ""
-	}
-
 	// Now that we have a new state snapshot based on the latest events,
 	// we can compare that new snapshot to the previous one and see what
 	// has changed. This gives us one list of removed state events and
@@ -277,6 +271,17 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
 	}
 
+	if removed := len(u.removed) - len(u.added); removed > 0 {
+		logrus.WithFields(logrus.Fields{
+			"event_id":      u.event.EventID(),
+			"room_id":       u.event.RoomID(),
+			"old_state_nid": u.oldStateNID,
+			"new_state_nid": u.newStateNID,
+			"old_latest":    u.oldLatest,
+			"new_latest":    u.latest,
+		}).Errorf("Unexpected state deletion (removing %d events)", removed)
+	}
+
 	// Also work out the state before the event removes and the event
 	// adds.
 	u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots(
diff --git a/roomserver/storage/tables/events_table_test.go b/roomserver/storage/tables/events_table_test.go
index 6f72a59b5..74502a31f 100644
--- a/roomserver/storage/tables/events_table_test.go
+++ b/roomserver/storage/tables/events_table_test.go
@@ -85,7 +85,6 @@ func Test_EventsTable(t *testing.T) {
 				assert.NoError(t, err)
 			}
 			stateAtEvent := types.StateAtEvent{
-				Overwrite:              false,
 				BeforeStateSnapshotNID: types.StateSnapshotNID(stateSnapshot),
 				IsRejected:             false,
 				StateEntry: types.StateEntry{
diff --git a/roomserver/types/types.go b/roomserver/types/types.go
index 62695aaee..cf12a3c1e 100644
--- a/roomserver/types/types.go
+++ b/roomserver/types/types.go
@@ -173,10 +173,6 @@ func DeduplicateStateEntries(a []StateEntry) []StateEntry {
 
 // StateAtEvent is the state before and after a matrix event.
 type StateAtEvent struct {
-	// Should this state overwrite the latest events and memberships of the room?
-	// This might be necessary when rejoining a federated room after a period of
-	// absence, as our state and latest events will be out of date.
-	Overwrite bool
 	// The state before the event.
 	BeforeStateSnapshotNID StateSnapshotNID
 	// True if this StateEntry is rejected. State resolution should then treat this

From 015465d4966ac69eed09e1d14343078bb5fe083f Mon Sep 17 00:00:00 2001
From: Brandon <brandon.wofford@unity3d.com>
Date: Wed, 25 May 2022 11:17:02 -0500
Subject: [PATCH 138/391] Update 1_planning.md (#2467)

* Update 1_planning.md

Modes section of the planning component of the documentation rewritten for grammar and clarity.

* Update 1_planning.md

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 docs/installation/1_planning.md | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md
index 89cc5b4a6..45012d80c 100644
--- a/docs/installation/1_planning.md
+++ b/docs/installation/1_planning.md
@@ -9,21 +9,19 @@ permalink: /installation/planning
 
 ## Modes
 
-Dendrite can be run in one of two configurations:
+Dendrite consists of several components, each responsible for a different aspect of the Matrix protocol.
+Users can run Dendrite in one of two modes which dictate how these components are executed and communicate.
 
-* **Monolith mode**: All components run in the same process. In this mode,
-   it is possible to run an in-process NATS Server instead of running a standalone deployment.
-   This will usually be the preferred model for low-to-mid volume deployments, providing the best
-   balance between performance and resource usage.
+* **Monolith mode** runs all components in a single process. Components communicate through an internal NATS
+  server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the
+  best balance between performance and resource usage for low-to-mid volume deployments.
 
-* **Polylith mode**: A cluster of individual components running in their own processes, dealing
-  with different aspects of the Matrix protocol. Components communicate with each other using
-  internal HTTP APIs and NATS Server. This will almost certainly be the preferred model for very
-  large deployments but scalability comes with a cost. API calls are expensive and therefore a
-  polylith deployment may end up using disproportionately more resources for a smaller number of
-  users compared to a monolith deployment.
+* **Polylith mode** runs all components in isolated processes. Components communicate through an external NATS
+  server and HTTP APIs, which incur considerable overhead. While this mode allows for more granular control of
+  resources dedicated toward individual processes, given the additional communications overhead, it is only
+  necessary for very large deployments.
 
-At present, we **recommend monolith mode deployments** in all cases.
+Given our current state of development, **we recommend monolith mode** for all deployments.
 
 ## Databases
 

From 9eb4fec33bfa6c4674f1ff920d3013cf1784209f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 26 May 2022 10:38:46 +0100
Subject: [PATCH 139/391] Make logging output for state deletions a bit better

---
 roomserver/internal/input/input_latest_events.go | 10 +++++-----
 roomserver/types/types.go                        |  8 ++++++++
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 4e5b73420..a1b094871 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -102,8 +102,8 @@ type latestEventsUpdater struct {
 	// The eventID of the event that was processed before this one.
 	lastEventIDSent string
 	// The latest events in the room after processing this event.
-	oldLatest []types.StateAtEventAndReference
-	latest    []types.StateAtEventAndReference
+	oldLatest types.StateAtEventAndReferences
+	latest    types.StateAtEventAndReferences
 	// The state entries removed from and added to the current state of the
 	// room as a result of processing this event. They are sorted lists.
 	removed []types.StateEntry
@@ -126,7 +126,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// state snapshot from somewhere else, e.g. a federated room join,
 	// then start with an empty set - none of the forward extremities
 	// that we knew about before matter anymore.
-	u.oldLatest = []types.StateAtEventAndReference{}
+	u.oldLatest = types.StateAtEventAndReferences{}
 	if !u.rewritesState {
 		u.oldStateNID = u.updater.CurrentStateSnapshotNID()
 		u.oldLatest = u.updater.LatestEvents()
@@ -277,8 +277,8 @@ func (u *latestEventsUpdater) latestState() error {
 			"room_id":       u.event.RoomID(),
 			"old_state_nid": u.oldStateNID,
 			"new_state_nid": u.newStateNID,
-			"old_latest":    u.oldLatest,
-			"new_latest":    u.latest,
+			"old_latest":    u.oldLatest.EventIDs(),
+			"new_latest":    u.latest.EventIDs(),
 		}).Errorf("Unexpected state deletion (removing %d events)", removed)
 	}
 
diff --git a/roomserver/types/types.go b/roomserver/types/types.go
index cf12a3c1e..bc01ca33c 100644
--- a/roomserver/types/types.go
+++ b/roomserver/types/types.go
@@ -210,6 +210,14 @@ func (s StateAtEventAndReferences) Swap(a, b int) {
 	s[a], s[b] = s[b], s[a]
 }
 
+func (s StateAtEventAndReferences) EventIDs() string {
+	strs := make([]string, 0, len(s))
+	for _, r := range s {
+		strs = append(strs, r.EventID)
+	}
+	return "[" + strings.Join(strs, " ") + "]"
+}
+
 // An Event is a gomatrixserverlib.Event with the numeric event ID attached.
 // It is when performing bulk event lookup in the database.
 type Event struct {

From 5a878b6e14c6c6243f5b7e574501b66a03046b6f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 26 May 2022 11:52:39 +0100
Subject: [PATCH 140/391] Update to matrix-org/gomatrixserverlib#314

---
 go.mod | 2 +-
 go.sum | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index 98a6e9de9..d1b3f2883 100644
--- a/go.mod
+++ b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220525153921-7597e29b8320
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220526105206-b1c5619fe0c2
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 29dba6bcd..46d5beb6f 100644
--- a/go.sum
+++ b/go.sum
@@ -795,8 +795,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220525153921-7597e29b8320 h1:J1R1R/JKI942vCVeLwPGu0xh/viAgvRrTh3ACP5xIBE=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220525153921-7597e29b8320/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220526105206-b1c5619fe0c2 h1:t5VdznF0DiPayir4lTWvdHuk2+rDlwe3qksrgua+f3Q=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220526105206-b1c5619fe0c2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -1136,7 +1136,6 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
 github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
 github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=

From 9e46d5025a5a817a618ffc79d6aae5970e6498d2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 26 May 2022 12:34:43 +0100
Subject: [PATCH 141/391] Upgrade various dependencies (#2496)

* Upgrade various dependencies

* Yaml dependency too
---
 go.mod                    |  45 +-
 go.sum                    | 869 ++------------------------------------
 internal/sqlutil/trace.go |   4 +-
 3 files changed, 67 insertions(+), 851 deletions(-)

diff --git a/go.mod b/go.mod
index d1b3f2883..5b732ab41 100644
--- a/go.mod
+++ b/go.mod
@@ -5,20 +5,24 @@ replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-serve
 replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
 
 require (
-	github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0
+	github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/codeclysm/extract v2.2.0+incompatible
-	github.com/containerd/containerd v1.6.2 // indirect
-	github.com/docker/docker v20.10.14+incompatible
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.16+incompatible
 	github.com/docker/go-connections v0.4.0
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/frankban/quicktest v1.14.3 // indirect
 	github.com/getsentry/sentry-go v0.13.0
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.7
+	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
@@ -33,35 +37,44 @@ require (
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220526105206-b1c5619fe0c2
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
-	github.com/mattn/go-sqlite3 v1.14.10
-	github.com/miekg/dns v1.1.31 // indirect
+	github.com/mattn/go-sqlite3 v1.14.13
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/miekg/dns v1.1.49 // indirect
+	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
 	github.com/nats-io/nats.go v1.14.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
-	github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
-	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
+	github.com/onsi/gomega v1.17.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose v2.7.0+incompatible
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.12.2
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.1
 	github.com/tidwall/gjson v1.14.1
 	github.com/tidwall/sjson v1.2.4
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.3
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122
-	golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
-	golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2
-	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
+	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
+	golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
+	golang.org/x/net v0.0.0-20220524220425-1d687d428aca
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
 	nhooyr.io/websocket v1.8.7
 )
 
diff --git a/go.sum b/go.sum
index 46d5beb6f..48fd50612 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,3 @@
-bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
-bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -17,11 +15,6 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -30,7 +23,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -46,30 +38,15 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0 h1:QUqcb7BOcBU2p7Nax7pESOb8hrZYtI0Ts6j4v4mvcQo=
 github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
+github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 h1:PsaZb47k7WB1V+AlGpb+W7SM+ZOhp16vVevg5gl9YkU=
+github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ=
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
-github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
@@ -83,41 +60,12 @@ github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGh
 github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
-github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
-github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
-github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
-github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
-github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
-github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
-github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
-github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
-github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
-github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
-github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
-github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
-github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
-github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
-github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
 github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
 github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
-github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
@@ -128,8 +76,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
 github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c=
 github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c=
 github.com/anacrolix/envpprof v1.1.1 h1:sHQCyj7HtiSfaZAzL2rJrQdyS7odLqlwO6nhk/tG/j8=
@@ -142,282 +88,78 @@ github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikz
 github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
 github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
-github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
-github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
-github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
-github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
-github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
-github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
-github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
-github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
-github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
-github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
-github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
-github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
-github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
-github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
-github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
-github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
-github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
-github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
-github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI=
 github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
-github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
-github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
-github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
-github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
-github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E=
-github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
-github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
-github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
-github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
-github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
-github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
-github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
-github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
-github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8=
-github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
-github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
-github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
-github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
-github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=
-github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
-github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
-github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
-github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
-github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
-github.com/containerd/containerd v1.6.2 h1:pcaPUGbYW8kBw6OgIZwIVIeEhdWVrBzsoCfVJ5BjrLU=
-github.com/containerd/containerd v1.6.2/go.mod h1:sidY30/InSE1j2vdD1ihtKoJz+lWdaXMdiAeIupaf+s=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
-github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
-github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
-github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
-github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk=
-github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
-github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
-github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
-github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
-github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=
-github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
-github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
-github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
-github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
-github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
-github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
-github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=
-github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=
-github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
-github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
-github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4=
-github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
-github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
-github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
-github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=
-github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
-github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
-github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
-github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
-github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
-github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
-github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
-github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
-github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
-github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=
-github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=
-github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
-github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
-github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y=
-github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
-github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
-github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE=
-github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
-github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
-github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
-github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
-github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
-github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
-github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
-github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
-github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
-github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
-github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
-github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
-github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w=
-github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
+github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405crbpxvWzKovETOQ=
+github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
-github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
-github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
-github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
-github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
-github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
 github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
-github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
@@ -433,36 +175,13 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI=
-github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
-github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
-github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -480,30 +199,15 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
-github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
-github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
-github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
-github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -512,7 +216,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -530,7 +233,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -541,7 +243,6 @@ github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85q
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -549,22 +250,16 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -572,108 +267,49 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
-github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
 github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
 github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
 github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
 github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
-github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -728,15 +364,11 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb
 github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
 github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
 github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4=
 github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
@@ -751,7 +383,6 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -761,7 +392,6 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
 github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
 github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A=
 github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
 github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
@@ -769,14 +399,7 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm
 github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
 github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
@@ -803,63 +426,36 @@ github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/em
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-colorable v0.0.6/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
-github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
-github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
-github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk=
-github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
+github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
 github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
-github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
-github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
+github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
-github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
-github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
-github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
-github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
-github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
-github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs=
-github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -872,13 +468,9 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
-github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a h1:lem6QCvxR0Y28gth9P+wV2K/zYUUAkJ+55U8cpS0p5I=
 github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
@@ -886,7 +478,6 @@ github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
-github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo=
@@ -897,157 +488,79 @@ github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQE
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
-github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31 h1:FFHgfAIoAXCCL4xBoAugZVpekfGmZ/fBBueneUKBv7I=
-github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k=
+github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 h1:Dmx8g2747UTVPzSkmohk84S3g/uWqd6+f4SSLPhLcfA=
+github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
-github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
-github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
-github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
-github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
-github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
-github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
-github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
-github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
-github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
-github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
-github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec=
+github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
-github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ=
 github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
-github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
+github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
-github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
-github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
-github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
-github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
-github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
 github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
@@ -1072,10 +585,7 @@ github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
 github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
-github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -1083,52 +593,28 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
-github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
-github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
 github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -1139,24 +625,15 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso
 github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
 github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
 github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
 github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
@@ -1165,22 +642,13 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
-github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yggdrasil-network/yggdrasil-go v0.4.3 h1:LNS7kNpKzFlxQ9xmD5tfmMEvzwa+utBoD6pV9t2a8q4=
@@ -1193,95 +661,39 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
-github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
-github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
-go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
-go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
-go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
-go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
-go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
-go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
-go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
-go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs=
-go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
-go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
-go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
-go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
-go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
-go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs=
-go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
-go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
-go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
-go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
-golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8=
-golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1299,8 +711,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20220321031419-a8550c1d254a h1:LnH9RNcpPv5Kzi15lXg42lYMPUf0x8CuPv1YnvBWZAg=
-golang.org/x/image v0.0.0-20220321031419-a8550c1d254a/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 h1:LRtI4W37N+KFebI/qV0OFiLUv4GLOWeEW5hn/KEJvxE=
+golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -1312,34 +724,27 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mobile v0.0.0-20220112015953-858099ff7816/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
 golang.org/x/mobile v0.0.0-20220325161704-447654d348e3/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
-golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2 h1:XEBytU1NHu2jr/7GWEJBRH3uEhegH+hQcF10Mj/7Cb8=
-golang.org/x/mobile v0.0.0-20220407111146-e579adbbc4a2/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
+golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd h1:x1GptNaTtxPAlTVIAJk61fuXg0y17h09DTxyb+VNC/k=
+golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1349,17 +754,13 @@ golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1376,31 +777,20 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 h1:EN5+DfgmRMvRUrMGERW2gQl3Vc+Z7ZMnI/xdEpPSf0c=
-golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220524220425-1d687d428aca h1:xTaFYiPROfpPhqrfTIDXj0ri1SpfueYT951s4bAuDO8=
+golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1408,14 +798,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1428,13 +811,10 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1446,38 +826,25 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1492,71 +859,41 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
+golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0=
@@ -1565,20 +902,13 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M=
 golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1592,23 +922,18 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -1623,27 +948,17 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
 golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0=
 golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
@@ -1660,7 +975,6 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ
 gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
-google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -1680,11 +994,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1693,8 +1002,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -1704,7 +1011,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
@@ -1713,7 +1019,6 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
@@ -1722,44 +1027,21 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -1768,19 +1050,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
-google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1795,10 +1064,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1806,37 +1073,27 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v1 v1.0.0-20161222125816-442357a80af5/go.mod h1:u0ALmqvLRxLI95fkdCEWrE6mhWYZW1aMOJHp5YXLHTg=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM=
 gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
 gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
 gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
 gopkg.in/httprequest.v1 v1.1.1/go.mod h1:/CkavNL+g3qLOrpFHVrEx4NKepeqR4XTZWNj4sGGjz0=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI=
 gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o=
 gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk=
 gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -1846,10 +1103,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
-gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
@@ -1862,49 +1117,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
-k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
-k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
-k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs=
-k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
-k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
-k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U=
-k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
-k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
-k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
-k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ=
-k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
-k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
-k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
-k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y=
-k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0=
-k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
-k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
-k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
-k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI=
-k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
-k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
-k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
-k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
-k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
 launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
@@ -1913,14 +1125,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go
index c16738616..7b637106b 100644
--- a/internal/sqlutil/trace.go
+++ b/internal/sqlutil/trace.go
@@ -37,7 +37,7 @@ type traceInterceptor struct {
 	sqlmw.NullInterceptor
 }
 
-func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (driver.Rows, error) {
+func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (context.Context, driver.Rows, error) {
 	startedAt := time.Now()
 	rows, err := stmt.QueryContext(ctx, args)
 
@@ -45,7 +45,7 @@ func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.St
 
 	logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args)
 
-	return rows, err
+	return ctx, rows, err
 }
 
 func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) {

From 45c4c006721756595b12b41638a86ec803ee9fbd Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 26 May 2022 13:01:04 +0100
Subject: [PATCH 142/391] Version 0.8.6 (#2497)

---
 CHANGES.md          | 19 +++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 3deebd8a0..c5e4f7f8c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,24 @@
 # Changelog
 
+## Dendrite 0.8.6 (2022-05-26)
+
+### Features
+
+* Room versions 8 and 9 are now marked as stable
+* Dendrite can now assist remote users to join restricted rooms via `/make_join` and `/send_join`
+
+### Fixes
+
+* The sync API no longer returns immediately on `/sync` requests unnecessarily if it can be avoided
+* A race condition has been fixed in the sync API when updating presence via `/sync`
+* A race condition has been fixed sending E2EE keys to remote servers over federation when joining rooms
+* The `trusted_private_chat` preset should now grant power level 100 to all participant users, which should improve the user experience of direct messages
+* Invited users are now authed correctly in restricted rooms
+* The `join_authorised_by_users_server` key is now correctly stripped in restricted rooms when updating the membership event
+* Appservices should now receive invite events correctly
+* Device list updates should no longer contain optional fields with `null` values
+* The `/deactivate` endpoint has been fixed to no longer confuse Element with incorrect completed flows
+
 ## Dendrite 0.8.5 (2022-05-13)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index 04c9a8a88..0957b4545 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 5
+	VersionPatch = 6
 	VersionTag   = "" // example: "rc1"
 )
 

From b541f3043f0ad8924547c540d9235d017d1792f6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 26 May 2022 15:08:17 +0100
Subject: [PATCH 143/391] Add support for MSC3787 and `org.matrix.msc3787` room
 version (update to matrix-org/gomatrixserverlib#310)

---
 federationapi/routing/join.go      | 2 +-
 go.mod                             | 2 +-
 go.sum                             | 4 ++--
 roomserver/internal/query/query.go | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index abf9c0945..8d779c3d7 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -425,7 +425,7 @@ func checkRestrictedJoin(
 	roomVersion gomatrixserverlib.RoomVersion,
 	roomID, userID string,
 ) (*util.JSONResponse, string, error) {
-	if allowRestricted, err := roomVersion.AllowRestrictedJoinsInEventAuth(); err != nil {
+	if allowRestricted, err := roomVersion.MayAllowRestrictedJoinsInEventAuth(); err != nil {
 		return nil, "", err
 	} else if !allowRestricted {
 		return nil, "", nil
diff --git a/go.mod b/go.mod
index 5b732ab41..9bb2b08e1 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220526105206-b1c5619fe0c2
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 48fd50612..0aae9b007 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220526105206-b1c5619fe0c2 h1:t5VdznF0DiPayir4lTWvdHuk2+rDlwe3qksrgua+f3Q=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220526105206-b1c5619fe0c2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d h1:IwyG/58rFn0/ugD0A/IdSIo7D/oLJ4+k3NznlYhzyHs=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 70613baab..6d306fb09 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -772,7 +772,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 	}
 	// If the room version doesn't allow restricted joins then don't
 	// try to process any further.
-	allowRestrictedJoins, err := roomInfo.RoomVersion.AllowRestrictedJoinsInEventAuth()
+	allowRestrictedJoins, err := roomInfo.RoomVersion.MayAllowRestrictedJoinsInEventAuth()
 	if err != nil {
 		return fmt.Errorf("roomInfo.RoomVersion.AllowRestrictedJoinsInEventAuth: %w", err)
 	} else if !allowRestrictedJoins {

From 500124dd1857fb0d03b7c2c1d9b816443871dac9 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 30 May 2022 09:50:23 +0100
Subject: [PATCH 144/391] Add support for room version 10 (update to
 matrix-org/gomatrixserverlib#315)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9bb2b08e1..580dc3568 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220530084946-3a4b148706bc
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 0aae9b007..75c22a7a7 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d h1:IwyG/58rFn0/ugD0A/IdSIo7D/oLJ4+k3NznlYhzyHs=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220530084946-3a4b148706bc h1:58tT3VznINdRWimb3yYb8QWmTAHX9AAuyOwzdmrp9q4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220530084946-3a4b148706bc/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 247604979a1816e05d11d214d71af2a07f56786f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 30 May 2022 11:01:20 +0100
Subject: [PATCH 145/391] Add missing indexes to federation API destination
 queues (#2502)

* Add missing indexes to federation API destination queues

* Fix duplicate name
---
 federationapi/storage/postgres/queue_edus_table.go | 4 ++++
 federationapi/storage/postgres/queue_json_table.go | 3 +++
 federationapi/storage/postgres/queue_pdus_table.go | 4 ++++
 federationapi/storage/sqlite3/queue_edus_table.go  | 4 ++++
 federationapi/storage/sqlite3/queue_json_table.go  | 3 +++
 federationapi/storage/sqlite3/queue_pdus_table.go  | 4 ++++
 6 files changed, 22 insertions(+)

diff --git a/federationapi/storage/postgres/queue_edus_table.go b/federationapi/storage/postgres/queue_edus_table.go
index 6cac489bf..1fedf0ef1 100644
--- a/federationapi/storage/postgres/queue_edus_table.go
+++ b/federationapi/storage/postgres/queue_edus_table.go
@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
     ON federationsender_queue_edus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_nid_idx
+    ON federationsender_queue_edus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
+    ON federationsender_queue_edus (server_name);
 `
 
 const insertQueueEDUSQL = "" +
diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go
index 853073741..e33074182 100644
--- a/federationapi/storage/postgres/queue_json_table.go
+++ b/federationapi/storage/postgres/queue_json_table.go
@@ -33,6 +33,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_json (
 	-- The JSON body. Text so that we preserve UTF-8.
 	json_body TEXT NOT NULL
 );
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_json_json_nid_idx
+    ON federationsender_queue_json (json_nid);
 `
 
 const insertJSONSQL = "" +
diff --git a/federationapi/storage/postgres/queue_pdus_table.go b/federationapi/storage/postgres/queue_pdus_table.go
index f9a477483..38ac5a6eb 100644
--- a/federationapi/storage/postgres/queue_pdus_table.go
+++ b/federationapi/storage/postgres/queue_pdus_table.go
@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
     ON federationsender_queue_pdus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_server_name_idx
+    ON federationsender_queue_pdus (server_name);
 `
 
 const insertQueuePDUSQL = "" +
diff --git a/federationapi/storage/sqlite3/queue_edus_table.go b/federationapi/storage/sqlite3/queue_edus_table.go
index a6d609508..f4c84f094 100644
--- a/federationapi/storage/sqlite3/queue_edus_table.go
+++ b/federationapi/storage/sqlite3/queue_edus_table.go
@@ -37,6 +37,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
     ON federationsender_queue_edus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_nid_idx
+    ON federationsender_queue_edus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
+    ON federationsender_queue_edus (server_name);
 `
 
 const insertQueueEDUSQL = "" +
diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go
index 3e3f60f63..fe5896c7f 100644
--- a/federationapi/storage/sqlite3/queue_json_table.go
+++ b/federationapi/storage/sqlite3/queue_json_table.go
@@ -35,6 +35,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_json (
 	-- The JSON body. Text so that we preserve UTF-8.
 	json_body TEXT NOT NULL
 );
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_json_json_nid_idx
+    ON federationsender_queue_json (json_nid);
 `
 
 const insertJSONSQL = "" +
diff --git a/federationapi/storage/sqlite3/queue_pdus_table.go b/federationapi/storage/sqlite3/queue_pdus_table.go
index e0fdbda5f..e818585a5 100644
--- a/federationapi/storage/sqlite3/queue_pdus_table.go
+++ b/federationapi/storage/sqlite3/queue_pdus_table.go
@@ -38,6 +38,10 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
     ON federationsender_queue_pdus (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_json_nid_idx
+    ON federationsender_queue_pdus (json_nid);
+CREATE INDEX IF NOT EXISTS federationsender_queue_pdus_server_name_idx
+    ON federationsender_queue_pdus (server_name);
 `
 
 const insertQueuePDUSQL = "" +

From 9869dc2cbe8ae9e465c70f9a60f7b367773bed3d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 30 May 2022 11:11:21 +0100
Subject: [PATCH 146/391] Update readme

---
 README.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index e8b7bd0e2..7c22b3692 100644
--- a/README.md
+++ b/README.md
@@ -96,10 +96,9 @@ than features that massive deployments may be interested in (User Directory, Ope
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)
-- Full support for room versions 1 to 7
-- Experimental support for room versions 8 to 9
+- Room versions 1 to 10 supported
 - Backfilling locally and via federation
-- Accounts, Profiles and Devices
+- Accounts, profiles and devices
 - Published room lists
 - Typing
 - Media APIs

From fb52b6cedcd35b43f799f99767c5c9e01876dd5e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 30 May 2022 14:38:50 +0100
Subject: [PATCH 147/391] Sort `resolve-state` output

---
 cmd/resolve-state/main.go | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index 3a9f3ce4f..26f25b9ec 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -4,7 +4,9 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/storage"
@@ -110,7 +112,8 @@ func main() {
 	}
 
 	fmt.Println("Resolving state")
-	resolved, err := gomatrixserverlib.ResolveConflicts(
+	var resolved Events
+	resolved, err = gomatrixserverlib.ResolveConflicts(
 		gomatrixserverlib.RoomVersion(*roomVersion),
 		events,
 		authEvents,
@@ -120,6 +123,7 @@ func main() {
 	}
 
 	fmt.Println("Resolved state contains", len(resolved), "events")
+	sort.Sort(resolved)
 	filteringEventType := *filterType
 	count := 0
 	for _, event := range resolved {
@@ -135,3 +139,25 @@ func main() {
 	fmt.Println()
 	fmt.Println("Returned", count, "state events after filtering")
 }
+
+type Events []*gomatrixserverlib.Event
+
+func (e Events) Len() int {
+	return len(e)
+}
+
+func (e Events) Swap(i, j int) {
+	e[i], e[j] = e[j], e[i]
+}
+
+func (e Events) Less(i, j int) bool {
+	typeDelta := strings.Compare(e[i].Type(), e[j].Type())
+	if typeDelta < 0 {
+		return true
+	}
+	if typeDelta > 0 {
+		return false
+	}
+	stateKeyDelta := strings.Compare(*e[i].StateKey(), *e[j].StateKey())
+	return stateKeyDelta < 0
+}

From 9f8b3136b225b888b5e5e680a425545367584639 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 30 May 2022 15:11:48 +0100
Subject: [PATCH 148/391] Fix bugs in `resolve-state` tool

---
 cmd/resolve-state/main.go | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index 26f25b9ec..6ed6ebdb8 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
@@ -59,25 +60,23 @@ func main() {
 		panic(err)
 	}
 
-	blockNIDs, err := roomserverDB.StateBlockNIDs(ctx, snapshotNIDs)
-	if err != nil {
-		panic(err)
-	}
+	stateres := state.NewStateResolution(roomserverDB, &types.RoomInfo{
+		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
+	})
 
-	var stateEntries []types.StateEntryList
-	for _, list := range blockNIDs {
-		entries, err2 := roomserverDB.StateEntries(ctx, list.StateBlockNIDs)
-		if err2 != nil {
-			panic(err2)
+	var stateEntries []types.StateEntry
+	for _, snapshotNID := range snapshotNIDs {
+		var entries []types.StateEntry
+		entries, err = stateres.LoadStateAtSnapshot(ctx, snapshotNID)
+		if err != nil {
+			panic(err)
 		}
 		stateEntries = append(stateEntries, entries...)
 	}
 
 	var eventNIDs []types.EventNID
 	for _, entry := range stateEntries {
-		for _, e := range entry.StateEntries {
-			eventNIDs = append(eventNIDs, e.EventNID)
-		}
+		eventNIDs = append(eventNIDs, entry.EventNID)
 	}
 
 	fmt.Println("Fetching", len(eventNIDs), "state events")

From beed39a8f4c613388eddea8ebf95d93f4380196c Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 31 May 2022 11:27:04 +0200
Subject: [PATCH 149/391] Fix `pq: invalid input syntax for integer` when
 trying to get a numeric localpart (#2505)

* Match at least once

* Add the test
---
 userapi/storage/postgres/accounts_table.go | 2 +-
 userapi/storage/storage_test.go            | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go
index f86812f17..268889acc 100644
--- a/userapi/storage/postgres/accounts_table.go
+++ b/userapi/storage/postgres/accounts_table.go
@@ -65,7 +65,7 @@ const selectPasswordHashSQL = "" +
 	"SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE"
 
 const selectNewNumericLocalpartSQL = "" +
-	"SELECT COALESCE(MAX(localpart::integer), 0) FROM account_accounts WHERE localpart ~ '^[0-9]*$'"
+	"SELECT COALESCE(MAX(localpart::integer), 0) FROM account_accounts WHERE localpart ~ '^[0-9]{1,}$'"
 
 type accountsStatements struct {
 	insertAccountStmt             *sql.Stmt
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index 5bee880d3..66c15454f 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -124,6 +124,15 @@ func Test_Accounts(t *testing.T) {
 
 		_, err = db.GetAccountByLocalpart(ctx, "unusename")
 		assert.Error(t, err, "expected an error for non existent localpart")
+
+		// create an empty localpart; this should never happen, but is required to test getting a numeric localpart
+		// if there's already a user without a localpart in the database
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeUser)
+		assert.NoError(t, err)
+
+		// test getting a numeric localpart, with an existing user without a localpart
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeGuest)
+		assert.NoError(t, err)
 	})
 }
 

From ae7b6dd5164eb34cc65b5e43e576c6fcd9779351 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 31 May 2022 14:36:51 +0200
Subject: [PATCH 150/391] Fix #2498 (#2506)

---
 userapi/storage/postgres/accounts_table.go | 2 +-
 userapi/storage/storage_test.go            | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go
index 268889acc..e3cab56ee 100644
--- a/userapi/storage/postgres/accounts_table.go
+++ b/userapi/storage/postgres/accounts_table.go
@@ -65,7 +65,7 @@ const selectPasswordHashSQL = "" +
 	"SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE"
 
 const selectNewNumericLocalpartSQL = "" +
-	"SELECT COALESCE(MAX(localpart::integer), 0) FROM account_accounts WHERE localpart ~ '^[0-9]{1,}$'"
+	"SELECT COALESCE(MAX(localpart::bigint), 0) FROM account_accounts WHERE localpart ~ '^[0-9]{1,}$'"
 
 type accountsStatements struct {
 	insertAccountStmt             *sql.Stmt
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index 66c15454f..a26097338 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -133,6 +133,14 @@ func Test_Accounts(t *testing.T) {
 		// test getting a numeric localpart, with an existing user without a localpart
 		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeGuest)
 		assert.NoError(t, err)
+
+		// Create a user with a high numeric localpart, out of range for the Postgres integer (2147483647) type
+		_, err = db.CreateAccount(ctx, "2147483650", "", "", api.AccountTypeUser)
+		assert.NoError(t, err)
+
+		// Now try to create a new guest user
+		_, err = db.CreateAccount(ctx, "", "", "", api.AccountTypeGuest)
+		assert.NoError(t, err)
 	})
 }
 

From ea16614f71bcc2792df717c99dc857a68ac39c2b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jun 2022 09:39:17 +0100
Subject: [PATCH 151/391] Update versions

---
 clientapi/routing/routing.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 94becf465..fab45fdf6 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -89,6 +89,9 @@ func Setup(
 					"r0.4.0",
 					"r0.5.0",
 					"r0.6.1",
+					"v1.0",
+					"v1.1",
+					"v1.2",
 				}, UnstableFeatures: unstableFeatures},
 			}
 		}),

From 3d9fe207480f8b77f46e3e91a4852f92bdc8eb2a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jun 2022 09:46:21 +0100
Subject: [PATCH 152/391] Fix bugs related to state resolution (#2507)

* Fix bugs related to state resolution

* Clean up `resolve-state`

* Don't panic when entries can't be found

* Ensure we have state entries for the auth events

* Revert "Ensure we have state entries for the auth events"

This reverts commit 9b13b7ed37f40ce6d1301d9cb423a27b0db9c897.

* Revert "Revert "Ensure we have state entries for the auth events""

This reverts commit d86db197e3e317f7d64ec6722cc60533872f4617.

* Fix bug

* Try that again

* Update gomatrixserverlib

* Remove recursion from `loadAuthEvents`
---
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 .../internal/input/input_latest_events.go     |   2 +-
 roomserver/state/state.go                     | 125 +++++++++++++-----
 roomserver/storage/shared/room_updater.go     |   4 +
 5 files changed, 99 insertions(+), 38 deletions(-)

diff --git a/go.mod b/go.mod
index 580dc3568..c9a7e09b0 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220530084946-3a4b148706bc
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 75c22a7a7..4460b3905 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220530084946-3a4b148706bc h1:58tT3VznINdRWimb3yYb8QWmTAHX9AAuyOwzdmrp9q4=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220530084946-3a4b148706bc/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee h1:56sxEWrwB3eOmwjP2S4JsrQf29uBUaf+8WrbQJmjaGE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index a1b094871..59b3fcb12 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -206,7 +206,7 @@ func (u *latestEventsUpdater) latestState() error {
 	// Work out if the state at the extremities has actually changed
 	// or not. If they haven't then we won't bother doing all of the
 	// hard work.
-	if u.event.StateKey() == nil {
+	if !u.stateAtEvent.IsStateEvent() {
 		stateChanged := false
 		oldStateNIDs := make([]types.StateSnapshotNID, 0, len(u.oldLatest))
 		newStateNIDs := make([]types.StateSnapshotNID, 0, len(u.latest))
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index 187b996cd..95abdcb36 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -39,6 +39,7 @@ type StateResolutionStorage interface {
 	StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error)
 	AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error)
 	Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error)
+	EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
 }
 
 type StateResolution struct {
@@ -659,15 +660,13 @@ func (v *StateResolution) calculateStateAfterManyEvents(
 	}
 
 	// Collect all the entries with the same type and key together.
-	// We don't care about the order here because the conflict resolution
-	// algorithm doesn't depend on the order of the prev events.
-	// Remove duplicate entires.
+	// This is done so findDuplicateStateKeys can work in groups.
+	// We remove duplicates (same type, state key and event NID) too.
 	combined = combined[:util.SortAndUnique(stateEntrySorter(combined))]
 
 	// Find the conflicts
-	conflicts := findDuplicateStateKeys(combined)
-
-	if len(conflicts) > 0 {
+	if conflicts := findDuplicateStateKeys(combined); len(conflicts) > 0 {
+		conflictMap := stateEntryMap(conflicts)
 		conflictLength = len(conflicts)
 
 		// 5) There are conflicting state events, for each conflict workout
@@ -676,7 +675,7 @@ func (v *StateResolution) calculateStateAfterManyEvents(
 		// Work out which entries aren't conflicted.
 		var notConflicted []types.StateEntry
 		for _, entry := range combined {
-			if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok {
+			if _, ok := conflictMap.lookup(entry.StateKeyTuple); !ok {
 				notConflicted = append(notConflicted, entry)
 			}
 		}
@@ -689,7 +688,7 @@ func (v *StateResolution) calculateStateAfterManyEvents(
 			return
 		}
 		algorithm = "full_state_with_conflicts"
-		state = resolved[:util.SortAndUnique(stateEntrySorter(resolved))]
+		state = resolved
 	} else {
 		algorithm = "full_state_no_conflicts"
 		// 6) There weren't any conflicts
@@ -818,39 +817,19 @@ func (v *StateResolution) resolveConflictsV2(
 	authDifference := make([]*gomatrixserverlib.Event, 0, estimate)
 
 	// For each conflicted event, let's try and get the needed auth events.
-	neededStateKeys := make([]string, 16)
-	authEntries := make([]types.StateEntry, 16)
 	for _, conflictedEvent := range conflictedEvents {
 		// Work out which auth events we need to load.
 		key := conflictedEvent.EventID()
-		needed := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{conflictedEvent})
-
-		// Find the numeric IDs for the necessary state keys.
-		neededStateKeys = neededStateKeys[:0]
-		neededStateKeys = append(neededStateKeys, needed.Member...)
-		neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...)
-		stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys)
-		if err != nil {
-			return nil, err
-		}
-
-		// Load the necessary auth events.
-		tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed)
-		authEntries = authEntries[:0]
-		for _, tuple := range tuplesNeeded {
-			if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok {
-				authEntries = append(authEntries, types.StateEntry{
-					StateKeyTuple: tuple,
-					EventNID:      eventNID,
-				})
-			}
-		}
 
 		// Store the newly found auth events in the auth set for this event.
-		authSets[key], _, err = v.loadStateEvents(ctx, authEntries)
+		var authEventMap map[string]types.StateEntry
+		authSets[key], authEventMap, err = v.loadAuthEvents(ctx, conflictedEvent)
 		if err != nil {
 			return nil, err
 		}
+		for k, v := range authEventMap {
+			eventIDMap[k] = v
+		}
 
 		// Only add auth events into the authEvents slice once, otherwise the
 		// check for the auth difference can become expensive and produce
@@ -909,7 +888,7 @@ func (v *StateResolution) resolveConflictsV2(
 	for _, resolvedEvent := range resolvedEvents {
 		entry, ok := eventIDMap[resolvedEvent.EventID()]
 		if !ok {
-			panic(fmt.Errorf("missing state entry for event ID %q", resolvedEvent.EventID()))
+			return nil, fmt.Errorf("missing state entry for event ID %q", resolvedEvent.EventID())
 		}
 		notConflicted = append(notConflicted, entry)
 	}
@@ -996,6 +975,84 @@ func (v *StateResolution) loadStateEvents(
 	return result, eventIDMap, nil
 }
 
+// loadAuthEvents loads all of the auth events for a given event recursively,
+// along with a map that contains state entries for all of the auth events.
+func (v *StateResolution) loadAuthEvents(
+	ctx context.Context, event *gomatrixserverlib.Event,
+) ([]*gomatrixserverlib.Event, map[string]types.StateEntry, error) {
+	eventMap := map[string]struct{}{}
+	var lookup []string
+	var authEvents []types.Event
+	queue := event.AuthEventIDs()
+	for i := 0; i < len(queue); i++ {
+		lookup = lookup[:0]
+		for _, authEventID := range queue {
+			if _, ok := eventMap[authEventID]; ok {
+				continue
+			}
+			lookup = append(lookup, authEventID)
+		}
+		if len(lookup) == 0 {
+			break
+		}
+		events, err := v.db.EventsFromIDs(ctx, lookup)
+		if err != nil {
+			return nil, nil, fmt.Errorf("v.db.EventsFromIDs: %w", err)
+		}
+		add := map[string]struct{}{}
+		for _, event := range events {
+			eventMap[event.EventID()] = struct{}{}
+			authEvents = append(authEvents, event)
+			for _, authEventID := range event.AuthEventIDs() {
+				if _, ok := eventMap[authEventID]; ok {
+					continue
+				}
+				add[authEventID] = struct{}{}
+			}
+			for authEventID := range add {
+				queue = append(queue, authEventID)
+			}
+		}
+	}
+	authEventTypes := map[string]struct{}{}
+	authEventStateKeys := map[string]struct{}{}
+	for _, authEvent := range authEvents {
+		authEventTypes[authEvent.Type()] = struct{}{}
+		authEventStateKeys[*authEvent.StateKey()] = struct{}{}
+	}
+	lookupAuthEventTypes := make([]string, 0, len(authEventTypes))
+	lookupAuthEventStateKeys := make([]string, 0, len(authEventStateKeys))
+	for eventType := range authEventTypes {
+		lookupAuthEventTypes = append(lookupAuthEventTypes, eventType)
+	}
+	for eventStateKey := range authEventStateKeys {
+		lookupAuthEventStateKeys = append(lookupAuthEventStateKeys, eventStateKey)
+	}
+	eventTypes, err := v.db.EventTypeNIDs(ctx, lookupAuthEventTypes)
+	if err != nil {
+		return nil, nil, fmt.Errorf("v.db.EventTypeNIDs: %w", err)
+	}
+	eventStateKeys, err := v.db.EventStateKeyNIDs(ctx, lookupAuthEventStateKeys)
+	if err != nil {
+		return nil, nil, fmt.Errorf("v.db.EventStateKeyNIDs: %w", err)
+	}
+	stateEntryMap := map[string]types.StateEntry{}
+	for _, authEvent := range authEvents {
+		stateEntryMap[authEvent.EventID()] = types.StateEntry{
+			EventNID: authEvent.EventNID,
+			StateKeyTuple: types.StateKeyTuple{
+				EventTypeNID:     eventTypes[authEvent.Type()],
+				EventStateKeyNID: eventStateKeys[*authEvent.StateKey()],
+			},
+		}
+	}
+	nakedEvents := make([]*gomatrixserverlib.Event, 0, len(authEvents))
+	for _, authEvent := range authEvents {
+		nakedEvents = append(nakedEvents, authEvent.Event)
+	}
+	return nakedEvents, stateEntryMap, nil
+}
+
 // findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list.
 // Returns a sorted list of those state entries.
 func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry {
diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go
index d4a2ee3b9..8f4e011bf 100644
--- a/roomserver/storage/shared/room_updater.go
+++ b/roomserver/storage/shared/room_updater.go
@@ -192,6 +192,10 @@ func (u *RoomUpdater) StateAtEventIDs(
 	return u.d.EventsTable.BulkSelectStateAtEventByID(ctx, u.txn, eventIDs)
 }
 
+func (u *RoomUpdater) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) {
+	return u.d.eventsFromIDs(ctx, u.txn, eventIDs, false)
+}
+
 func (u *RoomUpdater) UnsentEventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) {
 	return u.d.eventsFromIDs(ctx, u.txn, eventIDs, true)
 }

From 70cd8c68c2e562fa8bda6dcf92d7f895c819659d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jun 2022 09:49:46 +0100
Subject: [PATCH 153/391] Reduce error levels on device list update

---
 keyserver/internal/device_list_update.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 23f3e1a67..acbcd5b8f 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -374,7 +374,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 	// fetch stale device lists
 	userIDs, err := u.db.StaleDeviceLists(ctx, []gomatrixserverlib.ServerName{serverName})
 	if err != nil {
-		logger.WithError(err).Error("failed to load stale device lists")
+		logger.WithError(err).Error("Failed to load stale device lists")
 		return waitTime, true
 	}
 	failCount := 0
@@ -399,7 +399,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 				}
 			} else {
 				waitTime = time.Hour
-				logger.WithError(err).WithField("user_id", userID).Warn("GetUserDevices returned unknown error type")
+				logger.WithError(err).WithField("user_id", userID).Debug("GetUserDevices returned unknown error type")
 			}
 			continue
 		}
@@ -422,12 +422,12 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 		}
 		err = u.updateDeviceList(&res)
 		if err != nil {
-			logger.WithError(err).WithField("user_id", userID).Error("fetched device list but failed to store/emit it")
+			logger.WithError(err).WithField("user_id", userID).Error("Fetched device list but failed to store/emit it")
 			failCount += 1
 		}
 	}
 	if failCount > 0 {
-		logger.WithField("total", len(userIDs)).WithField("failed", failCount).WithField("wait", waitTime).Error("failed to query device keys for some users")
+		logger.WithField("total", len(userIDs)).WithField("failed", failCount).WithField("wait", waitTime).Warn("Failed to query device keys for some users")
 	}
 	for _, userID := range userIDs {
 		// always clear the channel to unblock Update calls regardless of success/failure

From 02e5c741015300183eff9dcf812174d0efd66c4d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jun 2022 10:09:27 +0100
Subject: [PATCH 154/391] Revert #2457

Squashed commit of the following:

commit 2bd0daf4d61376d2dd56628eaff267b0bc63e116
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Wed Jun 1 09:55:54 2022 +0100

    Revert resolving old extremities as well as new

    This may no longer be needed with the new state fixes and probably just burns more CPU time than is strictly necessary.
---
 .../internal/input/input_latest_events.go     | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 59b3fcb12..f772299ab 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -234,19 +234,12 @@ func (u *latestEventsUpdater) latestState() error {
 		}
 	}
 
-	// Take the old set of extremities and the new set of extremities and
-	// mash them together into a list. This may or may not include the new event
-	// from the input path, depending on whether it became a forward extremity
-	// or not. We'll then run state resolution across all of them to determine
-	// the new current state of the room. Including the old extremities here
-	// ensures that new forward extremities with bad state snapshots (from
-	// possible malicious actors) can't completely corrupt the room state
-	// away from what it was before.
-	combinedExtremities := types.StateAtEventAndReferences(append(u.oldLatest, u.latest...))
-	combinedExtremities = combinedExtremities[:util.SortAndUnique(combinedExtremities)]
-	latestStateAtEvents := make([]types.StateAtEvent, len(combinedExtremities))
-	for i := range combinedExtremities {
-		latestStateAtEvents[i] = combinedExtremities[i].StateAtEvent
+	// Get a list of the current latest events. This may or may not
+	// include the new event from the input path, depending on whether
+	// it is a forward extremity or not.
+	latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
+	for i := range u.latest {
+		latestStateAtEvents[i] = u.latest[i].StateAtEvent
 	}
 
 	// Takes the NIDs of the latest events and creates a state snapshot

From f41931b56692c553ddf625ad54ffc8916b52e621 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 1 Jun 2022 11:40:37 +0100
Subject: [PATCH 155/391] Version 0.8.7 (#2509)

---
 CHANGES.md          | 12 ++++++++++++
 internal/version.go |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index c5e4f7f8c..a09a80148 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,17 @@
 # Changelog
 
+## Dendrite 0.8.7 (2022-06-01)
+
+### Features
+
+* Support added for room version 10
+
+### Fixes
+
+* A number of state handling bugs have been fixed, which previously resulted in missing state events, unexpected state deletions, reverted memberships and unexpectedly rejected/soft-failed events in some specific cases
+* Fixed destination queue performance issues as a result of missing indexes, which speeds up outbound federation considerably
+* A bug which could cause the `/register` endpoint to return HTTP 500 has been fixed
+
 ## Dendrite 0.8.6 (2022-05-26)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index 0957b4545..2543ec90c 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 6
+	VersionPatch = 7
 	VersionTag   = "" // example: "rc1"
 )
 

From 3e9c734da5151aac6b7073c0797d26cde529ced7 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 3 Jun 2022 06:43:51 +0200
Subject: [PATCH 156/391] Make setting state idempotent (#2512)

* Make Setting state twice is idempotent pass

* Add passing tests

* PR comment & comments
---
 clientapi/routing/sendevent.go | 43 ++++++++++++++++++++++++++++++++--
 sytest-whitelist               |  5 +++-
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index 70bf72f88..2e864adef 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"reflect"
 	"sync"
 	"time"
 
@@ -96,14 +97,21 @@ func SendEvent(
 	mutex.(*sync.Mutex).Lock()
 	defer mutex.(*sync.Mutex).Unlock()
 
-	startedGeneratingEvent := time.Now()
-
 	var r map[string]interface{} // must be a JSON object
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
 		return *resErr
 	}
 
+	if stateKey != nil {
+		// If the existing/new state content are equal, return the existing event_id, making the request idempotent.
+		if resp := stateEqual(req.Context(), rsAPI, eventType, *stateKey, roomID, r); resp != nil {
+			return *resp
+		}
+	}
+
+	startedGeneratingEvent := time.Now()
+
 	// If we're sending a membership update, make sure to strip the authorised
 	// via key if it is present, otherwise other servers won't be able to auth
 	// the event if the room is set to the "restricted" join rule.
@@ -208,6 +216,37 @@ func SendEvent(
 	return res
 }
 
+// stateEqual compares the new and the existing state event content. If they are equal, returns a *util.JSONResponse
+// with the existing event_id, making this an idempotent request.
+func stateEqual(ctx context.Context, rsAPI api.ClientRoomserverAPI, eventType, stateKey, roomID string, newContent map[string]interface{}) *util.JSONResponse {
+	stateRes := api.QueryCurrentStateResponse{}
+	tuple := gomatrixserverlib.StateKeyTuple{
+		EventType: eventType,
+		StateKey:  stateKey,
+	}
+	err := rsAPI.QueryCurrentState(ctx, &api.QueryCurrentStateRequest{
+		RoomID:      roomID,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
+	}, &stateRes)
+	if err != nil {
+		return nil
+	}
+	if existingEvent, ok := stateRes.StateEvents[tuple]; ok {
+		var existingContent map[string]interface{}
+		if err = json.Unmarshal(existingEvent.Content(), &existingContent); err != nil {
+			return nil
+		}
+		if reflect.DeepEqual(existingContent, newContent) {
+			return &util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: sendEventResponse{existingEvent.EventID()},
+			}
+		}
+
+	}
+	return nil
+}
+
 func generateSendEvent(
 	ctx context.Context,
 	r map[string]interface{},
diff --git a/sytest-whitelist b/sytest-whitelist
index 6af8d89ff..5f6797a3e 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -715,4 +715,7 @@ Presence can be set from sync
 PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
 Unnamed room comes with a name summary
 Named room comes with just joined member count summary
-Room summary only has 5 heroes
\ No newline at end of file
+Room summary only has 5 heroes
+Setting state twice is idempotent
+Joining room twice is idempotent
+Inbound federation can return missing events for shared visibility
\ No newline at end of file

From 02597f15f0b60dbea49f8540c87981f7967d5509 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 6 Jun 2022 08:56:06 +0100
Subject: [PATCH 157/391] Fix panic in `QueryRestrictedJoinAllowed`

---
 roomserver/internal/query/query.go | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 6d306fb09..da1b32530 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -778,11 +778,18 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 	} else if !allowRestrictedJoins {
 		return nil
 	}
+	// Start off by populating the "resident" flag in the response. If we
+	// come across any rooms in the request that are missing, we will unset
+	// the flag.
+	res.Resident = true
 	// Get the join rules to work out if the join rule is "restricted".
 	joinRulesEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomJoinRules, "")
 	if err != nil {
 		return fmt.Errorf("r.DB.GetStateEvent: %w", err)
 	}
+	if joinRulesEvent == nil {
+		return nil
+	}
 	var joinRules gomatrixserverlib.JoinRuleContent
 	if err = json.Unmarshal(joinRulesEvent.Content(), &joinRules); err != nil {
 		return fmt.Errorf("json.Unmarshal: %w", err)
@@ -792,10 +799,6 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 	if !res.Restricted {
 		return nil
 	}
-	// Start off by populating the "resident" flag in the response. If we
-	// come across any rooms in the request that are missing, we will unset
-	// the flag.
-	res.Resident = true
 	// If the user is already invited to the room then the join is allowed
 	// but we don't specify an authorised via user, since the event auth
 	// will allow the join anyway.

From 2cb609c428bbff81634edbc02315861660c1bc25 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 6 Jun 2022 15:18:02 +0100
Subject: [PATCH 158/391] Room upgrade tweaks

Squashed commit of the following:

commit 7a1568c716866594af6d0b1d561c58c96de29b20
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Mon Jun 6 15:17:49 2022 +0100

    Make errors more useful

commit 64befe7c9a901b00650442171660c2dc4ea575fa
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Mon Jun 6 15:02:40 2022 +0100

    Tweak ordering a bit
---
 .../internal/perform/perform_upgrade.go       | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go
index fcd19b936..8d975209a 100644
--- a/roomserver/internal/perform/perform_upgrade.go
+++ b/roomserver/internal/perform/perform_upgrade.go
@@ -105,13 +105,13 @@ func (r *Upgrader) performRoomUpgrade(
 		return "", pErr
 	}
 
-	// 5. Send the tombstone event to the old room (must do this before we set the new canonical_alias)
-	if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil {
+	// Send the setup events to the new room
+	if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil {
 		return "", pErr
 	}
 
-	// Send the setup events to the new room
-	if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil {
+	// 5. Send the tombstone event to the old room
+	if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil {
 		return "", pErr
 	}
 
@@ -147,7 +147,7 @@ func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*goma
 	if err != nil {
 		util.GetLogger(ctx).WithError(err).Error()
 		return nil, &api.PerformError{
-			Msg: "powerLevel event was not actually a power level event",
+			Msg: "Power level event was invalid or malformed",
 		}
 	}
 	return powerLevelContent, nil
@@ -198,7 +198,7 @@ func moveLocalAliases(ctx context.Context,
 	aliasRes := api.GetAliasesForRoomIDResponse{}
 	if err = URSAPI.GetAliasesForRoomID(ctx, &aliasReq, &aliasRes); err != nil {
 		return &api.PerformError{
-			Msg: "Could not get aliases for old room",
+			Msg: fmt.Sprintf("Failed to get old room aliases: %s", err),
 		}
 	}
 
@@ -207,7 +207,7 @@ func moveLocalAliases(ctx context.Context,
 		removeAliasRes := api.RemoveRoomAliasResponse{}
 		if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil {
 			return &api.PerformError{
-				Msg: "api.RemoveRoomAlias failed",
+				Msg: fmt.Sprintf("Failed to remove old room alias: %s", err),
 			}
 		}
 
@@ -215,7 +215,7 @@ func moveLocalAliases(ctx context.Context,
 		setAliasRes := api.SetRoomAliasResponse{}
 		if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil {
 			return &api.PerformError{
-				Msg: "api.SetRoomAlias failed",
+				Msg: fmt.Sprintf("Failed to set new room alias: %s", err),
 			}
 		}
 	}
@@ -509,7 +509,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user
 		err = builder.SetContent(e.Content)
 		if err != nil {
 			return &api.PerformError{
-				Msg: "builder.SetContent failed",
+				Msg: fmt.Sprintf("Failed to set content of new %q event: %s", builder.Type, err),
 			}
 		}
 		if i > 0 {
@@ -519,13 +519,13 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user
 		event, err = r.buildEvent(&builder, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion))
 		if err != nil {
 			return &api.PerformError{
-				Msg: "buildEvent failed",
+				Msg: fmt.Sprintf("Failed to build new %q event: %s", builder.Type, err),
 			}
 		}
 
 		if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil {
 			return &api.PerformError{
-				Msg: "gomatrixserverlib.Allowed failed",
+				Msg: fmt.Sprintf("Failed to auth new %q event: %s", builder.Type, err),
 			}
 		}
 
@@ -534,7 +534,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user
 		err = authEvents.AddEvent(event)
 		if err != nil {
 			return &api.PerformError{
-				Msg: "authEvents.AddEvent failed",
+				Msg: fmt.Sprintf("Failed to add new %q event to auth set: %s", builder.Type, err),
 			}
 		}
 	}
@@ -550,7 +550,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user
 	}
 	if err = api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil {
 		return &api.PerformError{
-			Msg: "api.SendInputRoomEvents failed",
+			Msg: fmt.Sprintf("Failed to send new room %q to roomserver: %s", newRoomID, err),
 		}
 	}
 	return nil
@@ -582,7 +582,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user
 	err := builder.SetContent(event.Content)
 	if err != nil {
 		return nil, &api.PerformError{
-			Msg: "builder.SetContent failed",
+			Msg: fmt.Sprintf("Failed to set new %q event content: %s", builder.Type, err),
 		}
 	}
 	var queryRes api.QueryLatestEventsAndStateResponse
@@ -607,7 +607,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user
 		}
 	} else if err != nil {
 		return nil, &api.PerformError{
-			Msg: "eventutil.BuildEvent failed",
+			Msg: fmt.Sprintf("Failed to build new %q event: %s", builder.Type, err),
 		}
 	}
 	// check to see if this user can perform this operation
@@ -619,7 +619,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user
 	if err = gomatrixserverlib.Allowed(headeredEvent.Event, &provider); err != nil {
 		return nil, &api.PerformError{
 			Code: api.PerformErrorNotAllowed,
-			Msg:  err.Error(), // TODO: Is this error string comprehensible to the client?
+			Msg:  fmt.Sprintf("Failed to auth new %q event: %s", builder.Type, err), // TODO: Is this error string comprehensible to the client?
 		}
 	}
 
@@ -676,7 +676,7 @@ func (r *Upgrader) sendHeaderedEvent(
 	})
 	if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil {
 		return &api.PerformError{
-			Msg: "api.SendInputRoomEvents failed",
+			Msg: fmt.Sprintf("Failed to send new %q event to roomserver: %s", headeredEvent.Type(), err),
 		}
 	}
 
@@ -703,7 +703,7 @@ func (r *Upgrader) buildEvent(
 		r.Cfg.Matrix.PrivateKey, roomVersion,
 	)
 	if err != nil {
-		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err)
+		return nil, err
 	}
 	return event, nil
 }

From 0d7020fbaf97fb3275697033a7c3c2e6cf6eeacd Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 6 Jun 2022 17:27:50 +0100
Subject: [PATCH 159/391] Send tombstone to other servers when upgrading rooms

---
 roomserver/internal/perform/perform_upgrade.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go
index 8d975209a..393d7dd14 100644
--- a/roomserver/internal/perform/perform_upgrade.go
+++ b/roomserver/internal/perform/perform_upgrade.go
@@ -111,7 +111,7 @@ func (r *Upgrader) performRoomUpgrade(
 	}
 
 	// 5. Send the tombstone event to the old room
-	if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil {
+	if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent, string(r.Cfg.Matrix.ServerName)); pErr != nil {
 		return "", pErr
 	}
 
@@ -182,7 +182,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T
 			return resErr
 		}
 	} else {
-		if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered); resErr != nil {
+		if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered, api.DoNotSendToOtherServers); resErr != nil {
 			return resErr
 		}
 	}
@@ -253,7 +253,7 @@ func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api
 			return resErr
 		}
 	} else {
-		if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent); resErr != nil {
+		if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent, api.DoNotSendToOtherServers); resErr != nil {
 			return resErr
 		}
 	}
@@ -666,13 +666,14 @@ func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelC
 func (r *Upgrader) sendHeaderedEvent(
 	ctx context.Context,
 	headeredEvent *gomatrixserverlib.HeaderedEvent,
+	sendAsServer string,
 ) *api.PerformError {
 	var inputs []api.InputRoomEvent
 	inputs = append(inputs, api.InputRoomEvent{
 		Kind:         api.KindNew,
 		Event:        headeredEvent,
 		Origin:       r.Cfg.Matrix.ServerName,
-		SendAsServer: api.DoNotSendToOtherServers,
+		SendAsServer: sendAsServer,
 	})
 	if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil {
 		return &api.PerformError{

From aafb7bf120d30c37219686a5bb528794b0ab44a2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 7 Jun 2022 10:46:21 +0100
Subject: [PATCH 160/391] Avoid panicking unnecessarily when shutting down the
 `gobind` P2P demos (#2520)

---
 build/gobind-pinecone/monolith.go  | 18 +++++++++++++++---
 build/gobind-yggdrasil/monolith.go |  4 ++--
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 664ca85d9..b44e110ad 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -261,7 +261,7 @@ func (m *DendriteMonolith) Start() {
 	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-	if err := cfg.Derive(); err != nil {
+	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
 
@@ -342,11 +342,23 @@ func (m *DendriteMonolith) Start() {
 
 	go func() {
 		m.logger.Info("Listening on ", cfg.Global.ServerName)
-		m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")))
+
+		switch m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")) {
+		case net.ErrClosed, http.ErrServerClosed:
+			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
+		default:
+			m.logger.Fatal(err)
+		}
 	}()
 	go func() {
 		logrus.Info("Listening on ", m.listener.Addr())
-		logrus.Fatal(http.Serve(m.listener, httpRouter))
+
+		switch http.Serve(m.listener, httpRouter) {
+		case net.ErrClosed, http.ErrServerClosed:
+			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
+		default:
+			m.logger.Fatal(err)
+		}
 	}()
 }
 
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 991bc462f..99b180c81 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -170,11 +170,11 @@ func (m *DendriteMonolith) Start() {
 
 	go func() {
 		m.logger.Info("Listening on ", ygg.DerivedServerName())
-		m.logger.Fatal(m.httpServer.Serve(ygg))
+		m.logger.Error(m.httpServer.Serve(ygg))
 	}()
 	go func() {
 		logrus.Info("Listening on ", m.listener.Addr())
-		logrus.Fatal(http.Serve(m.listener, httpRouter))
+		logrus.Error(http.Serve(m.listener, httpRouter))
 	}()
 	go func() {
 		logrus.Info("Sending wake-up message to known nodes")

From 27948fb30468315ce613402dc8cc1fa7dba01679 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 7 Jun 2022 14:23:26 +0100
Subject: [PATCH 161/391] Optimise `loadAuthEvents`, add roomserver tracing

---
 roomserver/internal/input/input_events.go     |  12 +
 .../internal/input/input_latest_events.go     |  16 +-
 roomserver/internal/input/input_membership.go |   4 +
 roomserver/internal/input/input_missing.go    |  30 +++
 roomserver/state/state.go                     | 217 ++++++++++++++----
 5 files changed, 228 insertions(+), 51 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index d437d7768..deb88ea82 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -33,6 +33,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
@@ -75,6 +76,11 @@ func (r *Inputer) processRoomEvent(
 	default:
 	}
 
+	span, ctx := opentracing.StartSpanFromContext(ctx, "processRoomEvent")
+	span.SetTag("room_id", input.Event.RoomID())
+	span.SetTag("event_id", input.Event.EventID())
+	defer span.Finish()
+
 	// Measure how long it takes to process this event.
 	started := time.Now()
 	defer func() {
@@ -411,6 +417,9 @@ func (r *Inputer) fetchAuthEvents(
 	known map[string]*types.Event,
 	servers []gomatrixserverlib.ServerName,
 ) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "fetchAuthEvents")
+	defer span.Finish()
+
 	unknown := map[string]struct{}{}
 	authEventIDs := event.AuthEventIDs()
 	if len(authEventIDs) == 0 {
@@ -526,6 +535,9 @@ func (r *Inputer) calculateAndSetState(
 	event *gomatrixserverlib.Event,
 	isRejected bool,
 ) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "calculateAndSetState")
+	defer span.Finish()
+
 	var succeeded bool
 	updater, err := r.DB.GetRoomUpdater(ctx, roomInfo)
 	if err != nil {
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index f772299ab..9738ed4e6 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -27,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -56,6 +57,9 @@ func (r *Inputer) updateLatestEvents(
 	transactionID *api.TransactionID,
 	rewritesState bool,
 ) (err error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "updateLatestEvents")
+	defer span.Finish()
+
 	var succeeded bool
 	updater, err := r.DB.GetRoomUpdater(ctx, roomInfo)
 	if err != nil {
@@ -200,6 +204,9 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 }
 
 func (u *latestEventsUpdater) latestState() error {
+	span, ctx := opentracing.StartSpanFromContext(u.ctx, "processEventWithMissingState")
+	defer span.Finish()
+
 	var err error
 	roomState := state.NewStateResolution(u.updater, u.roomInfo)
 
@@ -246,7 +253,7 @@ func (u *latestEventsUpdater) latestState() error {
 	// of the state after the events. The snapshot state will be resolved
 	// using the correct state resolution algorithm for the room.
 	u.newStateNID, err = roomState.CalculateAndStoreStateAfterEvents(
-		u.ctx, latestStateAtEvents,
+		ctx, latestStateAtEvents,
 	)
 	if err != nil {
 		return fmt.Errorf("roomState.CalculateAndStoreStateAfterEvents: %w", err)
@@ -258,7 +265,7 @@ func (u *latestEventsUpdater) latestState() error {
 	// another list of added ones. Replacing a value for a state-key tuple
 	// will result one removed (the old event) and one added (the new event).
 	u.removed, u.added, err = roomState.DifferenceBetweeenStateSnapshots(
-		u.ctx, u.oldStateNID, u.newStateNID,
+		ctx, u.oldStateNID, u.newStateNID,
 	)
 	if err != nil {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
@@ -278,7 +285,7 @@ func (u *latestEventsUpdater) latestState() error {
 	// Also work out the state before the event removes and the event
 	// adds.
 	u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots(
-		u.ctx, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID,
+		ctx, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID,
 	)
 	if err != nil {
 		return fmt.Errorf("roomState.DifferenceBetweeenStateSnapshots: %w", err)
@@ -294,6 +301,9 @@ func (u *latestEventsUpdater) calculateLatest(
 	newEvent *gomatrixserverlib.Event,
 	newStateAndRef types.StateAtEventAndReference,
 ) (bool, error) {
+	span, _ := opentracing.StartSpanFromContext(u.ctx, "calculateLatest")
+	defer span.Finish()
+
 	// First of all, get a list of all of the events in our current
 	// set of forward extremities.
 	existingRefs := make(map[string]*types.StateAtEventAndReference)
diff --git a/roomserver/internal/input/input_membership.go b/roomserver/internal/input/input_membership.go
index 3953586b2..3ce8791a3 100644
--- a/roomserver/internal/input/input_membership.go
+++ b/roomserver/internal/input/input_membership.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/opentracing/opentracing-go"
 )
 
 // updateMembership updates the current membership and the invites for each
@@ -34,6 +35,9 @@ func (r *Inputer) updateMemberships(
 	updater *shared.RoomUpdater,
 	removed, added []types.StateEntry,
 ) ([]api.OutputEvent, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "updateMemberships")
+	defer span.Finish()
+
 	changes := membershipChanges(removed, added)
 	var eventNIDs []types.EventNID
 	for _, change := range changes {
diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index 9c70076c2..edc153b7f 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -15,6 +15,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -59,6 +60,9 @@ type missingStateReq struct {
 func (t *missingStateReq) processEventWithMissingState(
 	ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion,
 ) (*parsedRespState, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "processEventWithMissingState")
+	defer span.Finish()
+
 	// We are missing the previous events for this events.
 	// This means that there is a gap in our view of the history of the
 	// room. There two ways that we can handle such a gap:
@@ -235,6 +239,9 @@ func (t *missingStateReq) processEventWithMissingState(
 }
 
 func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) (*parsedRespState, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "lookupResolvedStateBeforeEvent")
+	defer span.Finish()
+
 	type respState struct {
 		// A snapshot is considered trustworthy if it came from our own roomserver.
 		// That's because the state will have been through state resolution once
@@ -310,6 +317,9 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e
 // lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event)
 // added into the mix.
 func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (*parsedRespState, bool, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateAfterEvent")
+	defer span.Finish()
+
 	// try doing all this locally before we resort to querying federation
 	respState := t.lookupStateAfterEventLocally(ctx, roomID, eventID)
 	if respState != nil {
@@ -361,6 +371,9 @@ func (t *missingStateReq) cacheAndReturn(ev *gomatrixserverlib.Event) *gomatrixs
 }
 
 func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, roomID, eventID string) *parsedRespState {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateAfterEventLocally")
+	defer span.Finish()
+
 	var res parsedRespState
 	roomInfo, err := t.db.RoomInfo(ctx, roomID)
 	if err != nil {
@@ -435,12 +448,17 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room
 // the server supports.
 func (t *missingStateReq) lookupStateBeforeEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (
 	*parsedRespState, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateBeforeEvent")
+	defer span.Finish()
 
 	// Attempt to fetch the missing state using /state_ids and /events
 	return t.lookupMissingStateViaStateIDs(ctx, roomID, eventID, roomVersion)
 }
 
 func (t *missingStateReq) resolveStatesAndCheck(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, states []*parsedRespState, backwardsExtremity *gomatrixserverlib.Event) (*parsedRespState, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "resolveStatesAndCheck")
+	defer span.Finish()
+
 	var authEventList []*gomatrixserverlib.Event
 	var stateEventList []*gomatrixserverlib.Event
 	for _, state := range states {
@@ -484,6 +502,9 @@ retryAllowedState:
 // get missing events for `e`. If `isGapFilled`=true then `newEvents` contains all the events to inject,
 // without `e`. If `isGapFilled=false` then `newEvents` contains the response to /get_missing_events
 func (t *missingStateReq) getMissingEvents(ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) (newEvents []*gomatrixserverlib.Event, isGapFilled, prevStateKnown bool, err error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "getMissingEvents")
+	defer span.Finish()
+
 	logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
 	latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID)
 	if err != nil {
@@ -608,6 +629,9 @@ func (t *missingStateReq) isPrevStateKnown(ctx context.Context, e *gomatrixserve
 func (t *missingStateReq) lookupMissingStateViaState(
 	ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
 ) (respState *parsedRespState, err error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "lookupMissingStateViaState")
+	defer span.Finish()
+
 	state, err := t.federation.LookupState(ctx, t.origin, roomID, eventID, roomVersion)
 	if err != nil {
 		return nil, err
@@ -637,6 +661,9 @@ func (t *missingStateReq) lookupMissingStateViaState(
 
 func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) (
 	*parsedRespState, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "lookupMissingStateViaStateIDs")
+	defer span.Finish()
+
 	util.GetLogger(ctx).WithField("room_id", roomID).Infof("lookupMissingStateViaStateIDs %s", eventID)
 	// fetch the state event IDs at the time of the event
 	stateIDs, err := t.federation.LookupStateIDs(ctx, t.origin, roomID, eventID)
@@ -799,6 +826,9 @@ func (t *missingStateReq) createRespStateFromStateIDs(
 }
 
 func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, _, missingEventID string, localFirst bool) (*gomatrixserverlib.Event, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "lookupEvent")
+	defer span.Finish()
+
 	if localFirst {
 		// fetch from the roomserver
 		events, err := t.db.EventsFromIDs(ctx, []string{missingEventID})
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index 95abdcb36..6c4e4b860 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -20,9 +20,11 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"sync"
 	"time"
 
 	"github.com/matrix-org/util"
+	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/matrix-org/dendrite/roomserver/types"
@@ -62,6 +64,9 @@ func NewStateResolution(db StateResolutionStorage, roomInfo *types.RoomInfo) Sta
 func (v *StateResolution) LoadStateAtSnapshot(
 	ctx context.Context, stateNID types.StateSnapshotNID,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAtSnapshot")
+	defer span.Finish()
+
 	stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID})
 	if err != nil {
 		return nil, err
@@ -100,6 +105,9 @@ func (v *StateResolution) LoadStateAtSnapshot(
 func (v *StateResolution) LoadStateAtEvent(
 	ctx context.Context, eventID string,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAtEvent")
+	defer span.Finish()
+
 	snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID)
 	if err != nil {
 		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err)
@@ -122,6 +130,9 @@ func (v *StateResolution) LoadStateAtEvent(
 func (v *StateResolution) LoadCombinedStateAfterEvents(
 	ctx context.Context, prevStates []types.StateAtEvent,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadCombinedStateAfterEvents")
+	defer span.Finish()
+
 	stateNIDs := make([]types.StateSnapshotNID, len(prevStates))
 	for i, state := range prevStates {
 		stateNIDs[i] = state.BeforeStateSnapshotNID
@@ -194,6 +205,9 @@ func (v *StateResolution) LoadCombinedStateAfterEvents(
 func (v *StateResolution) DifferenceBetweeenStateSnapshots(
 	ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID,
 ) (removed, added []types.StateEntry, err error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.DifferenceBetweeenStateSnapshots")
+	defer span.Finish()
+
 	if oldStateNID == newStateNID {
 		// If the snapshot NIDs are the same then nothing has changed
 		return nil, nil, nil
@@ -255,6 +269,9 @@ func (v *StateResolution) LoadStateAtSnapshotForStringTuples(
 	stateNID types.StateSnapshotNID,
 	stateKeyTuples []gomatrixserverlib.StateKeyTuple,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAtSnapshotForStringTuples")
+	defer span.Finish()
+
 	numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples)
 	if err != nil {
 		return nil, err
@@ -269,6 +286,9 @@ func (v *StateResolution) stringTuplesToNumericTuples(
 	ctx context.Context,
 	stringTuples []gomatrixserverlib.StateKeyTuple,
 ) ([]types.StateKeyTuple, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.stringTuplesToNumericTuples")
+	defer span.Finish()
+
 	eventTypes := make([]string, len(stringTuples))
 	stateKeys := make([]string, len(stringTuples))
 	for i := range stringTuples {
@@ -311,6 +331,9 @@ func (v *StateResolution) loadStateAtSnapshotForNumericTuples(
 	stateNID types.StateSnapshotNID,
 	stateKeyTuples []types.StateKeyTuple,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadStateAtSnapshotForNumericTuples")
+	defer span.Finish()
+
 	stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID})
 	if err != nil {
 		return nil, err
@@ -359,6 +382,9 @@ func (v *StateResolution) LoadStateAfterEventsForStringTuples(
 	prevStates []types.StateAtEvent,
 	stateKeyTuples []gomatrixserverlib.StateKeyTuple,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAfterEventsForStringTuples")
+	defer span.Finish()
+
 	numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples)
 	if err != nil {
 		return nil, err
@@ -371,6 +397,9 @@ func (v *StateResolution) loadStateAfterEventsForNumericTuples(
 	prevStates []types.StateAtEvent,
 	stateKeyTuples []types.StateKeyTuple,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadStateAfterEventsForNumericTuples")
+	defer span.Finish()
+
 	if len(prevStates) == 1 {
 		// Fast path for a single event.
 		prevState := prevStates[0]
@@ -543,6 +572,9 @@ func (v *StateResolution) CalculateAndStoreStateBeforeEvent(
 	event *gomatrixserverlib.Event,
 	isRejected bool,
 ) (types.StateSnapshotNID, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.CalculateAndStoreStateBeforeEvent")
+	defer span.Finish()
+
 	// Load the state at the prev events.
 	prevStates, err := v.db.StateAtEventIDs(ctx, event.PrevEventIDs())
 	if err != nil {
@@ -559,6 +591,9 @@ func (v *StateResolution) CalculateAndStoreStateAfterEvents(
 	ctx context.Context,
 	prevStates []types.StateAtEvent,
 ) (types.StateSnapshotNID, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.CalculateAndStoreStateAfterEvents")
+	defer span.Finish()
+
 	metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)}
 
 	if len(prevStates) == 0 {
@@ -631,6 +666,9 @@ func (v *StateResolution) calculateAndStoreStateAfterManyEvents(
 	prevStates []types.StateAtEvent,
 	metrics calculateStateMetrics,
 ) (types.StateSnapshotNID, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.calculateAndStoreStateAfterManyEvents")
+	defer span.Finish()
+
 	state, algorithm, conflictLength, err :=
 		v.calculateStateAfterManyEvents(ctx, v.roomInfo.RoomVersion, prevStates)
 	metrics.algorithm = algorithm
@@ -649,6 +687,9 @@ func (v *StateResolution) calculateStateAfterManyEvents(
 	ctx context.Context, roomVersion gomatrixserverlib.RoomVersion,
 	prevStates []types.StateAtEvent,
 ) (state []types.StateEntry, algorithm string, conflictLength int, err error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.calculateStateAfterManyEvents")
+	defer span.Finish()
+
 	var combined []types.StateEntry
 	// Conflict resolution.
 	// First stage: load the state after each of the prev events.
@@ -701,6 +742,9 @@ func (v *StateResolution) resolveConflicts(
 	ctx context.Context, version gomatrixserverlib.RoomVersion,
 	notConflicted, conflicted []types.StateEntry,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.resolveConflicts")
+	defer span.Finish()
+
 	stateResAlgo, err := version.StateResAlgorithm()
 	if err != nil {
 		return nil, err
@@ -725,6 +769,8 @@ func (v *StateResolution) resolveConflictsV1(
 	ctx context.Context,
 	notConflicted, conflicted []types.StateEntry,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.resolveConflictsV1")
+	defer span.Finish()
 
 	// Load the conflicted events
 	conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted)
@@ -788,6 +834,9 @@ func (v *StateResolution) resolveConflictsV2(
 	ctx context.Context,
 	notConflicted, conflicted []types.StateEntry,
 ) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.resolveConflictsV2")
+	defer span.Finish()
+
 	estimate := len(conflicted) + len(notConflicted)
 	eventIDMap := make(map[string]types.StateEntry, estimate)
 
@@ -815,31 +864,47 @@ func (v *StateResolution) resolveConflictsV2(
 	authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3)
 	gotAuthEvents := make(map[string]struct{}, estimate*3)
 	authDifference := make([]*gomatrixserverlib.Event, 0, estimate)
+	knownAuthEvents := make(map[string]types.Event, estimate*3)
 
 	// For each conflicted event, let's try and get the needed auth events.
-	for _, conflictedEvent := range conflictedEvents {
-		// Work out which auth events we need to load.
-		key := conflictedEvent.EventID()
+	if err = func() error {
+		span, sctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadAuthEvents")
+		defer span.Finish()
 
-		// Store the newly found auth events in the auth set for this event.
-		var authEventMap map[string]types.StateEntry
-		authSets[key], authEventMap, err = v.loadAuthEvents(ctx, conflictedEvent)
-		if err != nil {
-			return nil, err
-		}
-		for k, v := range authEventMap {
-			eventIDMap[k] = v
+		loader := authEventLoader{
+			v:              v,
+			lookupFromDB:   make([]string, 0, len(conflictedEvents)*3),
+			lookupFromMem:  make([]string, 0, len(conflictedEvents)*3),
+			lookedUpEvents: make([]types.Event, 0, len(conflictedEvents)*3),
+			eventMap:       map[string]types.Event{},
 		}
+		for _, conflictedEvent := range conflictedEvents {
+			// Work out which auth events we need to load.
+			key := conflictedEvent.EventID()
 
-		// Only add auth events into the authEvents slice once, otherwise the
-		// check for the auth difference can become expensive and produce
-		// duplicate entries, which just waste memory and CPU time.
-		for _, event := range authSets[key] {
-			if _, ok := gotAuthEvents[event.EventID()]; !ok {
-				authEvents = append(authEvents, event)
-				gotAuthEvents[event.EventID()] = struct{}{}
+			// Store the newly found auth events in the auth set for this event.
+			var authEventMap map[string]types.StateEntry
+			authSets[key], authEventMap, err = loader.loadAuthEvents(sctx, conflictedEvent, knownAuthEvents)
+			if err != nil {
+				return err
+			}
+			for k, v := range authEventMap {
+				eventIDMap[k] = v
+			}
+
+			// Only add auth events into the authEvents slice once, otherwise the
+			// check for the auth difference can become expensive and produce
+			// duplicate entries, which just waste memory and CPU time.
+			for _, event := range authSets[key] {
+				if _, ok := gotAuthEvents[event.EventID()]; !ok {
+					authEvents = append(authEvents, event)
+					gotAuthEvents[event.EventID()] = struct{}{}
+				}
 			}
 		}
+		return nil
+	}(); err != nil {
+		return nil, err
 	}
 
 	// Kill the reference to this so that the GC may pick it up, since we no
@@ -870,19 +935,29 @@ func (v *StateResolution) resolveConflictsV2(
 	// Look through all of the auth events that we've been given and work out if
 	// there are any events which don't appear in all of the auth sets. If they
 	// don't then we add them to the auth difference.
-	for _, event := range authEvents {
-		if !isInAllAuthLists(event) {
-			authDifference = append(authDifference, event)
+	func() {
+		span, _ := opentracing.StartSpanFromContext(ctx, "isInAllAuthLists")
+		defer span.Finish()
+
+		for _, event := range authEvents {
+			if !isInAllAuthLists(event) {
+				authDifference = append(authDifference, event)
+			}
 		}
-	}
+	}()
 
 	// Resolve the conflicts.
-	resolvedEvents := gomatrixserverlib.ResolveStateConflictsV2(
-		conflictedEvents,
-		nonConflictedEvents,
-		authEvents,
-		authDifference,
-	)
+	resolvedEvents := func() []*gomatrixserverlib.Event {
+		span, _ := opentracing.StartSpanFromContext(ctx, "gomatrixserverlib.ResolveStateConflictsV2")
+		defer span.Finish()
+
+		return gomatrixserverlib.ResolveStateConflictsV2(
+			conflictedEvents,
+			nonConflictedEvents,
+			authEvents,
+			authDifference,
+		)
+	}()
 
 	// Map from the full events back to numeric state entries.
 	for _, resolvedEvent := range resolvedEvents {
@@ -947,6 +1022,9 @@ func (v *StateResolution) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.E
 func (v *StateResolution) loadStateEvents(
 	ctx context.Context, entries []types.StateEntry,
 ) ([]*gomatrixserverlib.Event, map[string]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadStateEvents")
+	defer span.Finish()
+
 	result := make([]*gomatrixserverlib.Event, 0, len(entries))
 	eventEntries := make([]types.StateEntry, 0, len(entries))
 	eventNIDs := make([]types.EventNID, 0, len(entries))
@@ -975,43 +1053,86 @@ func (v *StateResolution) loadStateEvents(
 	return result, eventIDMap, nil
 }
 
+type authEventLoader struct {
+	sync.Mutex
+	v              *StateResolution
+	lookupFromDB   []string      // scratch space
+	lookupFromMem  []string      // scratch space
+	lookedUpEvents []types.Event // scratch space
+	eventMap       map[string]types.Event
+}
+
 // loadAuthEvents loads all of the auth events for a given event recursively,
 // along with a map that contains state entries for all of the auth events.
-func (v *StateResolution) loadAuthEvents(
-	ctx context.Context, event *gomatrixserverlib.Event,
+func (l *authEventLoader) loadAuthEvents(
+	ctx context.Context, event *gomatrixserverlib.Event, eventMap map[string]types.Event,
 ) ([]*gomatrixserverlib.Event, map[string]types.StateEntry, error) {
-	eventMap := map[string]struct{}{}
-	var lookup []string
-	var authEvents []types.Event
+	l.Lock()
+	defer l.Unlock()
+	authEvents := []types.Event{}     // our returned list
+	included := map[string]struct{}{} // dedupes authEvents above
 	queue := event.AuthEventIDs()
 	for i := 0; i < len(queue); i++ {
-		lookup = lookup[:0]
+		// Reuse the same underlying memory, since it reduces the
+		// amount of allocations we make the more times we call
+		// loadAuthEvents.
+		l.lookupFromDB = l.lookupFromDB[:0]
+		l.lookupFromMem = l.lookupFromMem[:0]
+		l.lookedUpEvents = l.lookedUpEvents[:0]
+
+		// Separate out the list of events in the queue based on if
+		// we think we already know the event in memory or not.
 		for _, authEventID := range queue {
-			if _, ok := eventMap[authEventID]; ok {
+			if _, ok := included[authEventID]; ok {
 				continue
 			}
-			lookup = append(lookup, authEventID)
+			if _, ok := eventMap[authEventID]; ok {
+				l.lookupFromMem = append(l.lookupFromMem, authEventID)
+			} else {
+				l.lookupFromDB = append(l.lookupFromDB, authEventID)
+			}
 		}
-		if len(lookup) == 0 {
+		// If there's nothing to do, stop here.
+		if len(l.lookupFromDB) == 0 && len(l.lookupFromMem) == 0 {
 			break
 		}
-		events, err := v.db.EventsFromIDs(ctx, lookup)
-		if err != nil {
-			return nil, nil, fmt.Errorf("v.db.EventsFromIDs: %w", err)
+
+		// If we need to get events from the database, go and fetch
+		// those now.
+		if len(l.lookupFromDB) > 0 {
+			eventsFromDB, err := l.v.db.EventsFromIDs(ctx, l.lookupFromDB)
+			if err != nil {
+				return nil, nil, fmt.Errorf("v.db.EventsFromIDs: %w", err)
+			}
+			l.lookedUpEvents = append(l.lookedUpEvents, eventsFromDB...)
+			for _, event := range eventsFromDB {
+				eventMap[event.EventID()] = event
+			}
 		}
+
+		// Fill in the gaps with events that we already have in memory.
+		if len(l.lookupFromMem) > 0 {
+			for _, eventID := range l.lookupFromMem {
+				l.lookedUpEvents = append(l.lookedUpEvents, eventMap[eventID])
+			}
+		}
+
+		// From the events that we've retrieved, work out which auth
+		// events to look up on the next iteration.
 		add := map[string]struct{}{}
-		for _, event := range events {
-			eventMap[event.EventID()] = struct{}{}
+		for _, event := range l.lookedUpEvents {
 			authEvents = append(authEvents, event)
+			included[event.EventID()] = struct{}{}
+
 			for _, authEventID := range event.AuthEventIDs() {
-				if _, ok := eventMap[authEventID]; ok {
+				if _, ok := included[authEventID]; ok {
 					continue
 				}
 				add[authEventID] = struct{}{}
 			}
-			for authEventID := range add {
-				queue = append(queue, authEventID)
-			}
+		}
+		for authEventID := range add {
+			queue = append(queue, authEventID)
 		}
 	}
 	authEventTypes := map[string]struct{}{}
@@ -1028,11 +1149,11 @@ func (v *StateResolution) loadAuthEvents(
 	for eventStateKey := range authEventStateKeys {
 		lookupAuthEventStateKeys = append(lookupAuthEventStateKeys, eventStateKey)
 	}
-	eventTypes, err := v.db.EventTypeNIDs(ctx, lookupAuthEventTypes)
+	eventTypes, err := l.v.db.EventTypeNIDs(ctx, lookupAuthEventTypes)
 	if err != nil {
 		return nil, nil, fmt.Errorf("v.db.EventTypeNIDs: %w", err)
 	}
-	eventStateKeys, err := v.db.EventStateKeyNIDs(ctx, lookupAuthEventStateKeys)
+	eventStateKeys, err := l.v.db.EventStateKeyNIDs(ctx, lookupAuthEventStateKeys)
 	if err != nil {
 		return nil, nil, fmt.Errorf("v.db.EventStateKeyNIDs: %w", err)
 	}

From 6d4bd5d890eeab47bddfad5a48d37766f954171f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 7 Jun 2022 14:24:04 +0100
Subject: [PATCH 162/391] Rate limiting changes (#2519)

* Rate limiting changes

This makes the following changes:

* For logged in users, the rate limiting now applies to the device session rather than the remote IP address;
* For non-logged in users, the rate limiting continues to apply to remote address as it does today;
* It is now possible to add user IDs to the `exempt_user_ids` option under `rate_limiting` to exclude bots from rate limiting;
* Admin and appservice users are now exempt from rate limiting by default.

* Fix build with media API
---
 clientapi/routing/routing.go       | 50 +++++++++++++++---------------
 dendrite-sample.monolith.yaml      |  5 ++-
 dendrite-sample.polylith.yaml      |  5 ++-
 internal/httputil/rate_limiting.go | 31 +++++++++++++++---
 mediaapi/routing/routing.go        |  6 ++--
 setup/config/config_clientapi.go   |  4 +++
 6 files changed, 67 insertions(+), 34 deletions(-)

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index fab45fdf6..aa4b5a235 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -140,7 +140,7 @@ func Setup(
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 				// not specced, but ensure we're rate limiting requests to this endpoint
-				if r := rateLimits.Limit(req); r != nil {
+				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
 				}
 				vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -160,7 +160,7 @@ func Setup(
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 				// not specced, but ensure we're rate limiting requests to this endpoint
-				if r := rateLimits.Limit(req); r != nil {
+				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
 				}
 				return SendServerNotice(
@@ -190,7 +190,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/join/{roomIDOrAlias}",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -206,7 +206,7 @@ func Setup(
 	if mscCfg.Enabled("msc2753") {
 		v3mux.Handle("/peek/{roomIDOrAlias}",
 			httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-				if r := rateLimits.Limit(req); r != nil {
+				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
 				}
 				vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -226,7 +226,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/join",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -240,7 +240,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/leave",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -274,7 +274,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/invite",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -392,14 +392,14 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		if r := rateLimits.Limit(req); r != nil {
+		if r := rateLimits.Limit(req, nil); r != nil {
 			return *r
 		}
 		return Register(req, userAPI, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
 	v3mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
-		if r := rateLimits.Limit(req); r != nil {
+		if r := rateLimits.Limit(req, nil); r != nil {
 			return *r
 		}
 		return RegisterAvailable(req, cfg, userAPI)
@@ -473,7 +473,7 @@ func Setup(
 
 	v3mux.Handle("/rooms/{roomID}/typing/{userID}",
 		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -530,7 +530,7 @@ func Setup(
 
 	v3mux.Handle("/account/whoami",
 		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			return Whoami(req, device)
@@ -539,7 +539,7 @@ func Setup(
 
 	v3mux.Handle("/account/password",
 		httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			return Password(req, userAPI, device, cfg)
@@ -548,7 +548,7 @@ func Setup(
 
 	v3mux.Handle("/account/deactivate",
 		httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			return Deactivate(req, userInteractiveAuth, userAPI, device)
@@ -559,7 +559,7 @@ func Setup(
 
 	v3mux.Handle("/login",
 		httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, nil); r != nil {
 				return *r
 			}
 			return Login(req, userAPI, cfg)
@@ -667,7 +667,7 @@ func Setup(
 
 	v3mux.Handle("/pushrules/{scope}/{kind}/{ruleID}",
 		httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -733,7 +733,7 @@ func Setup(
 
 	v3mux.Handle("/profile/{userID}/avatar_url",
 		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -758,7 +758,7 @@ func Setup(
 
 	v3mux.Handle("/profile/{userID}/displayname",
 		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -797,7 +797,7 @@ func Setup(
 
 	v3mux.Handle("/voip/turnServer",
 		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			return RequestTurnServer(req, device, cfg)
@@ -876,7 +876,7 @@ func Setup(
 
 	v3mux.Handle("/user/{userID}/openid/request_token",
 		httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -889,7 +889,7 @@ func Setup(
 
 	v3mux.Handle("/user_directory/search",
 		httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			postContent := struct {
@@ -935,7 +935,7 @@ func Setup(
 
 	v3mux.Handle("/rooms/{roomID}/read_markers",
 		httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -948,7 +948,7 @@ func Setup(
 
 	v3mux.Handle("/rooms/{roomID}/forget",
 		httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -1025,7 +1025,7 @@ func Setup(
 
 	v3mux.Handle("/pushers/set",
 		httputil.MakeAuthAPI("set_pushers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			return SetPusher(req, device, userAPI)
@@ -1083,7 +1083,7 @@ func Setup(
 
 	v3mux.Handle("/capabilities",
 		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			return GetCapabilities(req, rsAPI)
@@ -1299,7 +1299,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, device); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index e974dbcba..ce5da2788 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -160,11 +160,14 @@ client_api:
 
   # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
   # number of "slots" have been taken by requests from a specific host. Each "slot"
-  # will be released after the cooloff time in milliseconds.
+  # will be released after the cooloff time in milliseconds. Server administrators
+  # and appservice users are exempt from rate limiting by default.
   rate_limiting:
     enabled: true
     threshold: 5
     cooloff_ms: 500
+    exempt_user_ids:
+    #  - @user:domain.com
 
 # Configuration for the Federation API.
 federation_api:
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 4b67aaa94..439f09b0a 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -163,11 +163,14 @@ client_api:
 
   # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
   # number of "slots" have been taken by requests from a specific host. Each "slot"
-  # will be released after the cooloff time in milliseconds.
+  # will be released after the cooloff time in milliseconds. Server administrators
+  # and appservice users are exempt from rate limiting by default.
   rate_limiting:
     enabled: true
     threshold: 5
     cooloff_ms: 500
+    exempt_user_ids:
+    #  - @user:domain.com
 
 # Configuration for the Federation API.
 federation_api:
diff --git a/internal/httputil/rate_limiting.go b/internal/httputil/rate_limiting.go
index c4f47c7b5..dab36481e 100644
--- a/internal/httputil/rate_limiting.go
+++ b/internal/httputil/rate_limiting.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
@@ -17,6 +18,7 @@ type RateLimits struct {
 	enabled          bool
 	requestThreshold int64
 	cooloffDuration  time.Duration
+	exemptUserIDs    map[string]struct{}
 }
 
 func NewRateLimits(cfg *config.RateLimiting) *RateLimits {
@@ -25,6 +27,10 @@ func NewRateLimits(cfg *config.RateLimiting) *RateLimits {
 		enabled:          cfg.Enabled,
 		requestThreshold: cfg.Threshold,
 		cooloffDuration:  time.Duration(cfg.CooloffMS) * time.Millisecond,
+		exemptUserIDs:    map[string]struct{}{},
+	}
+	for _, userID := range cfg.ExemptUserIDs {
+		l.exemptUserIDs[userID] = struct{}{}
 	}
 	if l.enabled {
 		go l.clean()
@@ -52,7 +58,7 @@ func (l *RateLimits) clean() {
 	}
 }
 
-func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse {
+func (l *RateLimits) Limit(req *http.Request, device *userapi.Device) *util.JSONResponse {
 	// If rate limiting is disabled then do nothing.
 	if !l.enabled {
 		return nil
@@ -67,9 +73,26 @@ func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse {
 
 	// First of all, work out if X-Forwarded-For was sent to us. If not
 	// then we'll just use the IP address of the caller.
-	caller := req.RemoteAddr
-	if forwardedFor := req.Header.Get("X-Forwarded-For"); forwardedFor != "" {
-		caller = forwardedFor
+	var caller string
+	if device != nil {
+		switch device.AccountType {
+		case userapi.AccountTypeAdmin:
+			return nil // don't rate-limit server administrators
+		case userapi.AccountTypeAppService:
+			return nil // don't rate-limit appservice users
+		default:
+			if _, ok := l.exemptUserIDs[device.UserID]; ok {
+				// If the user is exempt from rate limiting then do nothing.
+				return nil
+			}
+			caller = device.UserID + device.ID
+		}
+	} else {
+		if forwardedFor := req.Header.Get("X-Forwarded-For"); forwardedFor != "" {
+			caller = forwardedFor
+		} else {
+			caller = req.RemoteAddr
+		}
 	}
 
 	// Look up the caller's channel, if they have one.
diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go
index 76f07415b..196908184 100644
--- a/mediaapi/routing/routing.go
+++ b/mediaapi/routing/routing.go
@@ -62,7 +62,7 @@ func Setup(
 	uploadHandler := httputil.MakeAuthAPI(
 		"upload", userAPI,
 		func(req *http.Request, dev *userapi.Device) util.JSONResponse {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, dev); r != nil {
 				return *r
 			}
 			return Upload(req, cfg, dev, db, activeThumbnailGeneration)
@@ -70,7 +70,7 @@ func Setup(
 	)
 
 	configHandler := httputil.MakeAuthAPI("config", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		if r := rateLimits.Limit(req); r != nil {
+		if r := rateLimits.Limit(req, device); r != nil {
 			return *r
 		}
 		respondSize := &cfg.MaxFileSizeBytes
@@ -126,7 +126,7 @@ func makeDownloadAPI(
 		// Ratelimit requests
 		// NOTSPEC: The spec says everything at /media/ should be rate limited, but this causes issues with thumbnails (#2243)
 		if name != "thumbnail" {
-			if r := rateLimits.Limit(req); r != nil {
+			if r := rateLimits.Limit(req, nil); r != nil {
 				if err := json.NewEncoder(w).Encode(r); err != nil {
 					w.WriteHeader(http.StatusInternalServerError)
 					return
diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go
index bb786a145..ecf8f6bd5 100644
--- a/setup/config/config_clientapi.go
+++ b/setup/config/config_clientapi.go
@@ -134,6 +134,10 @@ type RateLimiting struct {
 	// The cooloff period in milliseconds after a request before the "slot"
 	// is freed again
 	CooloffMS int64 `yaml:"cooloff_ms"`
+
+	// A list of users that are exempt from rate limiting, i.e. if you want
+	// to run Mjolnir or other bots.
+	ExemptUserIDs []string `yaml:"exempt_user_ids"`
 }
 
 func (r *RateLimiting) Verify(configErrs *ConfigErrors) {

From b21a2223ef337c0f53cddfcba0b2e9ea923cd296 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 7 Jun 2022 15:04:00 +0100
Subject: [PATCH 163/391] Optimise state res v2 by parsing power level content
 less often (update to matrix-org/gomatrixserverlib@e938c07)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index c9a7e09b0..6a91b565a 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 4460b3905..1953f9dec 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee h1:56sxEWrwB3eOmwjP2S4JsrQf29uBUaf+8WrbQJmjaGE=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e h1:fOMXFu4H0PpFtjocRx/EHxdTyQwQ2kZPA8lDTWYN6kU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From c4df6d77231358f161e5243e62af6510258943a5 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 7 Jun 2022 15:34:58 +0100
Subject: [PATCH 164/391] Optimise state res v2 by only updating the allower
 when necessary (update to matrix-org/gomatrixserverlib@e55d796)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 6a91b565a..ea6e8caeb 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 1953f9dec..e21794f41 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e h1:fOMXFu4H0PpFtjocRx/EHxdTyQwQ2kZPA8lDTWYN6kU=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3 h1:2eYcBt8Kg+nW/xIJY5x8Uo2dQLjUF+oxLap00uFC5l8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 3cdefcf765f0373209e10b4a4085a7b41667868d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 8 Jun 2022 09:13:25 +0100
Subject: [PATCH 165/391] Update sample configs for `exempt_user_ids`

---
 dendrite-sample.monolith.yaml | 2 +-
 dendrite-sample.polylith.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index ce5da2788..c0134c542 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -167,7 +167,7 @@ client_api:
     threshold: 5
     cooloff_ms: 500
     exempt_user_ids:
-    #  - @user:domain.com
+    #  - "@user:domain.com"
 
 # Configuration for the Federation API.
 federation_api:
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 439f09b0a..b9e3e5e56 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -170,7 +170,7 @@ client_api:
     threshold: 5
     cooloff_ms: 500
     exempt_user_ids:
-    #  - @user:domain.com
+    #  - "@user:domain.com"
 
 # Configuration for the Federation API.
 federation_api:

From 289b3c56082e243b2d9674c650d2460226013aa1 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 9 Jun 2022 12:26:48 +0200
Subject: [PATCH 166/391] Allow "registration is idempotent, with username
 specified" to pass (#2488)

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 clientapi/routing/register.go | 57 +++++++++++++++++++++++------------
 sytest-whitelist              |  3 +-
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go
index eba4920c6..c4ac0f2e7 100644
--- a/clientapi/routing/register.go
+++ b/clientapi/routing/register.go
@@ -29,9 +29,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/tokens"
@@ -68,9 +69,10 @@ const (
 // It shouldn't be passed by value because it contains a mutex.
 type sessionsDict struct {
 	sync.RWMutex
-	sessions map[string][]authtypes.LoginType
-	params   map[string]registerRequest
-	timer    map[string]*time.Timer
+	sessions               map[string][]authtypes.LoginType
+	sessionCompletedResult map[string]registerResponse
+	params                 map[string]registerRequest
+	timer                  map[string]*time.Timer
 	// deleteSessionToDeviceID protects requests to DELETE /devices/{deviceID} from being abused.
 	// If a UIA session is started by trying to delete device1, and then UIA is completed by deleting device2,
 	// the delete request will fail for device2 since the UIA was initiated by trying to delete device1.
@@ -115,6 +117,7 @@ func (d *sessionsDict) deleteSession(sessionID string) {
 	delete(d.params, sessionID)
 	delete(d.sessions, sessionID)
 	delete(d.deleteSessionToDeviceID, sessionID)
+	delete(d.sessionCompletedResult, sessionID)
 	// stop the timer, e.g. because the registration was completed
 	if t, ok := d.timer[sessionID]; ok {
 		if !t.Stop() {
@@ -130,6 +133,7 @@ func (d *sessionsDict) deleteSession(sessionID string) {
 func newSessionsDict() *sessionsDict {
 	return &sessionsDict{
 		sessions:                make(map[string][]authtypes.LoginType),
+		sessionCompletedResult:  make(map[string]registerResponse),
 		params:                  make(map[string]registerRequest),
 		timer:                   make(map[string]*time.Timer),
 		deleteSessionToDeviceID: make(map[string]string),
@@ -173,6 +177,19 @@ func (d *sessionsDict) addDeviceToDelete(sessionID, deviceID string) {
 	d.deleteSessionToDeviceID[sessionID] = deviceID
 }
 
+func (d *sessionsDict) addCompletedRegistration(sessionID string, response registerResponse) {
+	d.Lock()
+	defer d.Unlock()
+	d.sessionCompletedResult[sessionID] = response
+}
+
+func (d *sessionsDict) getCompletedRegistration(sessionID string) (registerResponse, bool) {
+	d.RLock()
+	defer d.RUnlock()
+	result, ok := d.sessionCompletedResult[sessionID]
+	return result, ok
+}
+
 func (d *sessionsDict) getDeviceToDelete(sessionID string) (string, bool) {
 	d.RLock()
 	defer d.RUnlock()
@@ -544,6 +561,14 @@ func Register(
 		r.DeviceID = data.DeviceID
 		r.InitialDisplayName = data.InitialDisplayName
 		r.InhibitLogin = data.InhibitLogin
+		// Check if the user already registered using this session, if so, return that result
+		if response, ok := sessions.getCompletedRegistration(sessionID); ok {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: response,
+			}
+		}
+
 	}
 	if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil {
 		return *resErr
@@ -839,13 +864,6 @@ func completeRegistration(
 	displayName, deviceID *string,
 	accType userapi.AccountType,
 ) util.JSONResponse {
-	var registrationOK bool
-	defer func() {
-		if registrationOK {
-			sessions.deleteSession(sessionID)
-		}
-	}()
-
 	if username == "" {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -886,7 +904,6 @@ func completeRegistration(
 	// Check whether inhibit_login option is set. If so, don't create an access
 	// token or a device for this user
 	if inhibitLogin {
-		registrationOK = true
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: registerResponse{
@@ -920,15 +937,17 @@ func completeRegistration(
 		}
 	}
 
-	registrationOK = true
+	result := registerResponse{
+		UserID:      devRes.Device.UserID,
+		AccessToken: devRes.Device.AccessToken,
+		HomeServer:  accRes.Account.ServerName,
+		DeviceID:    devRes.Device.ID,
+	}
+	sessions.addCompletedRegistration(sessionID, result)
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: registerResponse{
-			UserID:      devRes.Device.UserID,
-			AccessToken: devRes.Device.AccessToken,
-			HomeServer:  accRes.Account.ServerName,
-			DeviceID:    devRes.Device.ID,
-		},
+		JSON: result,
 	}
 }
 
diff --git a/sytest-whitelist b/sytest-whitelist
index 5f6797a3e..21bbc396b 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -716,6 +716,7 @@ PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
 Unnamed room comes with a name summary
 Named room comes with just joined member count summary
 Room summary only has 5 heroes
+registration is idempotent, with username specified
 Setting state twice is idempotent
 Joining room twice is idempotent
-Inbound federation can return missing events for shared visibility
\ No newline at end of file
+Inbound federation can return missing events for shared visibility

From 83797573be87616bea0644918b82e2b20e8b78ca Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 9 Jun 2022 12:18:50 +0100
Subject: [PATCH 167/391] Version 0.8.8 (#2525)

---
 CHANGES.md          | 17 +++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index a09a80148..0db25f05a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,22 @@
 # Changelog
 
+## Dendrite 0.8.8 (2022-06-09)
+
+### Features
+
+* The performance of state resolution has been increased significantly for larger rooms
+* A number of changes have been made to rate limiting:
+  * Logged in users will now be rate-limited on a per-session basis rather than by remote IP
+  * Rate limiting no longer applies to admin or appservice users
+  * It is now possible to configure additional users that are exempt from rate limiting using the `exempt_user_ids` option in the `rate_limiting` section of the Dendrite config
+* Setting state is now idempotent via the client API state endpoints
+
+### Fixes
+
+* Room upgrades now properly propagate tombstone events to remote servers
+* Room upgrades will no longer send tombstone events if creating the upgraded room fails
+* A crash has been fixed when evaluating restricted room joins
+
 ## Dendrite 0.8.7 (2022-06-01)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index 2543ec90c..e29996f36 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 7
+	VersionPatch = 8
 	VersionTag   = "" // example: "rc1"
 )
 

From 660f7839f52f319a205dc61b96e43c730e5cb91a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 9 Jun 2022 18:38:07 +0200
Subject: [PATCH 168/391] Correctly redact events over federation (#2526)

* Ensure we check powerlevel/origin before redacting an event

* Add passing test

* Use pl.UserLevel

* Make check more readable, also check for the sender
---
 roomserver/storage/shared/storage.go | 30 ++++++++++++++++++++++++++--
 sytest-whitelist                     |  1 +
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index cc4a9fff5..67dcfdf38 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -823,13 +823,39 @@ func (d *Database) handleRedactions(
 		return nil, "", nil
 	}
 
+	// Get the power level from the database, so we can verify the user is allowed to redact the event
+	powerLevels, err := d.GetStateEvent(ctx, event.RoomID(), gomatrixserverlib.MRoomPowerLevels, "")
+	if err != nil {
+		return nil, "", fmt.Errorf("d.GetStateEvent: %w", err)
+	}
+	pl, err := powerLevels.PowerLevels()
+	if err != nil {
+		return nil, "", fmt.Errorf("unable to get powerlevels for room: %w", err)
+	}
+
+	redactUser := pl.UserLevel(redactionEvent.Sender())
+	switch {
+	case redactUser >= pl.Redact:
+		// The power level of the redaction event’s sender is greater than or equal to the redact level.
+	case redactedEvent.Origin() == redactionEvent.Origin() && redactedEvent.Sender() == redactionEvent.Sender():
+		// The domain of the redaction event’s sender matches that of the original event’s sender.
+	default:
+		return nil, "", nil
+	}
+
 	// mark the event as redacted
+	if redactionsArePermanent {
+		redactedEvent.Event = redactedEvent.Redact()
+	}
+
 	err = redactedEvent.SetUnsignedField("redacted_because", redactionEvent)
 	if err != nil {
 		return nil, "", fmt.Errorf("redactedEvent.SetUnsignedField: %w", err)
 	}
-	if redactionsArePermanent {
-		redactedEvent.Event = redactedEvent.Redact()
+	// NOTSPEC: sytest relies on this unspecced field existing :(
+	err = redactedEvent.SetUnsignedField("redacted_by", redactionEvent.EventID())
+	if err != nil {
+		return nil, "", fmt.Errorf("redactedEvent.SetUnsignedField: %w", err)
 	}
 	// overwrite the eventJSON table
 	err = d.EventJSONTable.InsertEventJSON(ctx, txn, redactedEvent.EventNID, redactedEvent.JSON())
diff --git a/sytest-whitelist b/sytest-whitelist
index 21bbc396b..60a3b73f6 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -720,3 +720,4 @@ registration is idempotent, with username specified
 Setting state twice is idempotent
 Joining room twice is idempotent
 Inbound federation can return missing events for shared visibility
+Inbound federation ignores redactions from invalid servers room > v3
\ No newline at end of file

From e2a64773cef3ed22544990e7e04daf91a80a473e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 10 Jun 2022 10:14:15 +0100
Subject: [PATCH 169/391] Add new next steps page to the documentation

---
 docs/installation/10_next_steps.md | 71 ++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 docs/installation/10_next_steps.md

diff --git a/docs/installation/10_next_steps.md b/docs/installation/10_next_steps.md
new file mode 100644
index 000000000..3110b5562
--- /dev/null
+++ b/docs/installation/10_next_steps.md
@@ -0,0 +1,71 @@
+---
+title: Next steps
+parent: Installation
+has_toc: true
+nav_order: 10
+permalink: /installation/start/nextsteps
+---
+
+# Next steps
+
+Now that you have Dendrite running, the following tweaks will improve the reliability
+of your installation.
+
+## File descriptor limit
+
+Most platforms have a limit on how many file descriptors a single process can open. All
+connections made by Dendrite consume file descriptors — this includes database connections
+and network requests to remote homeservers. When participating in large federated rooms
+where Dendrite must talk to many remote servers, it is often very easy to exhaust default
+limits which are quite low.
+
+We currently recommend setting the file descriptor limit to 65535 to avoid such
+issues. Dendrite will log immediately after startup if the file descriptor limit is too low:
+
+```
+level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
+```
+
+UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit
+by running `ulimit -Sn` and the hard limit with `ulimit -Hn`:
+
+```
+$ ulimit -Hn
+1048576
+
+$ ulimit -Sn
+1024
+```
+
+Increase the soft limit before starting Dendrite:
+
+```
+ulimit -Sn 65535
+```
+
+The log line at startup should no longer appear if the limit is sufficient.
+
+If you are running under a systemd service, you can instead add `LimitNOFILE=65535` option
+to the `[Service]` section of your service unit file.
+
+## DNS caching
+
+Dendrite has a built-in DNS cache which significantly reduces the load that Dendrite will
+place on your DNS resolver. This may also speed up outbound federation.
+
+Consider enabling the DNS cache by modifying the `global` section of your configuration file:
+
+```
+  dns_cache:
+    enabled: true
+    cache_size: 4096
+    cache_lifetime: 600s
+```
+
+## Time synchronisation
+
+Matrix relies heavily on TLS which requires the system time to be correct. If the clock
+drifts then you may find that federation no works reliably (or at all) and clients may
+struggle to connect to your Dendrite server.
+
+Ensure that the time is synchronised on your system by enabling NTP sync.

From 16ed1633b6f50da7a865ae7a09b03168fd496c4f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 10 Jun 2022 10:15:14 +0100
Subject: [PATCH 170/391] Highlighting in docs

---
 docs/installation/10_next_steps.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/installation/10_next_steps.md b/docs/installation/10_next_steps.md
index 3110b5562..02542b082 100644
--- a/docs/installation/10_next_steps.md
+++ b/docs/installation/10_next_steps.md
@@ -29,7 +29,7 @@ level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, i
 UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit
 by running `ulimit -Sn` and the hard limit with `ulimit -Hn`:
 
-```
+```bash
 $ ulimit -Hn
 1048576
 
@@ -39,7 +39,7 @@ $ ulimit -Sn
 
 Increase the soft limit before starting Dendrite:
 
-```
+```bash
 ulimit -Sn 65535
 ```
 
@@ -55,7 +55,7 @@ place on your DNS resolver. This may also speed up outbound federation.
 
 Consider enabling the DNS cache by modifying the `global` section of your configuration file:
 
-```
+```yaml
   dns_cache:
     enabled: true
     cache_size: 4096

From 10300722859ba345b8b21980e39a8a7022e6d35b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 10 Jun 2022 10:18:32 +0100
Subject: [PATCH 171/391] Rename the page to "Optimise your installation"

---
 .../installation/{10_next_steps.md => 10_optimisation.md} | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
 rename docs/installation/{10_next_steps.md => 10_optimisation.md} (93%)

diff --git a/docs/installation/10_next_steps.md b/docs/installation/10_optimisation.md
similarity index 93%
rename from docs/installation/10_next_steps.md
rename to docs/installation/10_optimisation.md
index 02542b082..c19b7a75e 100644
--- a/docs/installation/10_next_steps.md
+++ b/docs/installation/10_optimisation.md
@@ -1,15 +1,15 @@
 ---
-title: Next steps
+title: Optimise your installation
 parent: Installation
 has_toc: true
 nav_order: 10
-permalink: /installation/start/nextsteps
+permalink: /installation/start/optimisation
 ---
 
-# Next steps
+# Optimise your installation
 
 Now that you have Dendrite running, the following tweaks will improve the reliability
-of your installation.
+and performance of your installation.
 
 ## File descriptor limit
 

From 89d2adadbdef4ce62338d55d56b58fa58055d495 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 10 Jun 2022 10:58:04 +0100
Subject: [PATCH 172/391] Attempt to raise the file descriptor limit at startup
 (#2527)

---
 setup/base/sanity_unix.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/setup/base/sanity_unix.go b/setup/base/sanity_unix.go
index 0c1543e0b..c630d3f19 100644
--- a/setup/base/sanity_unix.go
+++ b/setup/base/sanity_unix.go
@@ -15,8 +15,21 @@ func platformSanityChecks() {
 	// If we run out of file descriptors, we might run into problems accessing
 	// PostgreSQL amongst other things. Complain at startup if we think the
 	// number of file descriptors is too low.
-	var rLimit syscall.Rlimit
-	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err == nil && rLimit.Cur < 65535 {
+	warn := func(rLimit *syscall.Rlimit) {
 		logrus.Warnf("IMPORTANT: Process file descriptor limit is currently %d, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues", rLimit.Cur)
 	}
+	var rLimit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err == nil && rLimit.Cur < 65535 {
+		// The file descriptor count is too low. Let's try to raise it.
+		rLimit.Cur = 65535
+		if err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+			// We failed to raise it, so log an error.
+			logrus.WithError(err).Warn("IMPORTANT: Failed to raise the file descriptor limit")
+			warn(&rLimit)
+		} else if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err == nil && rLimit.Cur < 65535 {
+			// We think we successfully raised the limit, but a second call to
+			// get the limit told us that we didn't succeed. Log an error.
+			warn(&rLimit)
+		}
+	}
 }

From 0a7f7dc716a25b1071bacc16cb91b31441174453 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 13 Jun 2022 10:16:30 +0100
Subject: [PATCH 173/391] Add `--difference` to `resolve-state` tool

---
 cmd/resolve-state/main.go | 54 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index 6ed6ebdb8..da0f70f01 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -28,7 +28,9 @@ import (
 
 var roomVersion = flag.String("roomversion", "5", "the room version to parse events as")
 var filterType = flag.String("filtertype", "", "the event types to filter on")
+var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots")
 
+// nolint:gocyclo
 func main() {
 	ctx := context.Background()
 	cfg := setup.ParseFlags(true)
@@ -64,6 +66,58 @@ func main() {
 		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
 	})
 
+	if *difference {
+		if len(snapshotNIDs) != 2 {
+			panic("need exactly two state snapshot NIDs to calculate difference")
+		}
+
+		removed, added, err := stateres.DifferenceBetweeenStateSnapshots(ctx, snapshotNIDs[0], snapshotNIDs[1])
+		if err != nil {
+			panic(err)
+		}
+
+		var eventNIDs []types.EventNID
+		for _, entry := range append(removed, added...) {
+			eventNIDs = append(eventNIDs, entry.EventNID)
+		}
+
+		eventEntries, err := roomserverDB.Events(ctx, eventNIDs)
+		if err != nil {
+			panic(err)
+		}
+
+		events := make(map[types.EventNID]*gomatrixserverlib.Event, len(eventEntries))
+		for _, entry := range eventEntries {
+			events[entry.EventNID] = entry.Event
+		}
+
+		if len(removed) > 0 {
+			fmt.Println("Removed:")
+			for _, r := range removed {
+				event := events[r.EventNID]
+				fmt.Println()
+				fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey())
+				fmt.Printf("  %s\n", string(event.Content()))
+			}
+		}
+
+		if len(removed) > 0 && len(added) > 0 {
+			fmt.Println()
+		}
+
+		if len(added) > 0 {
+			fmt.Println("Added:")
+			for _, a := range added {
+				event := events[a.EventNID]
+				fmt.Println()
+				fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey())
+				fmt.Printf("  %s\n", string(event.Content()))
+			}
+		}
+
+		return
+	}
+
 	var stateEntries []types.StateEntry
 	for _, snapshotNID := range snapshotNIDs {
 		var entries []types.StateEntry

From e1136f4d3e372f9c0b8a309a387193621a53ae6d Mon Sep 17 00:00:00 2001
From: Till Faelligen <davidf@element.io>
Date: Mon, 13 Jun 2022 11:46:59 +0200
Subject: [PATCH 174/391] Make the linter happy again

---
 cmd/resolve-state/main.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index da0f70f01..c02140003 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -38,6 +38,7 @@ func main() {
 		Type:  "std",
 		Level: "error",
 	})
+	cfg.ClientAPI.RegistrationDisabled = true
 	base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
 	args := flag.Args()
 
@@ -70,8 +71,8 @@ func main() {
 		if len(snapshotNIDs) != 2 {
 			panic("need exactly two state snapshot NIDs to calculate difference")
 		}
-
-		removed, added, err := stateres.DifferenceBetweeenStateSnapshots(ctx, snapshotNIDs[0], snapshotNIDs[1])
+		var removed, added []types.StateEntry
+		removed, added, err = stateres.DifferenceBetweeenStateSnapshots(ctx, snapshotNIDs[0], snapshotNIDs[1])
 		if err != nil {
 			panic(err)
 		}
@@ -81,7 +82,8 @@ func main() {
 			eventNIDs = append(eventNIDs, entry.EventNID)
 		}
 
-		eventEntries, err := roomserverDB.Events(ctx, eventNIDs)
+		var eventEntries []types.Event
+		eventEntries, err = roomserverDB.Events(ctx, eventNIDs)
 		if err != nil {
 			panic(err)
 		}

From c50095858341cc051e2db97fb85a1bb985f90c66 Mon Sep 17 00:00:00 2001
From: Emanuele Aliberti <dev@mtka.eu>
Date: Mon, 13 Jun 2022 13:08:46 +0200
Subject: [PATCH 175/391] generic CaddyFile in front of Dendrite (monolith)
 (#2531)

for Caddy 2.5.x

Co-authored-by: emanuele.aliberti <emanuele.aliberti@mtka.eu>
---
 docs/caddy/monolith/CaddyFile | 68 +++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 docs/caddy/monolith/CaddyFile

diff --git a/docs/caddy/monolith/CaddyFile b/docs/caddy/monolith/CaddyFile
new file mode 100644
index 000000000..cd93f9e10
--- /dev/null
+++ b/docs/caddy/monolith/CaddyFile
@@ -0,0 +1,68 @@
+{
+    # debug
+    admin off
+    email example@example.com
+    default_sni example.com
+    # Debug endpoint
+    # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory	
+}
+
+#######################################################################
+#   Snippets
+#______________________________________________________________________
+
+(handle_errors_maintenance) {
+	handle_errors {
+		@maintenance expression {http.error.status_code} == 502
+		rewrite @maintenance maintenance.html
+		root * "/path/to/service/pages"
+		file_server
+	}
+}
+
+(matrix-well-known-header) {
+    # Headers
+    header Access-Control-Allow-Origin "*"
+    header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+    header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+    header Content-Type "application/json"
+}
+
+#######################################################################
+
+example.com {
+
+	# ...
+
+	handle /.well-known/matrix/server {
+		import matrix-well-known-header
+		respond `{ "m.server": "matrix.example.com:443" }` 200
+	}
+	
+	handle /.well-known/matrix/client {
+		import matrix-well-known-header
+		respond `{ "m.homeserver": { "base_url": "https://matrix.example.com" } }` 200
+	}
+
+	import handle_errors_maintenance
+}
+
+example.com:8448 {
+	# server<->server HTTPS traffic
+    reverse_proxy http://dendrite-host:8008
+}
+
+matrix.example.com {
+
+	handle /_matrix/* {
+		# client<->server HTTPS traffic
+		reverse_proxy  http://dendrite-host:8008
+	}
+
+	handle_path /* {
+		# Client webapp (Element SPA or ...)
+		file_server {
+			root /path/to/www/example.com/matrix-web-client/
+		}	
+	}
+}

From 4c2a10f1a61a79ed8bbe17af1b28532c3d24c261 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 13 Jun 2022 15:11:10 +0100
Subject: [PATCH 176/391] Handle state before, send history visibility in
 output (#2532)

* Check state before event

* Tweaks

* Refactor a bit, include in output events

* Don't waste time if soft failed either

* Tweak control flow, comments, use GMSL history visibility type
---
 clientapi/routing/aliases.go                  |   2 +-
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 roomserver/api/output.go                      |   5 +-
 roomserver/internal/input/input_events.go     | 114 +++++++++++++++++-
 .../internal/input/input_latest_events.go     |  35 +++---
 syncapi/routing/context.go                    |   2 +-
 7 files changed, 142 insertions(+), 22 deletions(-)

diff --git a/clientapi/routing/aliases.go b/clientapi/routing/aliases.go
index 504d60265..68d0f4195 100644
--- a/clientapi/routing/aliases.go
+++ b/clientapi/routing/aliases.go
@@ -44,7 +44,7 @@ func GetAliases(
 		return util.ErrorResponse(fmt.Errorf("rsAPI.QueryCurrentState: %w", err))
 	}
 
-	visibility := "invite"
+	visibility := gomatrixserverlib.HistoryVisibilityInvited
 	if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok {
 		var err error
 		visibility, err = historyVisEvent.HistoryVisibility()
diff --git a/go.mod b/go.mod
index ea6e8caeb..b2a096751 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index e21794f41..3a35c47da 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3 h1:2eYcBt8Kg+nW/xIJY5x8Uo2dQLjUF+oxLap00uFC5l8=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a h1:jOkrb6twViAGTHHadA51sQwdloHT0Vx1MCptk9InTHo=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/roomserver/api/output.go b/roomserver/api/output.go
index a82bf8701..36d0625c7 100644
--- a/roomserver/api/output.go
+++ b/roomserver/api/output.go
@@ -161,6 +161,8 @@ type OutputNewRoomEvent struct {
 	// The transaction ID of the send request if sent by a local user and one
 	// was specified
 	TransactionID *TransactionID `json:"transaction_id,omitempty"`
+	// The history visibility of the event.
+	HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"`
 }
 
 func (o *OutputNewRoomEvent) NeededStateEventIDs() ([]*gomatrixserverlib.HeaderedEvent, []string) {
@@ -187,7 +189,8 @@ func (o *OutputNewRoomEvent) NeededStateEventIDs() ([]*gomatrixserverlib.Headere
 // should build their current room state up from OutputNewRoomEvents only.
 type OutputOldRoomEvent struct {
 	// The Event.
-	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
+	Event             *gomatrixserverlib.HeaderedEvent    `json:"event"`
+	HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"`
 }
 
 // An OutputNewInviteEvent is written whenever an invite becomes active.
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index deb88ea82..ff05f798c 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -295,6 +295,22 @@ func (r *Inputer) processRoomEvent(
 		}
 	}
 
+	// Get the state before the event so that we can work out if the event was
+	// allowed at the time, and also to get the history visibility. We won't
+	// bother doing this if the event was already rejected as it just ends up
+	// burning CPU time.
+	historyVisibility := gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive.
+	if rejectionErr == nil && !isRejected && !softfail {
+		var err error
+		historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev)
+		if err != nil {
+			return fmt.Errorf("r.processStateBefore: %w", err)
+		}
+		if rejectionErr != nil {
+			isRejected = true
+		}
+	}
+
 	// Store the event.
 	_, _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected || softfail)
 	if err != nil {
@@ -360,6 +376,7 @@ func (r *Inputer) processRoomEvent(
 			input.SendAsServer,  // send as server
 			input.TransactionID, // transaction ID
 			input.HasState,      // rewrites state?
+			historyVisibility,   // the history visibility before the event
 		); err != nil {
 			return fmt.Errorf("r.updateLatestEvents: %w", err)
 		}
@@ -368,7 +385,8 @@ func (r *Inputer) processRoomEvent(
 			{
 				Type: api.OutputTypeOldRoomEvent,
 				OldRoomEvent: &api.OutputOldRoomEvent{
-					Event: headered,
+					Event:             headered,
+					HistoryVisibility: historyVisibility,
 				},
 			},
 		})
@@ -402,6 +420,100 @@ func (r *Inputer) processRoomEvent(
 	return nil
 }
 
+// processStateBefore works out what the state is before the event and
+// then checks the event auths against the state at the time. It also
+// tries to determine what the history visibility was of the event at
+// the time, so that it can be sent in the output event to downstream
+// components.
+// nolint:nakedret
+func (r *Inputer) processStateBefore(
+	ctx context.Context,
+	input *api.InputRoomEvent,
+	missingPrev bool,
+) (historyVisibility gomatrixserverlib.HistoryVisibility, rejectionErr error, err error) {
+	historyVisibility = gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive.
+	event := input.Event.Unwrap()
+	isCreateEvent := event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("")
+	var stateBeforeEvent []*gomatrixserverlib.Event
+	switch {
+	case isCreateEvent:
+		// There's no state before a create event so there is nothing
+		// else to do.
+		return
+	case input.HasState:
+		// If we're overriding the state then we need to go and retrieve
+		// them from the database. It's a hard error if they are missing.
+		stateEvents, err := r.DB.EventsFromIDs(ctx, input.StateEventIDs)
+		if err != nil {
+			return "", nil, fmt.Errorf("r.DB.EventsFromIDs: %w", err)
+		}
+		stateBeforeEvent = make([]*gomatrixserverlib.Event, 0, len(stateEvents))
+		for _, entry := range stateEvents {
+			stateBeforeEvent = append(stateBeforeEvent, entry.Event)
+		}
+	case missingPrev:
+		// We don't know all of the prev events, so we can't work out
+		// the state before the event. Reject it in that case.
+		rejectionErr = fmt.Errorf("event %q has missing prev events", event.EventID())
+		return
+	case len(event.PrevEventIDs()) == 0:
+		// There should be prev events since it's not a create event.
+		// A non-create event that claims to have no prev events is
+		// invalid, so reject it.
+		rejectionErr = fmt.Errorf("event %q must have prev events", event.EventID())
+		return
+	default:
+		// For all non-create events, there must be prev events, so we'll
+		// ask the query API for the relevant tuples needed for auth. We
+		// will include the history visibility here even though we don't
+		// actually need it for auth, because we want to send it in the
+		// output events.
+		tuplesNeeded := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{event}).Tuples()
+		tuplesNeeded = append(tuplesNeeded, gomatrixserverlib.StateKeyTuple{
+			EventType: gomatrixserverlib.MRoomHistoryVisibility,
+			StateKey:  "",
+		})
+		stateBeforeReq := &api.QueryStateAfterEventsRequest{
+			RoomID:       event.RoomID(),
+			PrevEventIDs: event.PrevEventIDs(),
+			StateToFetch: tuplesNeeded,
+		}
+		stateBeforeRes := &api.QueryStateAfterEventsResponse{}
+		if err := r.Queryer.QueryStateAfterEvents(ctx, stateBeforeReq, stateBeforeRes); err != nil {
+			return "", nil, fmt.Errorf("r.Queryer.QueryStateAfterEvents: %w", err)
+		}
+		switch {
+		case !stateBeforeRes.RoomExists:
+			rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID())
+			return
+		case !stateBeforeRes.PrevEventsExist:
+			rejectionErr = fmt.Errorf("prev events of %q are not known", event.EventID())
+			return
+		default:
+			stateBeforeEvent = gomatrixserverlib.UnwrapEventHeaders(stateBeforeRes.StateEvents)
+		}
+	}
+	// At this point, stateBeforeEvent should be populated either by
+	// the supplied state in the input request, or from the prev events.
+	// Check whether the event is allowed or not.
+	stateBeforeAuth := gomatrixserverlib.NewAuthEvents(stateBeforeEvent)
+	if rejectionErr = gomatrixserverlib.Allowed(event, &stateBeforeAuth); rejectionErr != nil {
+		return
+	}
+	// Work out what the history visibility was at the time of the
+	// event.
+	for _, event := range stateBeforeEvent {
+		if event.Type() != gomatrixserverlib.MRoomHistoryVisibility || !event.StateKeyEquals("") {
+			continue
+		}
+		if hisVis, err := event.HistoryVisibility(); err == nil {
+			historyVisibility = hisVis
+			break
+		}
+	}
+	return
+}
+
 // fetchAuthEvents will check to see if any of the
 // auth events specified by the given event are unknown. If they are
 // then we will go off and request them from the federation and then
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 9738ed4e6..e76f4ba8d 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -56,6 +56,7 @@ func (r *Inputer) updateLatestEvents(
 	sendAsServer string,
 	transactionID *api.TransactionID,
 	rewritesState bool,
+	historyVisibility gomatrixserverlib.HistoryVisibility,
 ) (err error) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "updateLatestEvents")
 	defer span.Finish()
@@ -69,15 +70,16 @@ func (r *Inputer) updateLatestEvents(
 	defer sqlutil.EndTransactionWithCheck(updater, &succeeded, &err)
 
 	u := latestEventsUpdater{
-		ctx:           ctx,
-		api:           r,
-		updater:       updater,
-		roomInfo:      roomInfo,
-		stateAtEvent:  stateAtEvent,
-		event:         event,
-		sendAsServer:  sendAsServer,
-		transactionID: transactionID,
-		rewritesState: rewritesState,
+		ctx:               ctx,
+		api:               r,
+		updater:           updater,
+		roomInfo:          roomInfo,
+		stateAtEvent:      stateAtEvent,
+		event:             event,
+		sendAsServer:      sendAsServer,
+		transactionID:     transactionID,
+		rewritesState:     rewritesState,
+		historyVisibility: historyVisibility,
 	}
 
 	if err = u.doUpdateLatestEvents(); err != nil {
@@ -119,6 +121,8 @@ type latestEventsUpdater struct {
 	// The snapshots of current state before and after processing this event
 	oldStateNID types.StateSnapshotNID
 	newStateNID types.StateSnapshotNID
+	// The history visibility of the event itself (from the state before the event).
+	historyVisibility gomatrixserverlib.HistoryVisibility
 }
 
 func (u *latestEventsUpdater) doUpdateLatestEvents() error {
@@ -365,12 +369,13 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error)
 	}
 
 	ore := api.OutputNewRoomEvent{
-		Event:           u.event.Headered(u.roomInfo.RoomVersion),
-		RewritesState:   u.rewritesState,
-		LastSentEventID: u.lastEventIDSent,
-		LatestEventIDs:  latestEventIDs,
-		TransactionID:   u.transactionID,
-		SendAsServer:    u.sendAsServer,
+		Event:             u.event.Headered(u.roomInfo.RoomVersion),
+		RewritesState:     u.rewritesState,
+		LastSentEventID:   u.lastEventIDSent,
+		LatestEventIDs:    latestEventIDs,
+		TransactionID:     u.transactionID,
+		SendAsServer:      u.sendAsServer,
+		HistoryVisibility: u.historyVisibility,
 	}
 
 	eventIDMap, err := u.stateEventMap()
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 96438e184..d021d365d 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -97,7 +97,7 @@ func Context(
 	state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
 	// verify the user is allowed to see the context for this room/event
 	for _, x := range state {
-		var hisVis string
+		var hisVis gomatrixserverlib.HistoryVisibility
 		hisVis, err = x.HistoryVisibility()
 		if err != nil {
 			continue

From 1b90cc95367947fa00616b4426d0c894b33c9862 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 15 Jun 2022 12:50:02 +0200
Subject: [PATCH 177/391] Fix rare panic when returning user devices over
 federation (#2534)

---
 federationapi/routing/devices.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go
index 1a092645f..2f9da1f25 100644
--- a/federationapi/routing/devices.go
+++ b/federationapi/routing/devices.go
@@ -85,6 +85,9 @@ func GetUserDevices(
 			if targetKey, ok := targetUser[gomatrixserverlib.KeyID(dev.DeviceID)]; ok {
 				for sourceUserID, forSourceUser := range targetKey {
 					for sourceKeyID, sourceKey := range forSourceUser {
+						if device.Keys.Signatures == nil {
+							device.Keys.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{}
+						}
 						if _, ok := device.Keys.Signatures[sourceUserID]; !ok {
 							device.Keys.Signatures[sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{}
 						}

From 7120eb6bc943af6f725b0c61cfd110330f04064a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 15 Jun 2022 14:27:07 +0100
Subject: [PATCH 178/391] Add `InputDeviceListUpdate` to the keyserver, remove
 old input API (#2536)

* Add `InputDeviceListUpdate` to the keyserver, remove old input API

* Fix copyright

* Log more information when a device list update fails
---
 federationapi/federationapi.go          |  1 +
 federationapi/producers/syncapi.go      | 17 +++++
 federationapi/routing/send.go           |  8 +--
 keyserver/api/api.go                    | 10 ---
 keyserver/consumers/devicelistupdate.go | 82 +++++++++++++++++++++++++
 keyserver/internal/internal.go          | 11 ----
 keyserver/inthttp/client.go             | 14 -----
 keyserver/inthttp/server.go             | 11 ----
 keyserver/keyserver.go                  | 10 ++-
 setup/jetstream/streams.go              |  6 ++
 syncapi/internal/keychange_test.go      |  3 -
 11 files changed, 117 insertions(+), 56 deletions(-)
 create mode 100644 keyserver/consumers/devicelistupdate.go

diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index ff159beea..97bcc12a5 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -63,6 +63,7 @@ func AddPublicRoutes(
 		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
 		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
 		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicDeviceListUpdate:  cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
 		ServerName:             cfg.Matrix.ServerName,
 		UserAPI:                userAPI,
 	}
diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go
index 494150036..6453d026c 100644
--- a/federationapi/producers/syncapi.go
+++ b/federationapi/producers/syncapi.go
@@ -17,6 +17,7 @@ package producers
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"strconv"
 	"time"
 
@@ -34,6 +35,7 @@ type SyncAPIProducer struct {
 	TopicSendToDeviceEvent string
 	TopicTypingEvent       string
 	TopicPresenceEvent     string
+	TopicDeviceListUpdate  string
 	JetStream              nats.JetStreamContext
 	ServerName             gomatrixserverlib.ServerName
 	UserAPI                userapi.UserInternalAPI
@@ -161,3 +163,18 @@ func (p *SyncAPIProducer) SendPresence(
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err
 }
+
+func (p *SyncAPIProducer) SendDeviceListUpdate(
+	ctx context.Context, deviceListUpdate *gomatrixserverlib.DeviceListUpdateEvent,
+) (err error) {
+	m := nats.NewMsg(p.TopicDeviceListUpdate)
+	m.Header.Set(jetstream.UserID, deviceListUpdate.UserID)
+	m.Data, err = json.Marshal(deviceListUpdate)
+	if err != nil {
+		return fmt.Errorf("json.Marshal: %w", err)
+	}
+
+	log.Debugf("Sending device list update: %+v", m.Header)
+	_, err = p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index c25dabce9..43003be38 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -501,11 +501,7 @@ func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverli
 	} else if serverName != t.Origin {
 		return
 	}
-	var inputRes keyapi.InputDeviceListUpdateResponse
-	t.keyAPI.InputDeviceListUpdate(context.Background(), &keyapi.InputDeviceListUpdateRequest{
-		Event: payload,
-	}, &inputRes)
-	if inputRes.Error != nil {
-		util.GetLogger(ctx).WithError(inputRes.Error).WithField("user_id", payload.UserID).Error("failed to InputDeviceListUpdate")
+	if err := t.producer.SendDeviceListUpdate(ctx, &payload); err != nil {
+		util.GetLogger(ctx).WithError(err).WithField("user_id", payload.UserID).Error("failed to InputDeviceListUpdate")
 	}
 }
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index 140f03569..c0a1eedbb 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -62,8 +62,6 @@ type FederationKeyAPI interface {
 	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse)
 	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse)
 	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse)
-	// InputDeviceListUpdate from a federated server EDU
-	InputDeviceListUpdate(ctx context.Context, req *InputDeviceListUpdateRequest, res *InputDeviceListUpdateResponse)
 	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse)
 	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse)
 }
@@ -337,11 +335,3 @@ type QuerySignaturesResponse struct {
 	// The request error, if any
 	Error *KeyError
 }
-
-type InputDeviceListUpdateRequest struct {
-	Event gomatrixserverlib.DeviceListUpdateEvent
-}
-
-type InputDeviceListUpdateResponse struct {
-	Error *KeyError
-}
diff --git a/keyserver/consumers/devicelistupdate.go b/keyserver/consumers/devicelistupdate.go
new file mode 100644
index 000000000..f4f246280
--- /dev/null
+++ b/keyserver/consumers/devicelistupdate.go
@@ -0,0 +1,82 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/keyserver/internal"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
+)
+
+// DeviceListUpdateConsumer consumes device list updates that came in over federation.
+type DeviceListUpdateConsumer struct {
+	ctx       context.Context
+	jetstream nats.JetStreamContext
+	durable   string
+	topic     string
+	updater   *internal.DeviceListUpdater
+}
+
+// NewDeviceListUpdateConsumer creates a new DeviceListConsumer. Call Start() to begin consuming from key servers.
+func NewDeviceListUpdateConsumer(
+	process *process.ProcessContext,
+	cfg *config.KeyServer,
+	js nats.JetStreamContext,
+	updater *internal.DeviceListUpdater,
+) *DeviceListUpdateConsumer {
+	return &DeviceListUpdateConsumer{
+		ctx:       process.Context(),
+		jetstream: js,
+		durable:   cfg.Matrix.JetStream.Prefixed("KeyServerInputDeviceListConsumer"),
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
+		updater:   updater,
+	}
+}
+
+// Start consuming from key servers
+func (t *DeviceListUpdateConsumer) Start() error {
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		nats.DeliverAll(), nats.ManualAck(),
+	)
+}
+
+// onMessage is called in response to a message received on the
+// key change events topic from the key server.
+func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	var m gomatrixserverlib.DeviceListUpdateEvent
+	if err := json.Unmarshal(msg.Data, &m); err != nil {
+		logrus.WithError(err).Errorf("Failed to read from device list update input topic")
+		return true
+	}
+	err := t.updater.Update(ctx, m)
+	if err != nil {
+		logrus.WithFields(logrus.Fields{
+			"user_id":   m.UserID,
+			"device_id": m.DeviceID,
+			"stream_id": m.StreamID,
+			"prev_id":   m.PrevID,
+		}).WithError(err).Errorf("Failed to update device list")
+		return false
+	}
+	return true
+}
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index f8d0d69c3..c146b2aa0 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -47,17 +47,6 @@ func (a *KeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) {
 	a.UserAPI = i
 }
 
-func (a *KeyInternalAPI) InputDeviceListUpdate(
-	ctx context.Context, req *api.InputDeviceListUpdateRequest, res *api.InputDeviceListUpdateResponse,
-) {
-	err := a.Updater.Update(ctx, req.Event)
-	if err != nil {
-		res.Error = &api.KeyError{
-			Err: fmt.Sprintf("failed to update device list: %s", err),
-		}
-	}
-}
-
 func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyChangesRequest, res *api.QueryKeyChangesResponse) {
 	userIDs, latest, err := a.DB.KeyChanges(ctx, req.Offset, req.ToOffset)
 	if err != nil {
diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go
index abce81582..dac61d1ea 100644
--- a/keyserver/inthttp/client.go
+++ b/keyserver/inthttp/client.go
@@ -63,20 +63,6 @@ type httpKeyInternalAPI struct {
 func (h *httpKeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) {
 	// no-op: doesn't need it
 }
-func (h *httpKeyInternalAPI) InputDeviceListUpdate(
-	ctx context.Context, req *api.InputDeviceListUpdateRequest, res *api.InputDeviceListUpdateResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputDeviceListUpdate")
-	defer span.Finish()
-
-	apiURL := h.apiURL + InputDeviceListUpdatePath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
-	if err != nil {
-		res.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
-}
 
 func (h *httpKeyInternalAPI) PerformClaimKeys(
 	ctx context.Context,
diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go
index 8d557a768..5bf5976a8 100644
--- a/keyserver/inthttp/server.go
+++ b/keyserver/inthttp/server.go
@@ -25,17 +25,6 @@ import (
 )
 
 func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) {
-	internalAPIMux.Handle(InputDeviceListUpdatePath,
-		httputil.MakeInternalAPI("inputDeviceListUpdate", func(req *http.Request) util.JSONResponse {
-			request := api.InputDeviceListUpdateRequest{}
-			response := api.InputDeviceListUpdateResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.InputDeviceListUpdate(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
 	internalAPIMux.Handle(PerformClaimKeysPath,
 		httputil.MakeInternalAPI("performClaimKeys", func(req *http.Request) util.JSONResponse {
 			request := api.PerformClaimKeysRequest{}
diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go
index 3ffd3ba1e..cd506f981 100644
--- a/keyserver/keyserver.go
+++ b/keyserver/keyserver.go
@@ -18,6 +18,7 @@ import (
 	"github.com/gorilla/mux"
 	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/keyserver/consumers"
 	"github.com/matrix-org/dendrite/keyserver/internal"
 	"github.com/matrix-org/dendrite/keyserver/inthttp"
 	"github.com/matrix-org/dendrite/keyserver/producers"
@@ -59,10 +60,17 @@ func NewInternalAPI(
 	updater := internal.NewDeviceListUpdater(db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable
 	ap.Updater = updater
 	go func() {
-		if err := updater.Start(); err != nil {
+		if err = updater.Start(); err != nil {
 			logrus.WithError(err).Panicf("failed to start device list updater")
 		}
 	}()
 
+	dlConsumer := consumers.NewDeviceListUpdateConsumer(
+		base.ProcessContext, cfg, js, updater,
+	)
+	if err = dlConsumer.Start(); err != nil {
+		logrus.WithError(err).Panic("failed to start device list consumer")
+	}
+
 	return ap
 }
diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go
index 6594e6941..110808b1b 100644
--- a/setup/jetstream/streams.go
+++ b/setup/jetstream/streams.go
@@ -16,6 +16,7 @@ const (
 
 var (
 	InputRoomEvent          = "InputRoomEvent"
+	InputDeviceListUpdate   = "InputDeviceListUpdate"
 	OutputRoomEvent         = "OutputRoomEvent"
 	OutputSendToDeviceEvent = "OutputSendToDeviceEvent"
 	OutputKeyChangeEvent    = "OutputKeyChangeEvent"
@@ -45,6 +46,11 @@ var streams = []*nats.StreamConfig{
 		Retention: nats.InterestPolicy,
 		Storage:   nats.FileStorage,
 	},
+	{
+		Name:      InputDeviceListUpdate,
+		Retention: nats.InterestPolicy,
+		Storage:   nats.FileStorage,
+	},
 	{
 		Name:      OutputRoomEvent,
 		Retention: nats.InterestPolicy,
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index d9fb9cf82..219b35e2c 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -43,9 +43,6 @@ func (k *mockKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneT
 }
 func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *keyapi.QueryDeviceMessagesRequest, res *keyapi.QueryDeviceMessagesResponse) {
 
-}
-func (k *mockKeyAPI) InputDeviceListUpdate(ctx context.Context, req *keyapi.InputDeviceListUpdateRequest, res *keyapi.InputDeviceListUpdateResponse) {
-
 }
 func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *keyapi.QuerySignaturesRequest, res *keyapi.QuerySignaturesResponse) {
 }

From 920a20821ba55a22248c5f78bb76b615fec60a7a Mon Sep 17 00:00:00 2001
From: Jean Lucas <jean@4ray.co>
Date: Mon, 27 Jun 2022 04:15:19 -0400
Subject: [PATCH 179/391] Fix nats.go commit (#2540)

Signed-off-by: Jean Lucas <jean@4ray.co>
---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index b2a096751..1f2d5d9fd 100644
--- a/go.mod
+++ b/go.mod
@@ -2,7 +2,7 @@ module github.com/matrix-org/dendrite
 
 replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f
 
-replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e
+replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673
 
 require (
 	github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
diff --git a/go.sum b/go.sum
index 3a35c47da..3ae7977f3 100644
--- a/go.sum
+++ b/go.sum
@@ -484,6 +484,8 @@ github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:
 github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4=
 github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y=
 github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673 h1:TcKfa3Tf0qwUotv63PQVu2d1bBoLi2iEA4RHVMGDh5M=
+github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=

From 2086992caf67d033c42db3fb6bc5c2a294e72bfe Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 29 Jun 2022 10:49:12 +0200
Subject: [PATCH 180/391] Don't return `end` if there are not more messages
 (#2542)

* Be more spec compliant

* Move lazyLoadMembers to own method
---
 syncapi/routing/messages.go | 68 +++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 26 deletions(-)

diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index e55c661d6..24745cd55 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -50,7 +50,7 @@ type messagesReq struct {
 type messagesResp struct {
 	Start       string                          `json:"start"`
 	StartStream string                          `json:"start_stream,omitempty"` // NOTSPEC: used by Cerulean, so clients can hit /messages then immediately /sync with a latest sync token
-	End         string                          `json:"end"`
+	End         string                          `json:"end,omitempty"`
 	Chunk       []gomatrixserverlib.ClientEvent `json:"chunk"`
 	State       []gomatrixserverlib.ClientEvent `json:"state"`
 }
@@ -200,30 +200,6 @@ func OnIncomingMessagesRequest(
 		return jsonerror.InternalServerError()
 	}
 
-	// at least fetch the membership events for the users returned in chunk if LazyLoadMembers is set
-	state := []gomatrixserverlib.ClientEvent{}
-	if filter.LazyLoadMembers {
-		membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent)
-		for _, evt := range clientEvents {
-			// Don't add membership events the client should already know about
-			if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached {
-				continue
-			}
-			membership, err := db.GetStateEvent(req.Context(), roomID, gomatrixserverlib.MRoomMember, evt.Sender)
-			if err != nil {
-				util.GetLogger(req.Context()).WithError(err).Error("failed to get membership event for user")
-				continue
-			}
-			if membership != nil {
-				membershipToUser[evt.Sender] = membership
-				lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID())
-			}
-		}
-		for _, evt := range membershipToUser {
-			state = append(state, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatSync))
-		}
-	}
-
 	util.GetLogger(req.Context()).WithFields(logrus.Fields{
 		"from":         from.String(),
 		"to":           to.String(),
@@ -237,7 +213,13 @@ func OnIncomingMessagesRequest(
 		Chunk: clientEvents,
 		Start: start.String(),
 		End:   end.String(),
-		State: state,
+	}
+	res.applyLazyLoadMembers(req.Context(), db, roomID, device, filter.LazyLoadMembers, lazyLoadCache)
+
+	// If we didn't return any events, set the end to an empty string, so it will be omitted
+	// in the response JSON.
+	if len(res.Chunk) == 0 {
+		res.End = ""
 	}
 	if fromStream != nil {
 		res.StartStream = fromStream.String()
@@ -250,6 +232,40 @@ func OnIncomingMessagesRequest(
 	}
 }
 
+// applyLazyLoadMembers loads membership events for users returned in Chunk, if the filter has
+// LazyLoadMembers enabled.
+func (m *messagesResp) applyLazyLoadMembers(
+	ctx context.Context,
+	db storage.Database,
+	roomID string,
+	device *userapi.Device,
+	lazyLoad bool,
+	lazyLoadCache caching.LazyLoadCache,
+) {
+	if !lazyLoad {
+		return
+	}
+	membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent)
+	for _, evt := range m.Chunk {
+		// Don't add membership events the client should already know about
+		if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached {
+			continue
+		}
+		membership, err := db.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomMember, evt.Sender)
+		if err != nil {
+			util.GetLogger(ctx).WithError(err).Error("failed to get membership event for user")
+			continue
+		}
+		if membership != nil {
+			membershipToUser[evt.Sender] = membership
+			lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID())
+		}
+	}
+	for _, evt := range membershipToUser {
+		m.State = append(m.State, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatSync))
+	}
+}
+
 func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (forgotten bool, exists bool, err error) {
 	req := api.QueryMembershipForUserRequest{
 		RoomID: roomID,

From 2dea466685d0d4ab74d4cbd84af16b621d1269b3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 29 Jun 2022 12:32:24 +0100
Subject: [PATCH 181/391] Return an error if trying to invite a malformed user
 ID (#2543)

---
 roomserver/internal/perform/perform_invite.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index b0148a314..644c954b6 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -56,7 +56,14 @@ func (r *Inviter) PerformInvite(
 		return nil, fmt.Errorf("failed to load RoomInfo: %w", err)
 	}
 
-	_, domain, _ := gomatrixserverlib.SplitID('@', targetUserID)
+	_, domain, err := gomatrixserverlib.SplitID('@', targetUserID)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("The user ID %q is invalid!", targetUserID),
+		}
+		return nil, nil
+	}
 	isTargetLocal := domain == r.Cfg.Matrix.ServerName
 	isOriginLocal := event.Origin() == r.Cfg.Matrix.ServerName
 

From 519bc1124b051273019aae9b11617ebd796e962f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 29 Jun 2022 15:29:39 +0100
Subject: [PATCH 182/391] Add `evacuateUser` endpoint, use it when deactivating
 accounts (#2545)

* Add `evacuateUser` endpoint, use it when deactivating accounts

* Populate the API

* Clean up user devices when deactivating

* Include invites, delete pushers
---
 clientapi/routing/admin.go                   | 37 +++++++++++
 clientapi/routing/routing.go                 |  6 ++
 docs/administration/4_adminapi.md            |  6 ++
 roomserver/api/api.go                        |  8 +--
 roomserver/api/api_trace.go                  |  9 +++
 roomserver/api/perform.go                    |  9 +++
 roomserver/internal/api.go                   |  1 +
 roomserver/internal/perform/perform_admin.go | 70 ++++++++++++++++++++
 roomserver/inthttp/client.go                 | 18 +++++
 roomserver/inthttp/server.go                 | 11 +++
 userapi/internal/api.go                      | 26 ++++++++
 userapi/userapi.go                           |  1 +
 12 files changed, 197 insertions(+), 5 deletions(-)

diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index 125b3847d..523b88c99 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -47,3 +47,40 @@ func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserv
 		},
 	}
 }
+
+func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+	if device.AccountType != userapi.AccountTypeAdmin {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+		}
+	}
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	userID, ok := vars["userID"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting user ID."),
+		}
+	}
+	res := &roomserverAPI.PerformAdminEvacuateUserResponse{}
+	rsAPI.PerformAdminEvacuateUser(
+		req.Context(),
+		&roomserverAPI.PerformAdminEvacuateUserRequest{
+			UserID: userID,
+		},
+		res,
+	)
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: map[string]interface{}{
+			"affected": res.Affected,
+		},
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index aa4b5a235..0460850ef 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -129,6 +129,12 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
+		httputil.MakeAuthAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminEvacuateUser(req, device, rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index e33482ec9..51f56374b 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -19,6 +19,12 @@ This endpoint will instruct Dendrite to part all local users from the given `roo
 in the URL. It may take some time to complete. A JSON body will be returned containing
 the user IDs of all affected users.
 
+## `/_dendrite/admin/evacuateUser/{userID}`
+
+This endpoint will instruct Dendrite to part the given local `userID` in the URL from
+all rooms which they are currently joined. A JSON body will be returned containing
+the room IDs of all affected rooms.
+
 ## `/_synapse/admin/v1/register`
 
 Shared secret registration — please see the [user creation page](createusers) for
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index f87ff2962..38baa617f 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -140,11 +140,8 @@ type ClientRoomserverAPI interface {
 
 	// PerformRoomUpgrade upgrades a room to a newer version
 	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
-	PerformAdminEvacuateRoom(
-		ctx context.Context,
-		req *PerformAdminEvacuateRoomRequest,
-		res *PerformAdminEvacuateRoomResponse,
-	)
+	PerformAdminEvacuateRoom(ctx context.Context, req *PerformAdminEvacuateRoomRequest, res *PerformAdminEvacuateRoomResponse)
+	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse)
 	PerformPeek(ctx context.Context, req *PerformPeekRequest, res *PerformPeekResponse)
 	PerformUnpeek(ctx context.Context, req *PerformUnpeekRequest, res *PerformUnpeekResponse)
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
@@ -161,6 +158,7 @@ type UserRoomserverAPI interface {
 	QueryLatestEventsAndStateAPI
 	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
 	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
+	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse)
 }
 
 type FederationRoomserverAPI interface {
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index 92c5c1b1d..211f320ff 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -113,6 +113,15 @@ func (t *RoomserverInternalAPITrace) PerformAdminEvacuateRoom(
 	util.GetLogger(ctx).Infof("PerformAdminEvacuateRoom req=%+v res=%+v", js(req), js(res))
 }
 
+func (t *RoomserverInternalAPITrace) PerformAdminEvacuateUser(
+	ctx context.Context,
+	req *PerformAdminEvacuateUserRequest,
+	res *PerformAdminEvacuateUserResponse,
+) {
+	t.Impl.PerformAdminEvacuateUser(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformAdminEvacuateUser req=%+v res=%+v", js(req), js(res))
+}
+
 func (t *RoomserverInternalAPITrace) PerformInboundPeek(
 	ctx context.Context,
 	req *PerformInboundPeekRequest,
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 30aa2cf1b..d9ea9dd1c 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -223,3 +223,12 @@ type PerformAdminEvacuateRoomResponse struct {
 	Affected []string `json:"affected"`
 	Error    *PerformError
 }
+
+type PerformAdminEvacuateUserRequest struct {
+	UserID string `json:"user_id"`
+}
+
+type PerformAdminEvacuateUserResponse struct {
+	Affected []string `json:"affected"`
+	Error    *PerformError
+}
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index afef52da4..acdaeef6f 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -170,6 +170,7 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio
 		Cfg:     r.Cfg,
 		Inputer: r.Inputer,
 		Queryer: r.Queryer,
+		Leaver:  r.Leaver,
 	}
 
 	if err := r.Inputer.Start(); err != nil {
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index 2de6477cc..d3fb71099 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -16,6 +16,7 @@ package perform
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"time"
@@ -34,6 +35,7 @@ type Admin struct {
 	Cfg     *config.RoomServer
 	Queryer *query.Queryer
 	Inputer *input.Inputer
+	Leaver  *Leaver
 }
 
 // PerformEvacuateRoom will remove all local users from the given room.
@@ -160,3 +162,71 @@ func (r *Admin) PerformAdminEvacuateRoom(
 	inputRes := &api.InputRoomEventsResponse{}
 	r.Inputer.InputRoomEvents(ctx, inputReq, inputRes)
 }
+
+func (r *Admin) PerformAdminEvacuateUser(
+	ctx context.Context,
+	req *api.PerformAdminEvacuateUserRequest,
+	res *api.PerformAdminEvacuateUserResponse,
+) {
+	_, domain, err := gomatrixserverlib.SplitID('@', req.UserID)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("Malformed user ID: %s", err),
+		}
+		return
+	}
+	if domain != r.Cfg.Matrix.ServerName {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  "Can only evacuate local users using this endpoint",
+		}
+		return
+	}
+
+	roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, gomatrixserverlib.Join)
+	if err != nil && err != sql.ErrNoRows {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.GetRoomsByMembership: %s", err),
+		}
+		return
+	}
+
+	inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, gomatrixserverlib.Invite)
+	if err != nil && err != sql.ErrNoRows {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  fmt.Sprintf("r.DB.GetRoomsByMembership: %s", err),
+		}
+		return
+	}
+
+	for _, roomID := range append(roomIDs, inviteRoomIDs...) {
+		leaveReq := &api.PerformLeaveRequest{
+			RoomID: roomID,
+			UserID: req.UserID,
+		}
+		leaveRes := &api.PerformLeaveResponse{}
+		outputEvents, err := r.Leaver.PerformLeave(ctx, leaveReq, leaveRes)
+		if err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("r.Leaver.PerformLeave: %s", err),
+			}
+			return
+		}
+		if len(outputEvents) == 0 {
+			continue
+		}
+		if err := r.Inputer.WriteOutputEvents(roomID, outputEvents); err != nil {
+			res.Error = &api.PerformError{
+				Code: api.PerformErrorBadRequest,
+				Msg:  fmt.Sprintf("r.Inputer.WriteOutputEvents: %s", err),
+			}
+			return
+		}
+
+		res.Affected = append(res.Affected, roomID)
+	}
+}
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index 7b10ae657..2fa8afc49 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -40,6 +40,7 @@ const (
 	RoomserverPerformInboundPeekPath       = "/roomserver/performInboundPeek"
 	RoomserverPerformForgetPath            = "/roomserver/performForget"
 	RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom"
+	RoomserverPerformAdminEvacuateUserPath = "/roomserver/performAdminEvacuateUser"
 
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
@@ -305,6 +306,23 @@ func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom(
 	}
 }
 
+func (h *httpRoomserverInternalAPI) PerformAdminEvacuateUser(
+	ctx context.Context,
+	req *api.PerformAdminEvacuateUserRequest,
+	res *api.PerformAdminEvacuateUserResponse,
+) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAdminEvacuateUser")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverPerformAdminEvacuateUserPath
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
+		}
+	}
+}
+
 // QueryLatestEventsAndState implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
 	ctx context.Context,
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index ad4fdc469..993381585 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -129,6 +129,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverPerformAdminEvacuateUserPath,
+		httputil.MakeInternalAPI("performAdminEvacuateUser", func(req *http.Request) util.JSONResponse {
+			var request api.PerformAdminEvacuateUserRequest
+			var response api.PerformAdminEvacuateUserResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			r.PerformAdminEvacuateUser(req.Context(), &request, &response)
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(
 		RoomserverQueryPublishedRoomsPath,
 		httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse {
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 9d2f63c72..27ed15a01 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -33,6 +33,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/pushrules"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/producers"
@@ -49,6 +50,7 @@ type UserInternalAPI struct {
 	// AppServices is the list of all registered AS
 	AppServices []config.ApplicationService
 	KeyAPI      keyapi.UserKeyAPI
+	RSAPI       rsapi.UserRoomserverAPI
 }
 
 func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error {
@@ -452,6 +454,30 @@ func (a *UserInternalAPI) queryAppServiceToken(ctx context.Context, token, appSe
 
 // PerformAccountDeactivation deactivates the user's account, removing all ability for the user to login again.
 func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *api.PerformAccountDeactivationRequest, res *api.PerformAccountDeactivationResponse) error {
+	evacuateReq := &rsapi.PerformAdminEvacuateUserRequest{
+		UserID: fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName),
+	}
+	evacuateRes := &rsapi.PerformAdminEvacuateUserResponse{}
+	a.RSAPI.PerformAdminEvacuateUser(ctx, evacuateReq, evacuateRes)
+	if err := evacuateRes.Error; err != nil {
+		logrus.WithError(err).Errorf("Failed to evacuate user after account deactivation")
+	}
+
+	deviceReq := &api.PerformDeviceDeletionRequest{
+		UserID: fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName),
+	}
+	deviceRes := &api.PerformDeviceDeletionResponse{}
+	if err := a.PerformDeviceDeletion(ctx, deviceReq, deviceRes); err != nil {
+		return err
+	}
+
+	pusherReq := &api.PerformPusherDeletionRequest{
+		Localpart: req.Localpart,
+	}
+	if err := a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil {
+		return err
+	}
+
 	err := a.DB.DeactivateAccount(ctx, req.Localpart)
 	res.AccountDeactivated = err == nil
 	return err
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 603b416bf..2e86d6aa7 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -78,6 +78,7 @@ func NewInternalAPI(
 		ServerName:           cfg.Matrix.ServerName,
 		AppServices:          appServices,
 		KeyAPI:               keyAPI,
+		RSAPI:                rsAPI,
 		DisableTLSValidation: cfg.PushGatewayDisableTLSValidation,
 	}
 

From 561c159ad71d49da1eb16c492ef3e53fa876480b Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 30 Jun 2022 12:34:37 +0200
Subject: [PATCH 183/391] Silence presence logs (#2547)

---
 federationapi/consumers/presence.go | 2 +-
 federationapi/producers/syncapi.go  | 2 +-
 syncapi/consumers/presence.go       | 2 +-
 syncapi/streams/stream_presence.go  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/federationapi/consumers/presence.go b/federationapi/consumers/presence.go
index bfce1b28b..a65d2aa04 100644
--- a/federationapi/consumers/presence.go
+++ b/federationapi/consumers/presence.go
@@ -133,7 +133,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) b
 		return true
 	}
 
-	log.Debugf("sending presence EDU to %d servers", len(joined))
+	log.Tracef("sending presence EDU to %d servers", len(joined))
 	if err = t.queues.SendEDU(edu, t.ServerName, joined); err != nil {
 		log.WithError(err).Error("failed to send EDU")
 		return false
diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go
index 6453d026c..e371baaaa 100644
--- a/federationapi/producers/syncapi.go
+++ b/federationapi/producers/syncapi.go
@@ -159,7 +159,7 @@ func (p *SyncAPIProducer) SendPresence(
 	lastActiveTS := gomatrixserverlib.AsTimestamp(time.Now().Add(-(time.Duration(lastActiveAgo) * time.Millisecond)))
 
 	m.Header.Set("last_active_ts", strconv.Itoa(int(lastActiveTS)))
-	log.Debugf("Sending presence to syncAPI: %+v", m.Header)
+	log.Tracef("Sending presence to syncAPI: %+v", m.Header)
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err
 }
diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go
index bfd72d604..0217e1956 100644
--- a/syncapi/consumers/presence.go
+++ b/syncapi/consumers/presence.go
@@ -138,7 +138,7 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 	presence := msg.Header.Get("presence")
 	timestamp := msg.Header.Get("last_active_ts")
 	fromSync, _ := strconv.ParseBool(msg.Header.Get("from_sync"))
-	logrus.Debugf("syncAPI received presence event: %+v", msg.Header)
+	logrus.Tracef("syncAPI received presence event: %+v", msg.Header)
 
 	if fromSync { // do not process local presence changes; we already did this synchronously.
 		return true
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 35ce53cb6..877bcf141 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -111,7 +111,7 @@ func (p *PresenceStreamProvider) IncrementalSync(
 			currentlyActive := prevPresence.CurrentlyActive()
 			skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID
 			if skip {
-				req.Log.Debugf("Skipping presence, no change (%s)", presence.UserID)
+				req.Log.Tracef("Skipping presence, no change (%s)", presence.UserID)
 				continue
 			}
 		}

From 54bed4c5937c82e8565ed7839d711dc498848500 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 1 Jul 2022 09:37:54 +0100
Subject: [PATCH 184/391] Blacklist `Guest users can join guest_access rooms`
 test until it can be investigated

---
 sytest-blacklist | 1 +
 sytest-whitelist | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/sytest-blacklist b/sytest-blacklist
index be0826eee..bcc345f6e 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -48,3 +48,4 @@ Notifications can be viewed with GET /notifications
 # More flakey
 
 If remote user leaves room we no longer receive device updates
+Guest users can join guest_access rooms
diff --git a/sytest-whitelist b/sytest-whitelist
index 60a3b73f6..ea25c75d0 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -241,7 +241,6 @@ Inbound federation can receive v2 /send_join
 Message history can be paginated
 Backfill works correctly with history visibility set to joined
 Guest user cannot call /events globally
-Guest users can join guest_access rooms
 Guest user can set display names
 Guest user cannot upgrade other users
 Guest non-joined user cannot call /events on shared room

From 086f182e24e0651d1320199e90215f280350ef44 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 1 Jul 2022 09:50:06 +0100
Subject: [PATCH 185/391] Disable WebAssembly builds for now

---
 .github/workflows/dendrite.yml | 1 +
 build.sh                       | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 5d60301c7..6ebef4e13 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -17,6 +17,7 @@ jobs:
     name: WASM build test
     timeout-minutes: 5
     runs-on: ubuntu-latest
+    if: ${{ false }} # disable for now
     steps:
       - uses: actions/checkout@v2
 
diff --git a/build.sh b/build.sh
index 700e6434f..f8b5001bf 100755
--- a/build.sh
+++ b/build.sh
@@ -21,4 +21,4 @@ mkdir -p bin
 
 CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
 
-CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone
+# CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

From 89cd0e8fc13b040470aebe2eb4d36a9235b1473d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 1 Jul 2022 11:49:26 +0200
Subject: [PATCH 186/391] Try to fix backfilling (#2548)

* Try to fix backfilling

* Return start/end to not confuse clients

* Update GMSL

* Update GMSL
---
 go.mod                                        |  2 +-
 go.sum                                        |  6 +--
 .../internal/perform/perform_backfill.go      | 45 +++++++++++++------
 syncapi/routing/context.go                    | 21 ++++++++-
 4 files changed, 54 insertions(+), 20 deletions(-)

diff --git a/go.mod b/go.mod
index 1f2d5d9fd..4c432611e 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 3ae7977f3..777a31228 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a h1:jOkrb6twViAGTHHadA51sQwdloHT0Vx1MCptk9InTHo=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f h1:XF2+J6sOq07yhK1I7ItwsgRwXorjj7gqiCvgZ4dn8W8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -482,8 +482,6 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo=
 github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4=
-github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y=
-github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673 h1:TcKfa3Tf0qwUotv63PQVu2d1bBoLi2iEA4RHVMGDh5M=
 github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 1bc4c75ce..9eddca733 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/getsentry/sentry-go"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -206,8 +207,17 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom
 			}
 			logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result)
 			for _, res := range result {
-				if res.Error != nil {
-					logger.WithError(res.Error).Warn("event failed PDU checks")
+				switch err := res.Error.(type) {
+				case nil:
+				case gomatrixserverlib.SignatureErr:
+					// The signature of the event might not be valid anymore, for example if
+					// the key ID was reused with a different signature.
+					logger.WithError(err).Errorf("event failed PDU checks, storing anyway")
+				case gomatrixserverlib.AuthChainErr, gomatrixserverlib.AuthRulesErr:
+					logger.WithError(err).Warn("event failed PDU checks")
+					continue
+				default:
+					logger.WithError(err).Warn("event failed PDU checks")
 					continue
 				}
 				missingMap[id] = res.Event
@@ -306,6 +316,7 @@ FederationHit:
 		b.eventIDToBeforeStateIDs[targetEvent.EventID()] = res
 		return res, nil
 	}
+	sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
 	return nil, lastErr
 }
 
@@ -366,19 +377,25 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
 		}
 	}
 
-	c := gomatrixserverlib.FederatedStateProvider{
-		FedClient:          b.fsAPI,
-		RememberAuthEvents: false,
-		Server:             b.servers[0],
+	var lastErr error
+	for _, srv := range b.servers {
+		c := gomatrixserverlib.FederatedStateProvider{
+			FedClient:          b.fsAPI,
+			RememberAuthEvents: false,
+			Server:             srv,
+		}
+		result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		for eventID, ev := range result {
+			b.eventIDMap[eventID] = ev
+		}
+		return result, nil
 	}
-	result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs)
-	if err != nil {
-		return nil, err
-	}
-	for eventID, ev := range result {
-		b.eventIDMap[eventID] = ev
-	}
-	return result, nil
+	sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
+	return nil, lastErr
 }
 
 // ServersAtEvent is called when trying to determine which server to request from.
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index d021d365d..f6b4d15e0 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
@@ -25,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/caching"
 	roomserver "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -149,13 +151,30 @@ func Context(
 	if len(response.State) > filter.Limit {
 		response.State = response.State[len(response.State)-filter.Limit:]
 	}
-
+	start, end, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter)
+	if err == nil {
+		response.End = end.String()
+		response.Start = start.String()
+	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: response,
 	}
 }
 
+func getStartEnd(ctx context.Context, syncDB storage.Database, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) {
+	if len(startEvents) > 0 {
+		start, err = syncDB.EventPositionInTopology(ctx, startEvents[0].EventID())
+		if err != nil {
+			return
+		}
+	}
+	if len(endEvents) > 0 {
+		end, err = syncDB.EventPositionInTopology(ctx, endEvents[0].EventID())
+	}
+	return
+}
+
 func applyLazyLoadMembers(
 	device *userapi.Device,
 	filter *gomatrixserverlib.RoomEventFilter,

From b50a24c666c4c45e1410dfc35d5ab2dc7e530a0f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 1 Jul 2022 10:54:07 +0100
Subject: [PATCH 187/391] Roomserver producers package (#2546)

* Give the roomserver a producers package

* Change init point

* Populate ACLs API

* Fix build issues

* `RoomEventProducer` naming
---
 roomserver/internal/alias.go                  |  3 +-
 roomserver/internal/api.go                    | 46 +++++-----
 roomserver/internal/input/input.go            | 87 +++---------------
 roomserver/internal/input/input_events.go     |  4 +-
 .../internal/input/input_latest_events.go     |  2 +-
 roomserver/internal/perform/perform_admin.go  |  2 +-
 .../internal/perform/perform_inbound_peek.go  |  2 +-
 roomserver/internal/perform/perform_peek.go   |  2 +-
 roomserver/internal/perform/perform_unpeek.go |  2 +-
 roomserver/producers/roomevent.go             | 89 +++++++++++++++++++
 roomserver/roomserver.go                      |  1 -
 11 files changed, 137 insertions(+), 103 deletions(-)
 create mode 100644 roomserver/producers/roomevent.go

diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go
index f47ae47fe..175bb9310 100644
--- a/roomserver/internal/alias.go
+++ b/roomserver/internal/alias.go
@@ -216,11 +216,10 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias(
 				return err
 			}
 
-			err = api.SendEvents(ctx, r.RSAPI, api.KindNew, []*gomatrixserverlib.HeaderedEvent{newEvent}, r.ServerName, r.ServerName, nil, false)
+			err = api.SendEvents(ctx, r, api.KindNew, []*gomatrixserverlib.HeaderedEvent{newEvent}, r.ServerName, r.ServerName, nil, false)
 			if err != nil {
 				return err
 			}
-
 		}
 	}
 
diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index acdaeef6f..d59b8be7a 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -12,8 +12,10 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/input"
 	"github.com/matrix-org/dendrite/roomserver/internal/perform"
 	"github.com/matrix-org/dendrite/roomserver/internal/query"
+	"github.com/matrix-org/dendrite/roomserver/producers"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -49,17 +51,21 @@ type RoomserverInternalAPI struct {
 	JetStream              nats.JetStreamContext
 	Durable                string
 	InputRoomEventTopic    string // JetStream topic for new input room events
-	OutputRoomEventTopic   string // JetStream topic for new output room events
+	OutputProducer         *producers.RoomEventProducer
 	PerspectiveServerNames []gomatrixserverlib.ServerName
 }
 
 func NewRoomserverAPI(
 	processCtx *process.ProcessContext, cfg *config.RoomServer, roomserverDB storage.Database,
-	consumer nats.JetStreamContext, nc *nats.Conn,
-	inputRoomEventTopic, outputRoomEventTopic string,
+	js nats.JetStreamContext, nc *nats.Conn, inputRoomEventTopic string,
 	caches caching.RoomServerCaches, perspectiveServerNames []gomatrixserverlib.ServerName,
 ) *RoomserverInternalAPI {
 	serverACLs := acls.NewServerACLs(roomserverDB)
+	producer := &producers.RoomEventProducer{
+		Topic:     string(cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent)),
+		JetStream: js,
+		ACLs:      serverACLs,
+	}
 	a := &RoomserverInternalAPI{
 		ProcessContext:         processCtx,
 		DB:                     roomserverDB,
@@ -68,8 +74,8 @@ func NewRoomserverAPI(
 		ServerName:             cfg.Matrix.ServerName,
 		PerspectiveServerNames: perspectiveServerNames,
 		InputRoomEventTopic:    inputRoomEventTopic,
-		OutputRoomEventTopic:   outputRoomEventTopic,
-		JetStream:              consumer,
+		OutputProducer:         producer,
+		JetStream:              js,
 		NATSClient:             nc,
 		Durable:                cfg.Matrix.JetStream.Durable("RoomserverInputConsumer"),
 		ServerACLs:             serverACLs,
@@ -92,19 +98,19 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio
 	r.KeyRing = keyRing
 
 	r.Inputer = &input.Inputer{
-		Cfg:                  r.Cfg,
-		ProcessContext:       r.ProcessContext,
-		DB:                   r.DB,
-		InputRoomEventTopic:  r.InputRoomEventTopic,
-		OutputRoomEventTopic: r.OutputRoomEventTopic,
-		JetStream:            r.JetStream,
-		NATSClient:           r.NATSClient,
-		Durable:              nats.Durable(r.Durable),
-		ServerName:           r.Cfg.Matrix.ServerName,
-		FSAPI:                fsAPI,
-		KeyRing:              keyRing,
-		ACLs:                 r.ServerACLs,
-		Queryer:              r.Queryer,
+		Cfg:                 r.Cfg,
+		ProcessContext:      r.ProcessContext,
+		DB:                  r.DB,
+		InputRoomEventTopic: r.InputRoomEventTopic,
+		OutputProducer:      r.OutputProducer,
+		JetStream:           r.JetStream,
+		NATSClient:          r.NATSClient,
+		Durable:             nats.Durable(r.Durable),
+		ServerName:          r.Cfg.Matrix.ServerName,
+		FSAPI:               fsAPI,
+		KeyRing:             keyRing,
+		ACLs:                r.ServerACLs,
+		Queryer:             r.Queryer,
 	}
 	r.Inviter = &perform.Inviter{
 		DB:      r.DB,
@@ -199,7 +205,7 @@ func (r *RoomserverInternalAPI) PerformInvite(
 	if len(outputEvents) == 0 {
 		return nil
 	}
-	return r.WriteOutputEvents(req.Event.RoomID(), outputEvents)
+	return r.OutputProducer.ProduceRoomEvents(req.Event.RoomID(), outputEvents)
 }
 
 func (r *RoomserverInternalAPI) PerformLeave(
@@ -215,7 +221,7 @@ func (r *RoomserverInternalAPI) PerformLeave(
 	if len(outputEvents) == 0 {
 		return nil
 	}
-	return r.WriteOutputEvents(req.RoomID, outputEvents)
+	return r.OutputProducer.ProduceRoomEvents(req.RoomID, outputEvents)
 }
 
 func (r *RoomserverInternalAPI) PerformForget(
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 600994c5a..fa07c1d2b 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -29,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/acls"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal/query"
+	"github.com/matrix-org/dendrite/roomserver/producers"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -37,16 +38,8 @@ import (
 	"github.com/nats-io/nats.go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
-	log "github.com/sirupsen/logrus"
-	"github.com/tidwall/gjson"
 )
 
-var keyContentFields = map[string]string{
-	"m.room.join_rules":         "join_rule",
-	"m.room.history_visibility": "history_visibility",
-	"m.room.member":             "membership",
-}
-
 // Inputer is responsible for consuming from the roomserver input
 // streams and processing the events. All input events are queued
 // into a single NATS stream and the order is preserved strictly.
@@ -75,19 +68,19 @@ var keyContentFields = map[string]string{
 // up, so they will do nothing until a new event comes in for B
 // or C.
 type Inputer struct {
-	Cfg                  *config.RoomServer
-	ProcessContext       *process.ProcessContext
-	DB                   storage.Database
-	NATSClient           *nats.Conn
-	JetStream            nats.JetStreamContext
-	Durable              nats.SubOpt
-	ServerName           gomatrixserverlib.ServerName
-	FSAPI                fedapi.RoomserverFederationAPI
-	KeyRing              gomatrixserverlib.JSONVerifier
-	ACLs                 *acls.ServerACLs
-	InputRoomEventTopic  string
-	OutputRoomEventTopic string
-	workers              sync.Map // room ID -> *worker
+	Cfg                 *config.RoomServer
+	ProcessContext      *process.ProcessContext
+	DB                  storage.Database
+	NATSClient          *nats.Conn
+	JetStream           nats.JetStreamContext
+	Durable             nats.SubOpt
+	ServerName          gomatrixserverlib.ServerName
+	FSAPI               fedapi.RoomserverFederationAPI
+	KeyRing             gomatrixserverlib.JSONVerifier
+	ACLs                *acls.ServerACLs
+	InputRoomEventTopic string
+	OutputProducer      *producers.RoomEventProducer
+	workers             sync.Map // room ID -> *worker
 
 	Queryer *query.Queryer
 }
@@ -370,58 +363,6 @@ func (r *Inputer) InputRoomEvents(
 	}
 }
 
-// WriteOutputEvents implements OutputRoomEventWriter
-func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) error {
-	var err error
-	for _, update := range updates {
-		msg := &nats.Msg{
-			Subject: r.OutputRoomEventTopic,
-			Header:  nats.Header{},
-		}
-		msg.Header.Set(jetstream.RoomID, roomID)
-		msg.Data, err = json.Marshal(update)
-		if err != nil {
-			return err
-		}
-		logger := log.WithFields(log.Fields{
-			"room_id": roomID,
-			"type":    update.Type,
-		})
-		if update.NewRoomEvent != nil {
-			eventType := update.NewRoomEvent.Event.Type()
-			logger = logger.WithFields(log.Fields{
-				"event_type":     eventType,
-				"event_id":       update.NewRoomEvent.Event.EventID(),
-				"adds_state":     len(update.NewRoomEvent.AddsStateEventIDs),
-				"removes_state":  len(update.NewRoomEvent.RemovesStateEventIDs),
-				"send_as_server": update.NewRoomEvent.SendAsServer,
-				"sender":         update.NewRoomEvent.Event.Sender(),
-			})
-			if update.NewRoomEvent.Event.StateKey() != nil {
-				logger = logger.WithField("state_key", *update.NewRoomEvent.Event.StateKey())
-			}
-			contentKey := keyContentFields[eventType]
-			if contentKey != "" {
-				value := gjson.GetBytes(update.NewRoomEvent.Event.Content(), contentKey)
-				if value.Exists() {
-					logger = logger.WithField("content_value", value.String())
-				}
-			}
-
-			if eventType == "m.room.server_acl" && update.NewRoomEvent.Event.StateKeyEquals("") {
-				ev := update.NewRoomEvent.Event.Unwrap()
-				defer r.ACLs.OnServerACLUpdate(ev)
-			}
-		}
-		logger.Tracef("Producing to topic '%s'", r.OutputRoomEventTopic)
-		if _, err := r.JetStream.PublishMsg(msg); err != nil {
-			logger.WithError(err).Errorf("Failed to produce to topic '%s': %s", r.OutputRoomEventTopic, err)
-			return err
-		}
-	}
-	return nil
-}
-
 var roomserverInputBackpressure = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Namespace: "dendrite",
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index ff05f798c..743b1efe6 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -381,7 +381,7 @@ func (r *Inputer) processRoomEvent(
 			return fmt.Errorf("r.updateLatestEvents: %w", err)
 		}
 	case api.KindOld:
-		err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{
+		err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{
 			{
 				Type: api.OutputTypeOldRoomEvent,
 				OldRoomEvent: &api.OutputOldRoomEvent{
@@ -400,7 +400,7 @@ func (r *Inputer) processRoomEvent(
 	// so notify downstream components to redact this event - they should have it if they've
 	// been tracking our output log.
 	if redactedEventID != "" {
-		err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{
+		err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{
 			{
 				Type: api.OutputTypeRedactedEvent,
 				RedactedEvent: &api.OutputRedactedEvent{
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index e76f4ba8d..f7d15fdb5 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -192,7 +192,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	// send the event asynchronously but we would need to ensure that 1) the events are written to the log in
 	// the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the
 	// necessary bookkeeping we'll keep the event sending synchronous for now.
-	if err = u.api.WriteOutputEvents(u.event.RoomID(), updates); err != nil {
+	if err = u.api.OutputProducer.ProduceRoomEvents(u.event.RoomID(), updates); err != nil {
 		return fmt.Errorf("u.api.WriteOutputEvents: %w", err)
 	}
 
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index d3fb71099..1cb52966a 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -219,7 +219,7 @@ func (r *Admin) PerformAdminEvacuateUser(
 		if len(outputEvents) == 0 {
 			continue
 		}
-		if err := r.Inputer.WriteOutputEvents(roomID, outputEvents); err != nil {
+		if err := r.Inputer.OutputProducer.ProduceRoomEvents(roomID, outputEvents); err != nil {
 			res.Error = &api.PerformError{
 				Code: api.PerformErrorBadRequest,
 				Msg:  fmt.Sprintf("r.Inputer.WriteOutputEvents: %s", err),
diff --git a/roomserver/internal/perform/perform_inbound_peek.go b/roomserver/internal/perform/perform_inbound_peek.go
index d19fc8386..32c81e849 100644
--- a/roomserver/internal/perform/perform_inbound_peek.go
+++ b/roomserver/internal/perform/perform_inbound_peek.go
@@ -113,7 +113,7 @@ func (r *InboundPeeker) PerformInboundPeek(
 		response.AuthChainEvents = append(response.AuthChainEvents, event.Headered(info.RoomVersion))
 	}
 
-	err = r.Inputer.WriteOutputEvents(request.RoomID, []api.OutputEvent{
+	err = r.Inputer.OutputProducer.ProduceRoomEvents(request.RoomID, []api.OutputEvent{
 		{
 			Type: api.OutputTypeNewInboundPeek,
 			NewInboundPeek: &api.OutputNewInboundPeek{
diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go
index 45e63888d..5560916b2 100644
--- a/roomserver/internal/perform/perform_peek.go
+++ b/roomserver/internal/perform/perform_peek.go
@@ -207,7 +207,7 @@ func (r *Peeker) performPeekRoomByID(
 
 	// TODO: handle federated peeks
 
-	err = r.Inputer.WriteOutputEvents(roomID, []api.OutputEvent{
+	err = r.Inputer.OutputProducer.ProduceRoomEvents(roomID, []api.OutputEvent{
 		{
 			Type: api.OutputTypeNewPeek,
 			NewPeek: &api.OutputNewPeek{
diff --git a/roomserver/internal/perform/perform_unpeek.go b/roomserver/internal/perform/perform_unpeek.go
index 1057499cb..1fe8d5a0f 100644
--- a/roomserver/internal/perform/perform_unpeek.go
+++ b/roomserver/internal/perform/perform_unpeek.go
@@ -96,7 +96,7 @@ func (r *Unpeeker) performUnpeekRoomByID(
 
 	// TODO: handle federated peeks
 
-	err = r.Inputer.WriteOutputEvents(req.RoomID, []api.OutputEvent{
+	err = r.Inputer.OutputProducer.ProduceRoomEvents(req.RoomID, []api.OutputEvent{
 		{
 			Type: api.OutputTypeRetirePeek,
 			RetirePeek: &api.OutputRetirePeek{
diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go
new file mode 100644
index 000000000..987e6c942
--- /dev/null
+++ b/roomserver/producers/roomevent.go
@@ -0,0 +1,89 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/roomserver/acls"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
+)
+
+var keyContentFields = map[string]string{
+	"m.room.join_rules":         "join_rule",
+	"m.room.history_visibility": "history_visibility",
+	"m.room.member":             "membership",
+}
+
+type RoomEventProducer struct {
+	Topic     string
+	ACLs      *acls.ServerACLs
+	JetStream nats.JetStreamContext
+}
+
+func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.OutputEvent) error {
+	var err error
+	for _, update := range updates {
+		msg := &nats.Msg{
+			Subject: r.Topic,
+			Header:  nats.Header{},
+		}
+		msg.Header.Set(jetstream.RoomID, roomID)
+		msg.Data, err = json.Marshal(update)
+		if err != nil {
+			return err
+		}
+		logger := log.WithFields(log.Fields{
+			"room_id": roomID,
+			"type":    update.Type,
+		})
+		if update.NewRoomEvent != nil {
+			eventType := update.NewRoomEvent.Event.Type()
+			logger = logger.WithFields(log.Fields{
+				"event_type":     eventType,
+				"event_id":       update.NewRoomEvent.Event.EventID(),
+				"adds_state":     len(update.NewRoomEvent.AddsStateEventIDs),
+				"removes_state":  len(update.NewRoomEvent.RemovesStateEventIDs),
+				"send_as_server": update.NewRoomEvent.SendAsServer,
+				"sender":         update.NewRoomEvent.Event.Sender(),
+			})
+			if update.NewRoomEvent.Event.StateKey() != nil {
+				logger = logger.WithField("state_key", *update.NewRoomEvent.Event.StateKey())
+			}
+			contentKey := keyContentFields[eventType]
+			if contentKey != "" {
+				value := gjson.GetBytes(update.NewRoomEvent.Event.Content(), contentKey)
+				if value.Exists() {
+					logger = logger.WithField("content_value", value.String())
+				}
+			}
+
+			if eventType == "m.room.server_acl" && update.NewRoomEvent.Event.StateKeyEquals("") {
+				ev := update.NewRoomEvent.Event.Unwrap()
+				defer r.ACLs.OnServerACLUpdate(ev)
+			}
+		}
+		logger.Tracef("Producing to topic '%s'", r.Topic)
+		if _, err := r.JetStream.PublishMsg(msg); err != nil {
+			logger.WithError(err).Errorf("Failed to produce to topic '%s': %s", r.Topic, err)
+			return err
+		}
+	}
+	return nil
+}
diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go
index 1480e8942..eb68100fe 100644
--- a/roomserver/roomserver.go
+++ b/roomserver/roomserver.go
@@ -55,7 +55,6 @@ func NewInternalAPI(
 	return internal.NewRoomserverAPI(
 		base.ProcessContext, cfg, roomserverDB, js, nc,
 		cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent),
-		cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
 		base.Caches, perspectiveServerNames,
 	)
 }

From b5c55faf9886bd66a33e5555ad0bb20465bf08f7 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 1 Jul 2022 12:00:32 +0100
Subject: [PATCH 188/391] Version 0.8.9 (#2549)

* Version 0.8.9

* Update changelog
---
 CHANGES.md          | 17 +++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 0db25f05a..3df03b2f6 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,22 @@
 # Changelog
 
+## Dendrite 0.8.9 (2022-07-01)
+
+### Features
+
+* Incoming device list updates over federation are now queued in JetStream for processing so that they will no longer block incoming federation transactions and should never end up dropped, which will hopefully help E2EE reliability
+* The `/context` endpoint now returns `"start"` and `"end"` parameters to allow pagination from a context call
+* The `/messages` endpoint will no longer return `"end"` when there are no more messages remaining
+* Deactivated user accounts will now leave all rooms automatically
+* New admin endpoint `/_dendrite/admin/evacuateUser/{userID}` has been added for forcing a local user to leave all joined rooms
+* Dendrite will now automatically attempt to raise the file descriptor limit at startup if it is too low
+
+### Fixes
+
+* A rare crash when retrieving remote device lists has been fixed
+* Fixes a bug where events were not redacted properly over federation
+* The `/invite` endpoints will now return an error instead of silently proceeding if the user ID is obviously malformed
+
 ## Dendrite 0.8.8 (2022-06-09)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index e29996f36..9568f08cb 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 8
+	VersionPatch = 9
 	VersionTag   = "" // example: "rc1"
 )
 

From 43147bd65415de2477826677a08f6655ece7f38c Mon Sep 17 00:00:00 2001
From: Kabir Kwatra <kabir@kwatra.me>
Date: Tue, 5 Jul 2022 14:53:51 +0300
Subject: [PATCH 189/391] feat+fix: Ignore unknown keys and verify required
 fields are present in appservice registration files (#2550)

* fix: ignore unknown keys in appservice configs

fixes matrix-org/dendrite#1567

* feat: verify required fields in appservice configs
---
 setup/config/config_appservice.go | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go
index ff3287714..9b89fc9af 100644
--- a/setup/config/config_appservice.go
+++ b/setup/config/config_appservice.go
@@ -187,7 +187,7 @@ func loadAppServices(config *AppServiceAPI, derived *Derived) error {
 		}
 
 		// Load the config data into our struct
-		if err = yaml.UnmarshalStrict(configData, &appservice); err != nil {
+		if err = yaml.Unmarshal(configData, &appservice); err != nil {
 			return err
 		}
 
@@ -315,6 +315,20 @@ func checkErrors(config *AppServiceAPI, derived *Derived) (err error) {
 			}
 		}
 
+		// Check required fields
+		if appservice.ID == "" {
+			return ConfigErrors([]string{"Application service ID is required"})
+		}
+		if appservice.ASToken == "" {
+			return ConfigErrors([]string{"Application service Token is required"})
+		}
+		if appservice.HSToken == "" {
+			return ConfigErrors([]string{"Homeserver Token is required"})
+		}
+		if appservice.SenderLocalpart == "" {
+			return ConfigErrors([]string{"Sender Localpart is required"})
+		}
+
 		// Check if the url has trailing /'s. If so, remove them
 		appservice.URL = strings.TrimRight(appservice.URL, "/")
 

From f29cdb26f6ca14b0f533ecdabda81aa7d9439db2 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 5 Jul 2022 14:50:24 +0200
Subject: [PATCH 190/391] Use new testrig for key changes tests (#2552)

* Use new testrig for tests

* Log the error message
---
 keyserver/storage/storage_test.go | 298 +++++++++++++++---------------
 1 file changed, 148 insertions(+), 150 deletions(-)

diff --git a/keyserver/storage/storage_test.go b/keyserver/storage/storage_test.go
index 9940eac60..44cfb5f2a 100644
--- a/keyserver/storage/storage_test.go
+++ b/keyserver/storage/storage_test.go
@@ -1,36 +1,26 @@
-package storage
+package storage_test
 
 import (
 	"context"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
 	"reflect"
 	"testing"
 
 	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/keyserver/storage"
 	"github.com/matrix-org/dendrite/keyserver/types"
-	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 )
 
 var ctx = context.Background()
 
-func MustCreateDatabase(t *testing.T) (Database, func()) {
-	tmpfile, err := ioutil.TempFile("", "keyserver_storage_test")
+func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	base, close := testrig.CreateBaseDendrite(t, dbType)
+	db, err := storage.NewDatabase(base, &base.Cfg.KeyServer.Database)
 	if err != nil {
-		log.Fatal(err)
-	}
-	t.Logf("Database %s", tmpfile.Name())
-	db, err := NewDatabase(nil, &config.DatabaseOptions{
-		ConnectionString: config.DataSource(fmt.Sprintf("file://%s", tmpfile.Name())),
-	})
-	if err != nil {
-		t.Fatalf("Failed to NewDatabase: %s", err)
-	}
-	return db, func() {
-		os.Remove(tmpfile.Name())
+		t.Fatalf("failed to create new database: %v", err)
 	}
+	return db, close
 }
 
 func MustNotError(t *testing.T, err error) {
@@ -42,151 +32,159 @@ func MustNotError(t *testing.T, err error) {
 }
 
 func TestKeyChanges(t *testing.T) {
-	db, clean := MustCreateDatabase(t)
-	defer clean()
-	_, err := db.StoreKeyChange(ctx, "@alice:localhost")
-	MustNotError(t, err)
-	deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost")
-	MustNotError(t, err)
-	deviceChangeIDC, err := db.StoreKeyChange(ctx, "@charlie:localhost")
-	MustNotError(t, err)
-	userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDB, types.OffsetNewest)
-	if err != nil {
-		t.Fatalf("Failed to KeyChanges: %s", err)
-	}
-	if latest != deviceChangeIDC {
-		t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDC)
-	}
-	if !reflect.DeepEqual(userIDs, []string{"@charlie:localhost"}) {
-		t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs)
-	}
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, clean := MustCreateDatabase(t, dbType)
+		defer clean()
+		_, err := db.StoreKeyChange(ctx, "@alice:localhost")
+		MustNotError(t, err)
+		deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost")
+		MustNotError(t, err)
+		deviceChangeIDC, err := db.StoreKeyChange(ctx, "@charlie:localhost")
+		MustNotError(t, err)
+		userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDB, types.OffsetNewest)
+		if err != nil {
+			t.Fatalf("Failed to KeyChanges: %s", err)
+		}
+		if latest != deviceChangeIDC {
+			t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDC)
+		}
+		if !reflect.DeepEqual(userIDs, []string{"@charlie:localhost"}) {
+			t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs)
+		}
+	})
 }
 
 func TestKeyChangesNoDupes(t *testing.T) {
-	db, clean := MustCreateDatabase(t)
-	defer clean()
-	deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost")
-	MustNotError(t, err)
-	deviceChangeIDB, err := db.StoreKeyChange(ctx, "@alice:localhost")
-	MustNotError(t, err)
-	if deviceChangeIDA == deviceChangeIDB {
-		t.Fatalf("Expected change ID to be different even when inserting key change for the same user, got %d for both changes", deviceChangeIDA)
-	}
-	deviceChangeID, err := db.StoreKeyChange(ctx, "@alice:localhost")
-	MustNotError(t, err)
-	userIDs, latest, err := db.KeyChanges(ctx, 0, types.OffsetNewest)
-	if err != nil {
-		t.Fatalf("Failed to KeyChanges: %s", err)
-	}
-	if latest != deviceChangeID {
-		t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeID)
-	}
-	if !reflect.DeepEqual(userIDs, []string{"@alice:localhost"}) {
-		t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs)
-	}
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, clean := MustCreateDatabase(t, dbType)
+		defer clean()
+		deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost")
+		MustNotError(t, err)
+		deviceChangeIDB, err := db.StoreKeyChange(ctx, "@alice:localhost")
+		MustNotError(t, err)
+		if deviceChangeIDA == deviceChangeIDB {
+			t.Fatalf("Expected change ID to be different even when inserting key change for the same user, got %d for both changes", deviceChangeIDA)
+		}
+		deviceChangeID, err := db.StoreKeyChange(ctx, "@alice:localhost")
+		MustNotError(t, err)
+		userIDs, latest, err := db.KeyChanges(ctx, 0, types.OffsetNewest)
+		if err != nil {
+			t.Fatalf("Failed to KeyChanges: %s", err)
+		}
+		if latest != deviceChangeID {
+			t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeID)
+		}
+		if !reflect.DeepEqual(userIDs, []string{"@alice:localhost"}) {
+			t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs)
+		}
+	})
 }
 
 func TestKeyChangesUpperLimit(t *testing.T) {
-	db, clean := MustCreateDatabase(t)
-	defer clean()
-	deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost")
-	MustNotError(t, err)
-	deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost")
-	MustNotError(t, err)
-	_, err = db.StoreKeyChange(ctx, "@charlie:localhost")
-	MustNotError(t, err)
-	userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDA, deviceChangeIDB)
-	if err != nil {
-		t.Fatalf("Failed to KeyChanges: %s", err)
-	}
-	if latest != deviceChangeIDB {
-		t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDB)
-	}
-	if !reflect.DeepEqual(userIDs, []string{"@bob:localhost"}) {
-		t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs)
-	}
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, clean := MustCreateDatabase(t, dbType)
+		defer clean()
+		deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost")
+		MustNotError(t, err)
+		deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost")
+		MustNotError(t, err)
+		_, err = db.StoreKeyChange(ctx, "@charlie:localhost")
+		MustNotError(t, err)
+		userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDA, deviceChangeIDB)
+		if err != nil {
+			t.Fatalf("Failed to KeyChanges: %s", err)
+		}
+		if latest != deviceChangeIDB {
+			t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDB)
+		}
+		if !reflect.DeepEqual(userIDs, []string{"@bob:localhost"}) {
+			t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs)
+		}
+	})
 }
 
 // The purpose of this test is to make sure that the storage layer is generating sequential stream IDs per user,
 // and that they are returned correctly when querying for device keys.
 func TestDeviceKeysStreamIDGeneration(t *testing.T) {
 	var err error
-	db, clean := MustCreateDatabase(t)
-	defer clean()
-	alice := "@alice:TestDeviceKeysStreamIDGeneration"
-	bob := "@bob:TestDeviceKeysStreamIDGeneration"
-	msgs := []api.DeviceMessage{
-		{
-			Type: api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{
-				DeviceID: "AAA",
-				UserID:   alice,
-				KeyJSON:  []byte(`{"key":"v1"}`),
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, clean := MustCreateDatabase(t, dbType)
+		defer clean()
+		alice := "@alice:TestDeviceKeysStreamIDGeneration"
+		bob := "@bob:TestDeviceKeysStreamIDGeneration"
+		msgs := []api.DeviceMessage{
+			{
+				Type: api.TypeDeviceKeyUpdate,
+				DeviceKeys: &api.DeviceKeys{
+					DeviceID: "AAA",
+					UserID:   alice,
+					KeyJSON:  []byte(`{"key":"v1"}`),
+				},
+				// StreamID: 1
 			},
-			// StreamID: 1
-		},
-		{
-			Type: api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{
-				DeviceID: "AAA",
-				UserID:   bob,
-				KeyJSON:  []byte(`{"key":"v1"}`),
+			{
+				Type: api.TypeDeviceKeyUpdate,
+				DeviceKeys: &api.DeviceKeys{
+					DeviceID: "AAA",
+					UserID:   bob,
+					KeyJSON:  []byte(`{"key":"v1"}`),
+				},
+				// StreamID: 1 as this is a different user
 			},
-			// StreamID: 1 as this is a different user
-		},
-		{
-			Type: api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{
-				DeviceID: "another_device",
-				UserID:   alice,
-				KeyJSON:  []byte(`{"key":"v1"}`),
+			{
+				Type: api.TypeDeviceKeyUpdate,
+				DeviceKeys: &api.DeviceKeys{
+					DeviceID: "another_device",
+					UserID:   alice,
+					KeyJSON:  []byte(`{"key":"v1"}`),
+				},
+				// StreamID: 2 as this is a 2nd device key
 			},
-			// StreamID: 2 as this is a 2nd device key
-		},
-	}
-	MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs))
-	if msgs[0].StreamID != 1 {
-		t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 but got %d", msgs[0].StreamID)
-	}
-	if msgs[1].StreamID != 1 {
-		t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 (different user) but got %d", msgs[1].StreamID)
-	}
-	if msgs[2].StreamID != 2 {
-		t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=2 (another device) but got %d", msgs[2].StreamID)
-	}
-
-	// updating a device sets the next stream ID for that user
-	msgs = []api.DeviceMessage{
-		{
-			Type: api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{
-				DeviceID: "AAA",
-				UserID:   alice,
-				KeyJSON:  []byte(`{"key":"v2"}`),
-			},
-			// StreamID: 3
-		},
-	}
-	MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs))
-	if msgs[0].StreamID != 3 {
-		t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=3 (new key same device) but got %d", msgs[0].StreamID)
-	}
-
-	// Querying for device keys returns the latest stream IDs
-	msgs, err = db.DeviceKeysForUser(ctx, alice, []string{"AAA", "another_device"}, false)
-	if err != nil {
-		t.Fatalf("DeviceKeysForUser returned error: %s", err)
-	}
-	wantStreamIDs := map[string]int64{
-		"AAA":            3,
-		"another_device": 2,
-	}
-	if len(msgs) != len(wantStreamIDs) {
-		t.Fatalf("DeviceKeysForUser: wrong number of devices, got %d want %d", len(msgs), len(wantStreamIDs))
-	}
-	for _, m := range msgs {
-		if m.StreamID != wantStreamIDs[m.DeviceID] {
-			t.Errorf("DeviceKeysForUser: wrong returned stream ID for key, got %d want %d", m.StreamID, wantStreamIDs[m.DeviceID])
 		}
-	}
+		MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs))
+		if msgs[0].StreamID != 1 {
+			t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 but got %d", msgs[0].StreamID)
+		}
+		if msgs[1].StreamID != 1 {
+			t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 (different user) but got %d", msgs[1].StreamID)
+		}
+		if msgs[2].StreamID != 2 {
+			t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=2 (another device) but got %d", msgs[2].StreamID)
+		}
+
+		// updating a device sets the next stream ID for that user
+		msgs = []api.DeviceMessage{
+			{
+				Type: api.TypeDeviceKeyUpdate,
+				DeviceKeys: &api.DeviceKeys{
+					DeviceID: "AAA",
+					UserID:   alice,
+					KeyJSON:  []byte(`{"key":"v2"}`),
+				},
+				// StreamID: 3
+			},
+		}
+		MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs))
+		if msgs[0].StreamID != 3 {
+			t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=3 (new key same device) but got %d", msgs[0].StreamID)
+		}
+
+		// Querying for device keys returns the latest stream IDs
+		msgs, err = db.DeviceKeysForUser(ctx, alice, []string{"AAA", "another_device"}, false)
+		if err != nil {
+			t.Fatalf("DeviceKeysForUser returned error: %s", err)
+		}
+		wantStreamIDs := map[string]int64{
+			"AAA":            3,
+			"another_device": 2,
+		}
+		if len(msgs) != len(wantStreamIDs) {
+			t.Fatalf("DeviceKeysForUser: wrong number of devices, got %d want %d", len(msgs), len(wantStreamIDs))
+		}
+		for _, m := range msgs {
+			if m.StreamID != wantStreamIDs[m.DeviceID] {
+				t.Errorf("DeviceKeysForUser: wrong returned stream ID for key, got %d want %d", m.StreamID, wantStreamIDs[m.DeviceID])
+			}
+		}
+	})
 }

From 5087b36af035bcf82a8655e35a2c661d7be72048 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 5 Jul 2022 14:50:56 +0200
Subject: [PATCH 191/391] Fix QuerySharedUsers for the SyncAPI keychange
 consumer (#2554)

* Make more use of base.BaseDendrite

* Fix QuerySharedUsers if no UserIDs are supplied
---
 roomserver/internal/api.go                    | 36 ++++++----
 roomserver/internal/input/input.go            |  6 +-
 roomserver/roomserver.go                      | 14 +---
 roomserver/roomserver_test.go                 | 69 +++++++++++++++++++
 .../storage/postgres/membership_table.go      | 22 +++++-
 roomserver/storage/shared/storage.go          |  7 ++
 .../storage/sqlite3/membership_table.go       | 14 +++-
 7 files changed, 137 insertions(+), 31 deletions(-)
 create mode 100644 roomserver/roomserver_test.go

diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go
index d59b8be7a..1a11586a5 100644
--- a/roomserver/internal/api.go
+++ b/roomserver/internal/api.go
@@ -14,6 +14,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/query"
 	"github.com/matrix-org/dendrite/roomserver/producers"
 	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -39,6 +40,7 @@ type RoomserverInternalAPI struct {
 	*perform.Upgrader
 	*perform.Admin
 	ProcessContext         *process.ProcessContext
+	Base                   *base.BaseDendrite
 	DB                     storage.Database
 	Cfg                    *config.RoomServer
 	Cache                  caching.RoomServerCaches
@@ -56,33 +58,38 @@ type RoomserverInternalAPI struct {
 }
 
 func NewRoomserverAPI(
-	processCtx *process.ProcessContext, cfg *config.RoomServer, roomserverDB storage.Database,
-	js nats.JetStreamContext, nc *nats.Conn, inputRoomEventTopic string,
-	caches caching.RoomServerCaches, perspectiveServerNames []gomatrixserverlib.ServerName,
+	base *base.BaseDendrite, roomserverDB storage.Database,
+	js nats.JetStreamContext, nc *nats.Conn,
 ) *RoomserverInternalAPI {
+	var perspectiveServerNames []gomatrixserverlib.ServerName
+	for _, kp := range base.Cfg.FederationAPI.KeyPerspectives {
+		perspectiveServerNames = append(perspectiveServerNames, kp.ServerName)
+	}
+
 	serverACLs := acls.NewServerACLs(roomserverDB)
 	producer := &producers.RoomEventProducer{
-		Topic:     string(cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent)),
+		Topic:     string(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)),
 		JetStream: js,
 		ACLs:      serverACLs,
 	}
 	a := &RoomserverInternalAPI{
-		ProcessContext:         processCtx,
+		ProcessContext:         base.ProcessContext,
 		DB:                     roomserverDB,
-		Cfg:                    cfg,
-		Cache:                  caches,
-		ServerName:             cfg.Matrix.ServerName,
+		Base:                   base,
+		Cfg:                    &base.Cfg.RoomServer,
+		Cache:                  base.Caches,
+		ServerName:             base.Cfg.Global.ServerName,
 		PerspectiveServerNames: perspectiveServerNames,
-		InputRoomEventTopic:    inputRoomEventTopic,
+		InputRoomEventTopic:    base.Cfg.Global.JetStream.Prefixed(jetstream.InputRoomEvent),
 		OutputProducer:         producer,
 		JetStream:              js,
 		NATSClient:             nc,
-		Durable:                cfg.Matrix.JetStream.Durable("RoomserverInputConsumer"),
+		Durable:                base.Cfg.Global.JetStream.Durable("RoomserverInputConsumer"),
 		ServerACLs:             serverACLs,
 		Queryer: &query.Queryer{
 			DB:         roomserverDB,
-			Cache:      caches,
-			ServerName: cfg.Matrix.ServerName,
+			Cache:      base.Caches,
+			ServerName: base.Cfg.Global.ServerName,
 			ServerACLs: serverACLs,
 		},
 		// perform-er structs get initialised when we have a federation sender to use
@@ -98,8 +105,9 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio
 	r.KeyRing = keyRing
 
 	r.Inputer = &input.Inputer{
-		Cfg:                 r.Cfg,
-		ProcessContext:      r.ProcessContext,
+		Cfg:                 &r.Base.Cfg.RoomServer,
+		Base:                r.Base,
+		ProcessContext:      r.Base.ProcessContext,
 		DB:                  r.DB,
 		InputRoomEventTopic: r.InputRoomEventTopic,
 		OutputProducer:      r.OutputProducer,
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index fa07c1d2b..ecd4ecbb5 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -31,6 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/query"
 	"github.com/matrix-org/dendrite/roomserver/producers"
 	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -69,6 +70,7 @@ import (
 // or C.
 type Inputer struct {
 	Cfg                 *config.RoomServer
+	Base                *base.BaseDendrite
 	ProcessContext      *process.ProcessContext
 	DB                  storage.Database
 	NATSClient          *nats.Conn
@@ -160,7 +162,9 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
 // will look to see if we have a worker for that room which has its
 // own consumer. If we don't, we'll start one.
 func (r *Inputer) Start() error {
-	prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration)
+	if r.Base.EnableMetrics {
+		prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration)
+	}
 	_, err := r.JetStream.Subscribe(
 		"", // This is blank because we specified it in BindStream.
 		func(m *nats.Msg) {
diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go
index eb68100fe..1f707735b 100644
--- a/roomserver/roomserver.go
+++ b/roomserver/roomserver.go
@@ -17,13 +17,10 @@ package roomserver
 import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/inthttp"
-	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/matrix-org/dendrite/roomserver/internal"
+	"github.com/matrix-org/dendrite/roomserver/inthttp"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/sirupsen/logrus"
 )
 
@@ -40,11 +37,6 @@ func NewInternalAPI(
 ) api.RoomserverInternalAPI {
 	cfg := &base.Cfg.RoomServer
 
-	var perspectiveServerNames []gomatrixserverlib.ServerName
-	for _, kp := range base.Cfg.FederationAPI.KeyPerspectives {
-		perspectiveServerNames = append(perspectiveServerNames, kp.ServerName)
-	}
-
 	roomserverDB, err := storage.Open(base, &cfg.Database, base.Caches)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to room server db")
@@ -53,8 +45,6 @@ func NewInternalAPI(
 	js, nc := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 
 	return internal.NewRoomserverAPI(
-		base.ProcessContext, cfg, roomserverDB, js, nc,
-		cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent),
-		base.Caches, perspectiveServerNames,
+		base, roomserverDB, js, nc,
 	)
 }
diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go
new file mode 100644
index 000000000..4e98af853
--- /dev/null
+++ b/roomserver/roomserver_test.go
@@ -0,0 +1,69 @@
+package roomserver_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func mustCreateDatabase(t *testing.T, dbType test.DBType) (*base.BaseDendrite, storage.Database, func()) {
+	base, close := testrig.CreateBaseDendrite(t, dbType)
+	db, err := storage.Open(base, &base.Cfg.KeyServer.Database, base.Caches)
+	if err != nil {
+		t.Fatalf("failed to create Database: %v", err)
+	}
+	return base, db, close
+}
+
+func Test_SharedUsers(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice, test.RoomPreset(test.PresetTrustedPrivateChat))
+
+	// Invite and join Bob
+	room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "invite",
+	}, test.WithStateKey(bob.ID))
+	room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, _, close := mustCreateDatabase(t, dbType)
+		defer close()
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		// SetFederationAPI starts the room event input consumer
+		rsAPI.SetFederationAPI(nil, nil)
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// Query the shared users for Alice, there should only be Bob.
+		// This is used by the SyncAPI keychange consumer.
+		res := &api.QuerySharedUsersResponse{}
+		if err := rsAPI.QuerySharedUsers(ctx, &api.QuerySharedUsersRequest{UserID: alice.ID}, res); err != nil {
+			t.Fatalf("unable to query known users: %v", err)
+		}
+		if _, ok := res.UserIDsToCount[bob.ID]; !ok {
+			t.Fatalf("expected to find %s in shared users, but didn't: %+v", bob.ID, res.UserIDsToCount)
+		}
+		// Also verify that we get the expected result when specifying OtherUserIDs.
+		// This is used by the SyncAPI when getting device list changes.
+		if err := rsAPI.QuerySharedUsers(ctx, &api.QuerySharedUsersRequest{UserID: alice.ID, OtherUserIDs: []string{bob.ID}}, res); err != nil {
+			t.Fatalf("unable to query known users: %v", err)
+		}
+		if _, ok := res.UserIDsToCount[bob.ID]; !ok {
+			t.Fatalf("expected to find %s in shared users, but didn't: %+v", bob.ID, res.UserIDsToCount)
+		}
+	})
+}
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index c01753c3a..ce626ad1d 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -65,12 +65,18 @@ CREATE TABLE IF NOT EXISTS roomserver_membership (
 );
 `
 
-var selectJoinedUsersSetForRoomsSQL = "" +
+var selectJoinedUsersSetForRoomsAndUserSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
 	" WHERE room_nid = ANY($1) AND target_nid = ANY($2) AND" +
 	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
 	" GROUP BY target_nid"
 
+var selectJoinedUsersSetForRoomsSQL = "" +
+	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
+	" WHERE room_nid = ANY($1) AND" +
+	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" GROUP BY target_nid"
+
 // Insert a row in to membership table so that it can be locked by the
 // SELECT FOR UPDATE
 const insertMembershipSQL = "" +
@@ -153,6 +159,7 @@ type membershipStatements struct {
 	selectLocalMembershipsFromRoomStmt              *sql.Stmt
 	updateMembershipStmt                            *sql.Stmt
 	selectRoomsWithMembershipStmt                   *sql.Stmt
+	selectJoinedUsersSetForRoomsAndUserStmt         *sql.Stmt
 	selectJoinedUsersSetForRoomsStmt                *sql.Stmt
 	selectKnownUsersStmt                            *sql.Stmt
 	updateMembershipForgetRoomStmt                  *sql.Stmt
@@ -178,6 +185,7 @@ func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.selectLocalMembershipsFromRoomStmt, selectLocalMembershipsFromRoomSQL},
 		{&s.updateMembershipStmt, updateMembershipSQL},
 		{&s.selectRoomsWithMembershipStmt, selectRoomsWithMembershipSQL},
+		{&s.selectJoinedUsersSetForRoomsAndUserStmt, selectJoinedUsersSetForRoomsAndUserSQL},
 		{&s.selectJoinedUsersSetForRoomsStmt, selectJoinedUsersSetForRoomsSQL},
 		{&s.selectKnownUsersStmt, selectKnownUsersSQL},
 		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
@@ -313,8 +321,18 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(
 	roomNIDs []types.RoomNID,
 	userNIDs []types.EventStateKeyNID,
 ) (map[types.EventStateKeyNID]int, error) {
+	var (
+		rows *sql.Rows
+		err  error
+	)
 	stmt := sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsStmt)
-	rows, err := stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(userNIDs))
+	if len(userNIDs) > 0 {
+		stmt = sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsAndUserStmt)
+		rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(userNIDs))
+	} else {
+		rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs))
+	}
+
 	if err != nil {
 		return nil, err
 	}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 67dcfdf38..5c633122d 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1214,6 +1214,13 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs [
 		stateKeyNIDs[i] = nid
 		i++
 	}
+	// If we didn't have any userIDs to look up, get the UserIDs for the returned userNIDToCount now
+	if len(userIDs) == 0 {
+		nidToUserID, err = d.EventStateKeys(ctx, stateKeyNIDs)
+		if err != nil {
+			return nil, err
+		}
+	}
 	result := make(map[string]int, len(userNIDToCount))
 	for nid, count := range userNIDToCount {
 		result[nidToUserID[nid]] = count
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index 6f0fe8b64..570d3919c 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -41,12 +41,18 @@ const membershipSchema = `
 	);
 `
 
-var selectJoinedUsersSetForRoomsSQL = "" +
+var selectJoinedUsersSetForRoomsAndUserSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
 	" WHERE room_nid IN ($1) AND target_nid IN ($2) AND" +
 	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
 	" GROUP BY target_nid"
 
+var selectJoinedUsersSetForRoomsSQL = "" +
+	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
+	" WHERE room_nid IN ($1) AND " +
+	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" GROUP BY target_nid"
+
 // Insert a row in to membership table so that it can be locked by the
 // SELECT FOR UPDATE
 const insertMembershipSQL = "" +
@@ -293,8 +299,12 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context,
 	for _, v := range userNIDs {
 		params = append(params, v)
 	}
+
 	query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
-	query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)), 1)
+	if len(userNIDs) > 0 {
+		query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
+		query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)), 1)
+	}
 	var rows *sql.Rows
 	var err error
 	if txn != nil {

From d4341a2d978ef9e391dd9f26cd2bf50ad1495fb2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 5 Jul 2022 15:01:34 +0100
Subject: [PATCH 192/391] Return clearer error when no state NID exists for an
 event (#2555)

---
 roomserver/storage/shared/storage.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 5c633122d..3191280cb 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -263,6 +263,12 @@ func (d *Database) snapshotNIDFromEventID(
 	ctx context.Context, txn *sql.Tx, eventID string,
 ) (types.StateSnapshotNID, error) {
 	_, stateNID, err := d.EventsTable.SelectEvent(ctx, txn, eventID)
+	if err != nil {
+		return 0, err
+	}
+	if stateNID == 0 {
+		return 0, sql.ErrNoRows // effectively there's no state entry
+	}
 	return stateNID, err
 }
 

From c0f824d4375493127799eb3c3ecf0327838813d6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 5 Jul 2022 15:06:10 +0100
Subject: [PATCH 193/391] Wrap error from `SnapshotNIDFromEventID`

---
 roomserver/state/state.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index 6c4e4b860..91f271652 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -110,7 +110,7 @@ func (v *StateResolution) LoadStateAtEvent(
 
 	snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID)
 	if err != nil {
-		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err)
+		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %w", eventID, err)
 	}
 	if snapshotNID == 0 {
 		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID)

From 460dccf93d5eb77db00620f0ef5a4f1a91bbe7ae Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 5 Jul 2022 17:13:26 +0100
Subject: [PATCH 194/391] Hopefully fix read receipts timestamps (#2557)

This should avoid coercions between signed and unsigned ints which might fix problems like `sql: converting argument $5 type: uint64 values with high bit set are not supported`.
---
 clientapi/producers/syncapi.go      | 3 ++-
 federationapi/consumers/receipts.go | 2 +-
 federationapi/producers/syncapi.go  | 2 +-
 syncapi/consumers/receipts.go       | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go
index 48b1ae88d..0ac637793 100644
--- a/clientapi/producers/syncapi.go
+++ b/clientapi/producers/syncapi.go
@@ -17,6 +17,7 @@ package producers
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"strconv"
 	"time"
 
@@ -83,7 +84,7 @@ func (p *SyncAPIProducer) SendReceipt(
 	m.Header.Set(jetstream.RoomID, roomID)
 	m.Header.Set(jetstream.EventID, eventID)
 	m.Header.Set("type", receiptType)
-	m.Header.Set("timestamp", strconv.Itoa(int(timestamp)))
+	m.Header.Set("timestamp", fmt.Sprintf("%d", timestamp))
 
 	log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent)
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go
index 9300451eb..2c9d79bcb 100644
--- a/federationapi/consumers/receipts.go
+++ b/federationapi/consumers/receipts.go
@@ -90,7 +90,7 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msg *nats.Msg) bo
 		return true
 	}
 
-	timestamp, err := strconv.Atoi(msg.Header.Get("timestamp"))
+	timestamp, err := strconv.ParseUint(msg.Header.Get("timestamp"), 10, 64)
 	if err != nil {
 		// If the message was invalid, log it and move on to the next message in the stream
 		log.WithError(err).Errorf("EDU output log: message parse failure")
diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go
index e371baaaa..43dd08dd8 100644
--- a/federationapi/producers/syncapi.go
+++ b/federationapi/producers/syncapi.go
@@ -53,7 +53,7 @@ func (p *SyncAPIProducer) SendReceipt(
 	m.Header.Set(jetstream.RoomID, roomID)
 	m.Header.Set(jetstream.EventID, eventID)
 	m.Header.Set("type", receiptType)
-	m.Header.Set("timestamp", strconv.Itoa(int(timestamp)))
+	m.Header.Set("timestamp", fmt.Sprintf("%d", timestamp))
 
 	log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent)
 	_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
diff --git a/syncapi/consumers/receipts.go b/syncapi/consumers/receipts.go
index 6bb0747f0..83156cf93 100644
--- a/syncapi/consumers/receipts.go
+++ b/syncapi/consumers/receipts.go
@@ -87,7 +87,7 @@ func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msg *nats.Ms
 		Type:    msg.Header.Get("type"),
 	}
 
-	timestamp, err := strconv.Atoi(msg.Header.Get("timestamp"))
+	timestamp, err := strconv.ParseUint(msg.Header.Get("timestamp"), 10, 64)
 	if err != nil {
 		// If the message was invalid, log it and move on to the next message in the stream
 		log.WithError(err).Errorf("output log: message parse failure")

From f3e8a9a4cbd63e0bd724bbef4172b810c8040fa6 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 7 Jul 2022 11:40:53 +0200
Subject: [PATCH 195/391] Fix nil pointer access when redacting events (#2560)

---
 roomserver/storage/shared/storage.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 3191280cb..ba937ba30 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -834,6 +834,9 @@ func (d *Database) handleRedactions(
 	if err != nil {
 		return nil, "", fmt.Errorf("d.GetStateEvent: %w", err)
 	}
+	if powerLevels == nil {
+		return nil, "", fmt.Errorf("unable to fetch m.room.power_levels event from database for room %s", event.RoomID())
+	}
 	pl, err := powerLevels.PowerLevels()
 	if err != nil {
 		return nil, "", fmt.Errorf("unable to get powerlevels for room: %w", err)

From f76f28e6db85df145d54f4dab86be721ef641a83 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 7 Jul 2022 16:29:25 +0200
Subject: [PATCH 196/391] Fix issue `uint64 values with high bit are not
 supported` in presence (#2562)

* Fix issue #2528

* Use gomatrixserverlib.Timestamp

* Use ParseUint instead of ParseInt
---
 syncapi/consumers/presence.go    | 8 ++++----
 syncapi/sync/requestpool.go      | 4 ++--
 syncapi/sync/requestpool_test.go | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go
index 0217e1956..db7d67fa6 100644
--- a/syncapi/consumers/presence.go
+++ b/syncapi/consumers/presence.go
@@ -144,7 +144,7 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 		return true
 	}
 
-	ts, err := strconv.Atoi(timestamp)
+	ts, err := strconv.ParseUint(timestamp, 10, 64)
 	if err != nil {
 		return true
 	}
@@ -157,12 +157,12 @@ func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 	// already checked, so no need to check error
 	p, _ := types.PresenceFromString(presence)
 
-	s.EmitPresence(ctx, userID, p, statusMsg, ts, fromSync)
+	s.EmitPresence(ctx, userID, p, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
 	return true
 }
 
-func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
-	pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, gomatrixserverlib.Timestamp(ts), fromSync)
+func (s *PresenceConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts gomatrixserverlib.Timestamp, fromSync bool) {
+	pos, err := s.db.UpdatePresence(ctx, userID, presence, statusMsg, ts, fromSync)
 	if err != nil {
 		logrus.WithError(err).WithField("user", userID).WithField("presence", presence).Warn("failed to updated presence for user")
 		return
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 7b9526b53..6f0849e08 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -61,7 +61,7 @@ type PresencePublisher interface {
 }
 
 type PresenceConsumer interface {
-	EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool)
+	EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts gomatrixserverlib.Timestamp, fromSync bool)
 }
 
 // NewRequestPool makes a new RequestPool
@@ -171,7 +171,7 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
 	// the /sync response else we may not return presence: online immediately.
 	rp.consumer.EmitPresence(
 		context.Background(), userID, presenceID, newPresence.ClientFields.StatusMsg,
-		int(gomatrixserverlib.AsTimestamp(time.Now())), true,
+		gomatrixserverlib.AsTimestamp(time.Now()), true,
 	)
 }
 
diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go
index 0c7209521..48e6c6c7a 100644
--- a/syncapi/sync/requestpool_test.go
+++ b/syncapi/sync/requestpool_test.go
@@ -40,7 +40,7 @@ func (d dummyDB) MaxStreamPositionForPresence(ctx context.Context) (types.Stream
 
 type dummyConsumer struct{}
 
-func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts int, fromSync bool) {
+func (d dummyConsumer) EmitPresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, ts gomatrixserverlib.Timestamp, fromSync bool) {
 
 }
 

From eb8dc50a970cc1bfd82dc4bace76aba00181df6e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 8 Jul 2022 14:54:08 +0100
Subject: [PATCH 197/391] Update Pinecone to matrix-org/pinecone@1ce778f

---
 build/gobind-pinecone/monolith.go  | 2 +-
 cmd/dendrite-demo-pinecone/main.go | 2 +-
 go.mod                             | 2 +-
 go.sum                             | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index b44e110ad..f3895ae23 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -239,7 +239,7 @@ func (m *DendriteMonolith) Start() {
 	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
 	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
 	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
-	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter)
+	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
 
 	prefix := hex.EncodeToString(pk)
 	cfg := &config.Dendrite{}
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 703436051..8fa935ddf 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -92,7 +92,7 @@ func main() {
 	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
 	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
 	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
-	pManager := pineconeConnections.NewConnectionManager(pRouter)
+	pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
 	pMulticast.Start()
 	if instancePeer != nil && *instancePeer != "" {
 		pManager.AddPeer(*instancePeer)
diff --git a/go.mod b/go.mod
index 4c432611e..cf178cfcc 100644
--- a/go.mod
+++ b/go.mod
@@ -35,7 +35,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f
-	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
+	github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
diff --git a/go.sum b/go.sum
index 777a31228..3ff09b2dc 100644
--- a/go.sum
+++ b/go.sum
@@ -420,8 +420,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f h1:XF2+J6sOq07yhK1I7ItwsgRwXorjj7gqiCvgZ4dn8W8=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
-github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
-github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
+github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs=
+github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 3ea21273bcc151b36eec412d0ec550642fe9b04f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 11 Jul 2022 14:31:31 +0100
Subject: [PATCH 198/391] Ristretto cache (#2563)

* Try Ristretto cache

* Tweak

* It's beautiful

* Update GMSL

* More strict keyable interface

* Fix that some more

* Make less panicky

* Don't enforce mutability checks for now

* Determine mutability using deep equality

* Tweaks

* Namespace keys

* Make federation caches mutable

* Update cost estimation, add metric

* Update GMSL

* Estimate cost for metrics better

* Reduce counters a bit

* Try caching events

* Some guards

* Try again

* Try this

* Use separate caches for hopefully better hash distribution

* Fix bug with admitting events into cache

* Try to fix bugs

* Check nil

* Try that again

* Preserve order jeezo this is messy

* thanks VS Code for doing exactly the wrong thing

* Try this again

* Be more specific

* aaaaargh

* One more time

* That might be better

* Stronger sorting

* Cache expiries, async publishing of EDUs

* Put it back

* Use a shared cache again

* Cost estimation fixes

* Update ristretto

* Reduce counters a bit

* Clean up a bit

* Update GMSL

* 1GB

* Configurable cache sizees

* Tweaks

* Add `config.DataUnit` for specifying friendly cache sizes

* Various tweaks

* Update GMSL

* Add back some lazy loading caching

* Include key in cost

* Include key in cost

* Tweak max age handling, config key name

* Only register prometheus metrics if requested

* Review comments @S7evinK

* Don't return errors when creating caches (it is better just to crash since otherwise we'll `nil`-pointer exception everywhere)

* Review comments

* Update sample configs

* Update GHA Workflow

* Update Complement images to Go 1.18

* Remove the cache test from the federation API as we no longer guarantee immediate cache admission

* Don't check the caches in the renewal test

* Possibly fix the upgrade tests

* Update to matrix-org/gomatrixserverlib#322

* Update documentation to refer to Go 1.18
---
 .github/workflows/dendrite.yml                |  10 +-
 README.md                                     |   2 +-
 build/scripts/Complement.Dockerfile           |   8 +-
 build/scripts/ComplementLocal.Dockerfile      |  28 +--
 build/scripts/ComplementPostgres.Dockerfile   |  34 +--
 cmd/dendrite-demo-yggdrasil/README.md         |   6 +-
 cmd/dendrite-upgrade-tests/main.go            |   2 +-
 cmd/resolve-state/main.go                     |  11 +-
 dendrite-sample.monolith.yaml                 |  19 ++
 dendrite-sample.polylith.yaml                 |  19 ++
 docs/installation/1_planning.md               |   2 +-
 federationapi/federationapi_keys_test.go      | 118 +----------
 federationapi/routing/invite.go               |  12 +-
 federationapi/routing/join.go                 |  11 +-
 federationapi/routing/leave.go                |  11 +-
 go.mod                                        |  85 +++++---
 go.sum                                        | 174 +--------------
 internal/caching/cache_federationevents.go    |  39 +---
 internal/caching/cache_lazy_load_members.go   |  68 ++----
 internal/caching/cache_roomevents.go          |  21 ++
 internal/caching/cache_roominfo.go            |  17 +-
 internal/caching/cache_roomservernids.go      |  20 +-
 internal/caching/cache_roomversions.go        |  15 +-
 internal/caching/cache_serverkeys.go          |  24 +--
 internal/caching/cache_space_rooms.go         |  17 +-
 internal/caching/caches.go                    |  52 +++--
 internal/caching/impl_inmemorylru.go          | 189 -----------------
 internal/caching/impl_ristretto.go            | 200 ++++++++++++++++++
 internal/eventutil/events.go                  |  18 +-
 roomserver/internal/input/input_events.go     |   4 +-
 roomserver/internal/input/input_test.go       |   6 +-
 .../internal/perform/perform_backfill.go      |   5 +-
 roomserver/state/state.go                     |   2 +-
 roomserver/storage/shared/storage.go          |  39 +++-
 setup/base/base.go                            |   7 +-
 setup/config/config_global.go                 |  46 ++++
 setup/config/config_test.go                   |  21 ++
 syncapi/storage/shared/syncserver.go          |   5 +-
 38 files changed, 603 insertions(+), 764 deletions(-)
 create mode 100644 internal/caching/cache_roomevents.go
 delete mode 100644 internal/caching/impl_inmemorylru.go
 create mode 100644 internal/caching/impl_ristretto.go

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 6ebef4e13..9b452d849 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -24,7 +24,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16
+          go-version: 1.18
 
       - uses: actions/cache@v2
         with:
@@ -97,7 +97,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.16", "1.17", "1.18"]
+        go: ["1.18"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
@@ -127,7 +127,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.16", "1.17", "1.18"]
+        go: ["1.18"]
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
@@ -160,7 +160,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.16", "1.17", "1.18"]
+        go: ["1.18"]
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
@@ -209,7 +209,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v2
         with:
-          go-version: "1.16"
+          go-version: "1.18"
       - uses: actions/cache@v3
         with:
           path: |
diff --git a/README.md b/README.md
index 7c22b3692..8f54db7b7 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 
 ## Requirements
 
-To build Dendrite, you will need Go 1.16 or later.
+To build Dendrite, you will need Go 1.18 or later.
 
 For a usable federating Dendrite deployment, you will also need:
 
diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile
index 63e3890ee..56877051b 100644
--- a/build/scripts/Complement.Dockerfile
+++ b/build/scripts/Complement.Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.16-stretch as build
+FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
 
@@ -27,6 +27,6 @@ EXPOSE 8008 8448
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
 CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
- ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
- cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+    cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+    ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile
index a9feb4cd1..704359a28 100644
--- a/build/scripts/ComplementLocal.Dockerfile
+++ b/build/scripts/ComplementLocal.Dockerfile
@@ -6,7 +6,7 @@
 #
 # Use these mounts to make use of this dockerfile:
 # COMPLEMENT_HOST_MOUNTS='/your/local/dendrite:/dendrite:ro;/your/go/path:/go:ro'
-FROM golang:1.16-stretch
+FROM golang:1.18-stretch
 RUN apt-get update && apt-get install -y sqlite3
 
 WORKDIR /runtime
@@ -16,24 +16,24 @@ EXPOSE 8008 8448
 
 # This script compiles Dendrite for us.
 RUN echo '\
-#!/bin/bash -eux \n\
-if test -f "/runtime/dendrite-monolith-server"; then \n\
+    #!/bin/bash -eux \n\
+    if test -f "/runtime/dendrite-monolith-server"; then \n\
     echo "Skipping compilation; binaries exist" \n\
     exit 0 \n\
-fi \n\
-cd /dendrite \n\
-go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
-' > compile.sh && chmod +x compile.sh
+    fi \n\
+    cd /dendrite \n\
+    go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
+    ' > compile.sh && chmod +x compile.sh
 
 # This script runs Dendrite for us. Must be run in the /runtime directory.
 RUN echo '\
-#!/bin/bash -eu \n\
-./generate-keys --private-key matrix_key.pem \n\
-./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
-./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
-cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
-./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
-' > run.sh && chmod +x run.sh
+    #!/bin/bash -eu \n\
+    ./generate-keys --private-key matrix_key.pem \n\
+    ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
+    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
+    cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
+    ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    ' > run.sh && chmod +x run.sh
 
 
 WORKDIR /cache
diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile
index 4e26faa58..a8b4fbb1d 100644
--- a/build/scripts/ComplementPostgres.Dockerfile
+++ b/build/scripts/ComplementPostgres.Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.16-stretch as build
+FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
@@ -9,16 +9,16 @@ RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
-#!/bin/bash -eu \n\
-pg_lsclusters \n\
-pg_ctlcluster 9.6 main start \n\
- \n\
-until pg_isready \n\
-do \n\
-  echo "Waiting for postgres"; \n\
-  sleep 1; \n\
-done \n\
-' > run_postgres.sh && chmod +x run_postgres.sh
+    #!/bin/bash -eu \n\
+    pg_lsclusters \n\
+    pg_ctlcluster 9.6 main start \n\
+    \n\
+    until pg_isready \n\
+    do \n\
+    echo "Waiting for postgres"; \n\
+    sleep 1; \n\
+    done \n\
+    ' > run_postgres.sh && chmod +x run_postgres.sh
 
 # we will dump the binaries and config file to this location to ensure any local untracked files
 # that come from the COPY . . file don't contaminate the build
@@ -46,9 +46,9 @@ EXPOSE 8008 8448
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
 CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
- ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
- # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
- sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
- sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
- cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
+    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+    # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
+    sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
+    sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
+    cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+    ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md
index 946333576..14fc3a2db 100644
--- a/cmd/dendrite-demo-yggdrasil/README.md
+++ b/cmd/dendrite-demo-yggdrasil/README.md
@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.16 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run:
 
@@ -13,10 +13,10 @@ The following command line arguments are accepted:
 * `-peer tcp://a.b.c.d:e` to specify a static Yggdrasil peer to connect to - you will need to supply this if you do not have another Yggdrasil node on your network
 * `-port 12345` to specify a port to listen on for client connections
 
-If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/). 
+If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/).
 
 Then point your favourite Matrix client to  the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in.
 
 If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
 
-Once logged in, you should be able to open the room directory or join a room by its ID.
\ No newline at end of file
+Once logged in, you should be able to open the room directory or join a room by its ID.
diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index b7e7da07d..cabd07e70 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -48,7 +48,7 @@ const HEAD = "HEAD"
 // due to the error:
 //   When using COPY with more than one source file, the destination must be a directory and end with a /
 // We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
-const Dockerfile = `FROM golang:1.16-stretch as build
+const Dockerfile = `FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index c02140003..6852ab4fb 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/state"
@@ -53,12 +54,10 @@ func main() {
 
 	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
 
-	cache, err := caching.NewInMemoryLRUCache(true)
-	if err != nil {
-		panic(err)
-	}
-
-	roomserverDB, err := storage.Open(base, &cfg.RoomServer.Database, cache)
+	roomserverDB, err := storage.Open(
+		base, &cfg.RoomServer.Database,
+		caching.NewRistrettoCache(128*1024*1024, time.Hour, true),
+	)
 	if err != nil {
 		panic(err)
 	}
diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index c0134c542..c6050e407 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -41,6 +41,25 @@ global:
     max_idle_conns: 5
     conn_max_lifetime: -1
 
+  # Configuration for in-memory caches. Caches can often improve performance by
+  # keeping frequently accessed items (like events, identifiers etc.) in memory
+  # rather than having to read them from the database.
+  cache:
+    # The estimated maximum size for the global cache in bytes, or in terabytes,
+    # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
+    # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
+    # memory limit for the entire process. A cache that is too small may ultimately
+    # provide little or no benefit.
+    max_size_estimated: 1gb
+
+    # The maximum amount of time that a cache entry can live for in memory before
+    # it will be evicted and/or refreshed from the database. Lower values result in
+    # easier admission of new cache entries but may also increase database load in
+    # comparison to higher values, so adjust conservatively. Higher values may make
+    # it harder for new items to make it into the cache, e.g. if new rooms suddenly
+    # become popular.
+    max_age: 1h
+
   # The server name to delegate server-server communications to, with optional port
   # e.g. localhost:443
   well_known_server_name: ""
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index b9e3e5e56..ea3d9d689 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -31,6 +31,25 @@ global:
   # considered valid by other homeservers.
   key_validity_period: 168h0m0s
 
+  # Configuration for in-memory caches. Caches can often improve performance by
+  # keeping frequently accessed items (like events, identifiers etc.) in memory
+  # rather than having to read them from the database.
+  cache:
+    # The estimated maximum size for the global cache in bytes, or in terabytes,
+    # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
+    # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
+    # memory limit for the entire process. A cache that is too small may ultimately
+    # provide little or no benefit.
+    max_size_estimated: 1gb
+
+    # The maximum amount of time that a cache entry can live for in memory before
+    # it will be evicted and/or refreshed from the database. Lower values result in
+    # easier admission of new cache entries but may also increase database load in
+    # comparison to higher values, so adjust conservatively. Higher values may make
+    # it harder for new items to make it into the cache, e.g. if new rooms suddenly
+    # become popular.
+    max_age: 1h
+
   # The server name to delegate server-server communications to, with optional port
   # e.g. localhost:443
   well_known_server_name: ""
diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md
index 45012d80c..d4f3d7052 100644
--- a/docs/installation/1_planning.md
+++ b/docs/installation/1_planning.md
@@ -75,7 +75,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie
 
 ### Go
 
-At this time, Dendrite supports being built with Go 1.16 or later. We do not support building
+At this time, Dendrite supports being built with Go 1.18 or later. We do not support building
 Dendrite with older versions of Go than this. If you are installing Go using a package manager,
 you should check (by running `go version`) that you are using a suitable version before you start.
 
diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go
index 73f737717..d1bfe1847 100644
--- a/federationapi/federationapi_keys_test.go
+++ b/federationapi/federationapi_keys_test.go
@@ -9,7 +9,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"reflect"
 	"testing"
 	"time"
 
@@ -64,10 +63,7 @@ func TestMain(m *testing.M) {
 			}
 
 			// Create a new cache but don't enable prometheus!
-			s.cache, err = caching.NewInMemoryLRUCache(false)
-			if err != nil {
-				panic("can't create cache: " + err.Error())
-			}
+			s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false)
 
 			// Create a temporary directory for JetStream.
 			d, err := ioutil.TempDir("./", "jetstream*")
@@ -170,72 +166,6 @@ func TestServersRequestOwnKeys(t *testing.T) {
 	}
 }
 
-func TestCachingBehaviour(t *testing.T) {
-	// Server A will request Server B's key, which has a validity
-	// period of an hour from now. We should retrieve the key and
-	// it should make it into the cache automatically.
-
-	req := gomatrixserverlib.PublicKeyLookupRequest{
-		ServerName: serverB.name,
-		KeyID:      serverKeyID,
-	}
-	ts := gomatrixserverlib.AsTimestamp(time.Now())
-
-	res, err := serverA.api.FetchKeys(
-		context.Background(),
-		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
-			req: ts,
-		},
-	)
-	if err != nil {
-		t.Fatalf("server A failed to retrieve server B key: %s", err)
-	}
-	if len(res) != 1 {
-		t.Fatalf("server B should have returned one key but instead returned %d keys", len(res))
-	}
-	if _, ok := res[req]; !ok {
-		t.Fatalf("server B isn't included in the key fetch response")
-	}
-
-	// At this point, if the previous key request was a success,
-	// then the cache should now contain the key. Check if that's
-	// the case - if it isn't then there's something wrong with
-	// the cache implementation or we failed to get the key.
-
-	cres, ok := serverA.cache.GetServerKey(req, ts)
-	if !ok {
-		t.Fatalf("server B key should be in cache but isn't")
-	}
-	if !reflect.DeepEqual(cres, res[req]) {
-		t.Fatalf("the cached result from server B wasn't what server B gave us")
-	}
-
-	// If we ask the cache for the same key but this time for an event
-	// that happened in +30 minutes. Since the validity period is for
-	// another hour, then we should get a response back from the cache.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*30)),
-	)
-	if !ok {
-		t.Fatalf("server B key isn't in cache when it should be (+30 minutes)")
-	}
-
-	// If we ask the cache for the same key but this time for an event
-	// that happened in +90 minutes then we should expect to get no
-	// cache result. This is because the cache shouldn't return a result
-	// that is obviously past the validity of the event.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*90)),
-	)
-	if ok {
-		t.Fatalf("server B key is in cache when it shouldn't be (+90 minutes)")
-	}
-}
-
 func TestRenewalBehaviour(t *testing.T) {
 	// Server A will request Server C's key but their validity period
 	// is an hour in the past. We'll retrieve the key as, even though it's
@@ -262,32 +192,7 @@ func TestRenewalBehaviour(t *testing.T) {
 		t.Fatalf("server C isn't included in the key fetch response")
 	}
 
-	// If we ask the cache for the server key for an event that happened
-	// 90 minutes ago then we should get a cache result, as the key hadn't
-	// passed its validity by that point. The fact that the key is now in
-	// the cache is, in itself, proof that we successfully retrieved the
-	// key before.
-
-	oldcached, ok := serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*90)),
-	)
-	if !ok {
-		t.Fatalf("server C key isn't in cache when it should be (-90 minutes)")
-	}
-
-	// If we now ask the cache for the same key but this time for an event
-	// that only happened 30 minutes ago then we shouldn't get a cached
-	// result, as the event happened after the key validity expired. This
-	// is really just for sanity checking.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
-	)
-	if ok {
-		t.Fatalf("server B key is in cache when it shouldn't be (-30 minutes)")
-	}
+	originalValidity := res[req].ValidUntilTS
 
 	// We're now going to kick server C into renewing its key. Since we're
 	// happy at this point that the key that we already have is from the past
@@ -308,24 +213,13 @@ func TestRenewalBehaviour(t *testing.T) {
 	if len(res) != 1 {
 		t.Fatalf("server C should have returned one key but instead returned %d keys", len(res))
 	}
-	if _, ok = res[req]; !ok {
+	if _, ok := res[req]; !ok {
 		t.Fatalf("server C isn't included in the key fetch response")
 	}
 
-	// We're now going to ask the cache what the new key validity is. If
-	// it is still the same as the previous validity then we've failed to
-	// retrieve the renewed key. If it's newer then we've successfully got
-	// the renewed key.
+	currentValidity := res[req].ValidUntilTS
 
-	newcached, ok := serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
-	)
-	if !ok {
-		t.Fatalf("server B key isn't in cache when it shouldn't be (post-renewal)")
+	if originalValidity == currentValidity {
+		t.Fatalf("server C key should have renewed but didn't")
 	}
-	if oldcached.ValidUntilTS >= newcached.ValidUntilTS {
-		t.Fatalf("the server B key should have been renewed but wasn't")
-	}
-	t.Log(res)
 }
diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go
index a5797645e..cde87a0ac 100644
--- a/federationapi/routing/invite.go
+++ b/federationapi/routing/invite.go
@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
@@ -141,10 +142,17 @@ func processInvite(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: invite.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 8d779c3d7..41004cf51 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -266,10 +266,17 @@ func SendJoin(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: join.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index 54b2c3e84..dbaf68e5b 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -231,10 +231,17 @@ func SendLeave(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: leave.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}
diff --git a/go.mod b/go.mod
index cf178cfcc..2a2a037c1 100644
--- a/go.mod
+++ b/go.mod
@@ -7,56 +7,39 @@ replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0
 require (
 	github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/DATA-DOG/go-sqlmock v1.5.0
-	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/codeclysm/extract v2.2.0+incompatible
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d
 	github.com/docker/docker v20.10.16+incompatible
 	github.com/docker/go-connections v0.4.0
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/frankban/quicktest v1.14.3 // indirect
 	github.com/getsentry/sentry-go v0.13.0
-	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gologme/log v1.3.0
 	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
-	github.com/h2non/filetype v1.1.3 // indirect
-	github.com/hashicorp/golang-lru v0.5.4
-	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
 	github.com/kardianos/minwinsvc v1.0.0
 	github.com/lib/pq v1.10.5
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c
 	github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/miekg/dns v1.1.49 // indirect
-	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
 	github.com/nats-io/nats.go v1.14.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
-	github.com/onsi/gomega v1.17.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose v2.7.0+incompatible
 	github.com/prometheus/client_golang v1.12.2
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.7.1
 	github.com/tidwall/gjson v1.14.1
 	github.com/tidwall/sjson v1.2.4
@@ -68,14 +51,68 @@ require (
 	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
 	golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
 	golang.org/x/net v0.0.0-20220524220425-1d687d428aca
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
-	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0 // indirect
-	gotest.tools/v3 v3.0.3 // indirect
 	nhooyr.io/websocket v1.8.7
 )
 
-go 1.16
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
+	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/frankban/quicktest v1.14.3 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/h2non/filetype v1.1.3 // indirect
+	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
+	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
+	github.com/klauspost/compress v1.14.4 // indirect
+	github.com/lucas-clemente/quic-go v0.26.0 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/miekg/dns v1.1.49 // indirect
+	github.com/minio/highwayhash v1.0.2 // indirect
+	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect
+	github.com/nats-io/nkeys v0.3.0 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/gomega v1.17.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
+	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
+	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/macaroon.v2 v2.1.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gotest.tools/v3 v3.0.3 // indirect
+)
+
+go 1.18
diff --git a/go.sum b/go.sum
index 3ff09b2dc..98549f702 100644
--- a/go.sum
+++ b/go.sum
@@ -38,7 +38,6 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
 github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 h1:PsaZb47k7WB1V+AlGpb+W7SM+ZOhp16vVevg5gl9YkU=
 github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
@@ -49,13 +48,10 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
-github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
-github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc=
 github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@@ -65,10 +61,8 @@ github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpz
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
 github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
 github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
-github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
-github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -88,8 +82,6 @@ github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikz
 github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
 github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -114,18 +106,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI=
 github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk=
+github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -137,16 +126,14 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
-github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
@@ -157,7 +144,6 @@ github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUork
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
 github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
 github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -165,11 +151,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
-github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
-github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -181,7 +165,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -189,7 +172,6 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
-github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
@@ -202,8 +184,8 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -239,7 +221,6 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
 github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo=
 github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4=
-github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -289,24 +270,13 @@ github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
-github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
-github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
-github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
-github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -318,67 +288,28 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/juju/ansiterm v0.0.0-20160907234532-b99631de12cf/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
-github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
-github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA=
-github.com/juju/cmd v0.0.0-20171107070456-e74f39857ca0/go.mod h1:yWJQHl73rdSX4DHVKGqkAip+huBslxRwS8m9CrOLq18=
-github.com/juju/collections v0.0.0-20200605021417-0d0ec82b7271/go.mod h1:5XgO71dV1JClcOJE+4dzdn4HrI5LiyKd7PlVG6eZYhY=
-github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
-github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 h1:EJHbsNpQyupmMeWTq7inn+5L/WZ7JfzCVPJ+DP9McCQ=
 github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9/go.mod h1:TRm7EVGA3mQOqSVcBySRY7a9Y1/gyVhh/WTCnc5sD4U=
-github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
-github.com/juju/httpprof v0.0.0-20141217160036-14bf14c30767/go.mod h1:+MaLYz4PumRkkyHYeXJ2G5g5cIW0sli2bOfpmbaMV/g=
-github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
-github.com/juju/loggo v0.0.0-20200526014432-9ce3a2e09b5e/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4 h1:NO5tuyw++EGLnz56Q8KMyDZRwJwWO8jQnj285J3FOmY=
-github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4/go.mod h1:NIXFioti1SmKAlKNuUwbMenNdef59IF52+ZzuOmHYkg=
 github.com/juju/mgo/v2 v2.0.0-20210302023703-70d5d206e208 h1:/WiCm+Vpj87e4QWuWwPD/bNE9kDrWCLvPBHOQNcG2+A=
-github.com/juju/mgo/v2 v2.0.0-20210302023703-70d5d206e208/go.mod h1:0OChplkvPTZ174D2FYZXg4IB9hbEwyHkD+zT+/eK+Fg=
-github.com/juju/mutex v0.0.0-20171110020013-1fe2a4bf0a3a/go.mod h1:Y3oOzHH8CQ0Ppt0oCKJ2JFO81/EsWenH5AEqigLH+yY=
-github.com/juju/retry v0.0.0-20151029024821-62c620325291/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4=
-github.com/juju/retry v0.0.0-20180821225755-9058e192b216/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4=
-github.com/juju/testing v0.0.0-20180402130637-44801989f0f7/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
-github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
-github.com/juju/testing v0.0.0-20210302031854-2c7ee8570c07/go.mod h1:7lxZW0B50+xdGFkvhAb8bwAGt6IU87JB1H9w4t8MNVM=
-github.com/juju/testing v0.0.0-20220202055744-1ad0816210a6/go.mod h1:QgWc2UdIPJ8t3rnvv95tFNOsQDfpXYEZDbP281o3b2c=
 github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 h1:XEDzpuZb8Ma7vLja3+5hzUqVTvAqm5Y+ygvnDs5iTMM=
 github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494/go.mod h1:rUquetT0ALL48LHZhyRGvjjBH8xZaZ8dFClulKK5wK4=
-github.com/juju/utils v0.0.0-20180424094159-2000ea4ff043/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk=
-github.com/juju/utils v0.0.0-20200116185830-d40c2fe10647/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk=
-github.com/juju/utils/v2 v2.0.0-20200923005554-4646bfea2ef1/go.mod h1:fdlDtQlzundleLLz/ggoYinEt/LmnrpNKcNTABQATNI=
-github.com/juju/utils/v3 v3.0.0-20220130232349-cd7ecef0e94a/go.mod h1:LzwbbEN7buYjySp4nqnti6c6olSqRXUk6RkbSUUP1n8=
-github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U=
-github.com/juju/version v0.0.0-20180108022336-b64dbd566305/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U=
-github.com/juju/version v0.0.0-20191219164919-81c1be00b9a6/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U=
-github.com/juju/version/v2 v2.0.0-20211007103408-2e8da085dc23/go.mod h1:Ljlbryh9sYaUSGXucslAEDf0A2XUSGvDbHJgW8ps6nc=
-github.com/julienschmidt/httprouter v1.1.1-0.20151013225520-77a895ad01eb/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA=
 github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc=
-github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=
-github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=
-github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
-github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
-github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4=
 github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -386,19 +317,15 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
-github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
 github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A=
 github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
-github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
@@ -407,10 +334,6 @@ github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWCh
 github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
 github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y=
 github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
-github.com/masterzen/azure-sdk-for-go v3.2.0-beta.0.20161014135628-ee4f0065d00c+incompatible/go.mod h1:mf8fjOu33zCqxUjuiU3I8S1lJMyEAlH+0F2+M5xl3hE=
-github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc=
-github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zwz5gJiFhZJz49Uzk7mEBHIceWmbFmYx7Hf7E=
-github.com/masterzen/xmlpath v0.0.0-20140218185901-13f4951698ad/go.mod h1:A0zPC53iKKKcXYxr4ROjpQRQ5FgJXtelNdSmHHuq/tY=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA=
@@ -418,43 +341,31 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f h1:XF2+J6sOq07yhK1I7ItwsgRwXorjj7gqiCvgZ4dn8W8=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c h1:mt30TDK8kXKV+nCmVfnqoXsh842N+74kvZw7DXuS/JQ=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
-github.com/mattn/go-colorable v0.0.6/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
-github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
 github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
-github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
 github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
@@ -467,7 +378,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
-github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -491,7 +401,6 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S
 github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 h1:Dmx8g2747UTVPzSkmohk84S3g/uWqd6+f4SSLPhLcfA=
 github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@@ -515,10 +424,8 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
-github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -558,9 +465,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
-github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
 github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
@@ -582,7 +487,6 @@ github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b
 github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
 github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
 github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
 github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -593,15 +497,9 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
@@ -631,30 +529,16 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK
 github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
-github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
-github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yggdrasil-network/yggdrasil-go v0.4.3 h1:LNS7kNpKzFlxQ9xmD5tfmMEvzwa+utBoD6pV9t2a8q4=
 github.com/yggdrasil-network/yggdrasil-go v0.4.3/go.mod h1:A1/8kOQT7vzBxlkQtLf1KzJR0cbfL/2zjOCiYOAdjjo=
-github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
-github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -672,26 +556,20 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
-golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -736,21 +614,19 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -759,7 +635,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -776,7 +651,6 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
@@ -785,10 +659,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220524220425-1d687d428aca h1:xTaFYiPROfpPhqrfTIDXj0ri1SpfueYT951s4bAuDO8=
 golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -816,10 +688,8 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -829,9 +699,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -879,14 +747,11 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -895,29 +760,24 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0=
 golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M=
 golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -960,8 +820,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
-golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0=
 golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1066,32 +927,22 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/errgo.v1 v1.0.0-20161222125816-442357a80af5/go.mod h1:u0ALmqvLRxLI95fkdCEWrE6mhWYZW1aMOJHp5YXLHTg=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM=
 gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
 gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
 gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
-gopkg.in/httprequest.v1 v1.1.1/go.mod h1:/CkavNL+g3qLOrpFHVrEx4NKepeqR4XTZWNj4sGGjz0=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI=
 gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o=
-gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
-gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
-gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk=
-gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -1100,7 +951,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
@@ -1117,8 +967,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
-launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/internal/caching/cache_federationevents.go b/internal/caching/cache_federationevents.go
index b79cc809f..24af51bdc 100644
--- a/internal/caching/cache_federationevents.go
+++ b/internal/caching/cache_federationevents.go
@@ -1,18 +1,9 @@
 package caching
 
 import (
-	"fmt"
-
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-const (
-	FederationEventCacheName       = "federation_event"
-	FederationEventCacheMaxEntries = 256
-	FederationEventCacheMutable    = true // to allow use of Unset only
-	FederationEventCacheMaxAge     = CacheNoMaxAge
-)
-
 // FederationCache contains the subset of functions needed for
 // a federation event cache.
 type FederationCache interface {
@@ -26,43 +17,25 @@ type FederationCache interface {
 }
 
 func (c Caches) GetFederationQueuedPDU(eventNID int64) (*gomatrixserverlib.HeaderedEvent, bool) {
-	key := fmt.Sprintf("%d", eventNID)
-	val, found := c.FederationEvents.Get(key)
-	if found && val != nil {
-		if event, ok := val.(*gomatrixserverlib.HeaderedEvent); ok {
-			return event, true
-		}
-	}
-	return nil, false
+	return c.FederationPDUs.Get(eventNID)
 }
 
 func (c Caches) StoreFederationQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) {
-	key := fmt.Sprintf("%d", eventNID)
-	c.FederationEvents.Set(key, event)
+	c.FederationPDUs.Set(eventNID, event)
 }
 
 func (c Caches) EvictFederationQueuedPDU(eventNID int64) {
-	key := fmt.Sprintf("%d", eventNID)
-	c.FederationEvents.Unset(key)
+	c.FederationPDUs.Unset(eventNID)
 }
 
 func (c Caches) GetFederationQueuedEDU(eventNID int64) (*gomatrixserverlib.EDU, bool) {
-	key := fmt.Sprintf("%d", eventNID)
-	val, found := c.FederationEvents.Get(key)
-	if found && val != nil {
-		if event, ok := val.(*gomatrixserverlib.EDU); ok {
-			return event, true
-		}
-	}
-	return nil, false
+	return c.FederationEDUs.Get(eventNID)
 }
 
 func (c Caches) StoreFederationQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) {
-	key := fmt.Sprintf("%d", eventNID)
-	c.FederationEvents.Set(key, event)
+	c.FederationEDUs.Set(eventNID, event)
 }
 
 func (c Caches) EvictFederationQueuedEDU(eventNID int64) {
-	key := fmt.Sprintf("%d", eventNID)
-	c.FederationEvents.Unset(key)
+	c.FederationEDUs.Unset(eventNID)
 }
diff --git a/internal/caching/cache_lazy_load_members.go b/internal/caching/cache_lazy_load_members.go
index f0d495065..0d7009c94 100644
--- a/internal/caching/cache_lazy_load_members.go
+++ b/internal/caching/cache_lazy_load_members.go
@@ -1,67 +1,35 @@
 package caching
 
 import (
-	"fmt"
-	"time"
-
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
-const (
-	LazyLoadCacheName           = "lazy_load_members"
-	LazyLoadCacheMaxEntries     = 128
-	LazyLoadCacheMaxUserEntries = 128
-	LazyLoadCacheMutable        = true
-	LazyLoadCacheMaxAge         = time.Minute * 30
-)
+type lazyLoadingCacheKey struct {
+	UserID       string // the user we're querying on behalf of
+	DeviceID     string // the user we're querying on behalf of
+	RoomID       string // the room in question
+	TargetUserID string // the user whose membership we're asking about
+}
 
 type LazyLoadCache interface {
 	StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string)
 	IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool)
 }
 
-func (c Caches) lazyLoadCacheForUser(device *userapi.Device) (*InMemoryLRUCachePartition, error) {
-	cacheName := fmt.Sprintf("%s/%s", device.UserID, device.ID)
-	userCache, ok := c.LazyLoading.Get(cacheName)
-	if ok && userCache != nil {
-		if cache, ok := userCache.(*InMemoryLRUCachePartition); ok {
-			return cache, nil
-		}
-	}
-	cache, err := NewInMemoryLRUCachePartition(
-		LazyLoadCacheName,
-		LazyLoadCacheMutable,
-		LazyLoadCacheMaxUserEntries,
-		LazyLoadCacheMaxAge,
-		false,
-	)
-	if err != nil {
-		return nil, err
-	}
-	c.LazyLoading.Set(cacheName, cache)
-	go cacheCleaner(cache)
-	return cache, nil
-}
-
 func (c Caches) StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) {
-	cache, err := c.lazyLoadCacheForUser(device)
-	if err != nil {
-		return
-	}
-	cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID)
-	cache.Set(cacheKey, eventID)
+	c.LazyLoading.Set(lazyLoadingCacheKey{
+		UserID:       device.UserID,
+		DeviceID:     device.ID,
+		RoomID:       roomID,
+		TargetUserID: userID,
+	}, eventID)
 }
 
 func (c Caches) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool) {
-	cache, err := c.lazyLoadCacheForUser(device)
-	if err != nil {
-		return "", false
-	}
-
-	cacheKey := fmt.Sprintf("%s/%s/%s/%s", device.UserID, device.ID, roomID, userID)
-	val, ok := cache.Get(cacheKey)
-	if !ok {
-		return "", ok
-	}
-	return val.(string), ok
+	return c.LazyLoading.Get(lazyLoadingCacheKey{
+		UserID:       device.UserID,
+		DeviceID:     device.ID,
+		RoomID:       roomID,
+		TargetUserID: userID,
+	})
 }
diff --git a/internal/caching/cache_roomevents.go b/internal/caching/cache_roomevents.go
new file mode 100644
index 000000000..9d5d3b912
--- /dev/null
+++ b/internal/caching/cache_roomevents.go
@@ -0,0 +1,21 @@
+package caching
+
+import (
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// RoomServerEventsCache contains the subset of functions needed for
+// a roomserver event cache.
+type RoomServerEventsCache interface {
+	GetRoomServerEvent(eventNID types.EventNID) (*gomatrixserverlib.Event, bool)
+	StoreRoomServerEvent(eventNID types.EventNID, event *gomatrixserverlib.Event)
+}
+
+func (c Caches) GetRoomServerEvent(eventNID types.EventNID) (*gomatrixserverlib.Event, bool) {
+	return c.RoomServerEvents.Get(int64(eventNID))
+}
+
+func (c Caches) StoreRoomServerEvent(eventNID types.EventNID, event *gomatrixserverlib.Event) {
+	c.RoomServerEvents.Set(int64(eventNID), event)
+}
diff --git a/internal/caching/cache_roominfo.go b/internal/caching/cache_roominfo.go
index 60d221285..d03a61077 100644
--- a/internal/caching/cache_roominfo.go
+++ b/internal/caching/cache_roominfo.go
@@ -1,8 +1,6 @@
 package caching
 
 import (
-	"time"
-
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
@@ -14,13 +12,6 @@ import (
 // used from other components as we currently have no way to invalidate
 // the cache in downstream components.
 
-const (
-	RoomInfoCacheName       = "roominfo"
-	RoomInfoCacheMaxEntries = 1024
-	RoomInfoCacheMutable    = true
-	RoomInfoCacheMaxAge     = time.Minute * 5
-)
-
 // RoomInfosCache contains the subset of functions needed for
 // a room Info cache. It must only be used from the roomserver only
 // It is not safe for use from other components.
@@ -32,13 +23,7 @@ type RoomInfoCache interface {
 // GetRoomInfo must only be called from the roomserver only. It is not
 // safe for use from other components.
 func (c Caches) GetRoomInfo(roomID string) (types.RoomInfo, bool) {
-	val, found := c.RoomInfos.Get(roomID)
-	if found && val != nil {
-		if roomInfo, ok := val.(types.RoomInfo); ok {
-			return roomInfo, true
-		}
-	}
-	return types.RoomInfo{}, false
+	return c.RoomInfos.Get(roomID)
 }
 
 // StoreRoomInfo must only be called from the roomserver only. It is not
diff --git a/internal/caching/cache_roomservernids.go b/internal/caching/cache_roomservernids.go
index 1918a2f1e..b409aeef2 100644
--- a/internal/caching/cache_roomservernids.go
+++ b/internal/caching/cache_roomservernids.go
@@ -1,22 +1,14 @@
 package caching
 
 import (
-	"strconv"
-
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
-const (
-	RoomServerRoomIDsCacheName       = "roomserver_room_ids"
-	RoomServerRoomIDsCacheMaxEntries = 1024
-	RoomServerRoomIDsCacheMutable    = false
-	RoomServerRoomIDsCacheMaxAge     = CacheNoMaxAge
-)
-
 type RoomServerCaches interface {
 	RoomServerNIDsCache
 	RoomVersionCache
 	RoomInfoCache
+	RoomServerEventsCache
 }
 
 // RoomServerNIDsCache contains the subset of functions needed for
@@ -27,15 +19,9 @@ type RoomServerNIDsCache interface {
 }
 
 func (c Caches) GetRoomServerRoomID(roomNID types.RoomNID) (string, bool) {
-	val, found := c.RoomServerRoomIDs.Get(strconv.Itoa(int(roomNID)))
-	if found && val != nil {
-		if roomID, ok := val.(string); ok {
-			return roomID, true
-		}
-	}
-	return "", false
+	return c.RoomServerRoomIDs.Get(int64(roomNID))
 }
 
 func (c Caches) StoreRoomServerRoomID(roomNID types.RoomNID, roomID string) {
-	c.RoomServerRoomIDs.Set(strconv.Itoa(int(roomNID)), roomID)
+	c.RoomServerRoomIDs.Set(int64(roomNID), roomID)
 }
diff --git a/internal/caching/cache_roomversions.go b/internal/caching/cache_roomversions.go
index 92d2eab08..afc3d36da 100644
--- a/internal/caching/cache_roomversions.go
+++ b/internal/caching/cache_roomversions.go
@@ -2,13 +2,6 @@ package caching
 
 import "github.com/matrix-org/gomatrixserverlib"
 
-const (
-	RoomVersionCacheName       = "room_versions"
-	RoomVersionCacheMaxEntries = 1024
-	RoomVersionCacheMutable    = false
-	RoomVersionCacheMaxAge     = CacheNoMaxAge
-)
-
 // RoomVersionsCache contains the subset of functions needed for
 // a room version cache.
 type RoomVersionCache interface {
@@ -17,13 +10,7 @@ type RoomVersionCache interface {
 }
 
 func (c Caches) GetRoomVersion(roomID string) (gomatrixserverlib.RoomVersion, bool) {
-	val, found := c.RoomVersions.Get(roomID)
-	if found && val != nil {
-		if roomVersion, ok := val.(gomatrixserverlib.RoomVersion); ok {
-			return roomVersion, true
-		}
-	}
-	return "", false
+	return c.RoomVersions.Get(roomID)
 }
 
 func (c Caches) StoreRoomVersion(roomID string, roomVersion gomatrixserverlib.RoomVersion) {
diff --git a/internal/caching/cache_serverkeys.go b/internal/caching/cache_serverkeys.go
index 4eb10fe6f..cffa101d5 100644
--- a/internal/caching/cache_serverkeys.go
+++ b/internal/caching/cache_serverkeys.go
@@ -6,13 +6,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-const (
-	ServerKeyCacheName       = "server_key"
-	ServerKeyCacheMaxEntries = 4096
-	ServerKeyCacheMutable    = true
-	ServerKeyCacheMaxAge     = CacheNoMaxAge
-)
-
 // ServerKeyCache contains the subset of functions needed for
 // a server key cache.
 type ServerKeyCache interface {
@@ -34,18 +27,13 @@ func (c Caches) GetServerKey(
 ) (gomatrixserverlib.PublicKeyLookupResult, bool) {
 	key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID)
 	val, found := c.ServerKeys.Get(key)
-	if found && val != nil {
-		if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok {
-			if !keyLookupResult.WasValidAt(timestamp, true) {
-				// The key wasn't valid at the requested timestamp so don't
-				// return it. The caller will have to work out what to do.
-				c.ServerKeys.Unset(key)
-				return gomatrixserverlib.PublicKeyLookupResult{}, false
-			}
-			return keyLookupResult, true
-		}
+	if found && !val.WasValidAt(timestamp, true) {
+		// The key wasn't valid at the requested timestamp so don't
+		// return it. The caller will have to work out what to do.
+		c.ServerKeys.Unset(key)
+		return gomatrixserverlib.PublicKeyLookupResult{}, false
 	}
-	return gomatrixserverlib.PublicKeyLookupResult{}, false
+	return val, found
 }
 
 func (c Caches) StoreServerKey(
diff --git a/internal/caching/cache_space_rooms.go b/internal/caching/cache_space_rooms.go
index 6d56cce5f..697f99269 100644
--- a/internal/caching/cache_space_rooms.go
+++ b/internal/caching/cache_space_rooms.go
@@ -1,31 +1,16 @@
 package caching
 
 import (
-	"time"
-
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-const (
-	SpaceSummaryRoomsCacheName       = "space_summary_rooms"
-	SpaceSummaryRoomsCacheMaxEntries = 100
-	SpaceSummaryRoomsCacheMutable    = true
-	SpaceSummaryRoomsCacheMaxAge     = time.Minute * 5
-)
-
 type SpaceSummaryRoomsCache interface {
 	GetSpaceSummary(roomID string) (r gomatrixserverlib.MSC2946SpacesResponse, ok bool)
 	StoreSpaceSummary(roomID string, r gomatrixserverlib.MSC2946SpacesResponse)
 }
 
 func (c Caches) GetSpaceSummary(roomID string) (r gomatrixserverlib.MSC2946SpacesResponse, ok bool) {
-	val, found := c.SpaceSummaryRooms.Get(roomID)
-	if found && val != nil {
-		if resp, ok := val.(gomatrixserverlib.MSC2946SpacesResponse); ok {
-			return resp, true
-		}
-	}
-	return r, false
+	return c.SpaceSummaryRooms.Get(roomID)
 }
 
 func (c Caches) StoreSpaceSummary(roomID string, r gomatrixserverlib.MSC2946SpacesResponse) {
diff --git a/internal/caching/caches.go b/internal/caching/caches.go
index 173e47e5b..14b232dd0 100644
--- a/internal/caching/caches.go
+++ b/internal/caching/caches.go
@@ -1,28 +1,52 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package caching
 
 import (
-	"time"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // Caches contains a set of references to caches. They may be
 // different implementations as long as they satisfy the Cache
 // interface.
 type Caches struct {
-	RoomVersions       Cache // RoomVersionCache
-	ServerKeys         Cache // ServerKeyCache
-	RoomServerRoomNIDs Cache // RoomServerNIDsCache
-	RoomServerRoomIDs  Cache // RoomServerNIDsCache
-	RoomInfos          Cache // RoomInfoCache
-	FederationEvents   Cache // FederationEventsCache
-	SpaceSummaryRooms  Cache // SpaceSummaryRoomsCache
-	LazyLoading        Cache // LazyLoadCache
+	RoomVersions       Cache[string, gomatrixserverlib.RoomVersion]           // room ID -> room version
+	ServerKeys         Cache[string, gomatrixserverlib.PublicKeyLookupResult] // server name -> server keys
+	RoomServerRoomNIDs Cache[string, types.RoomNID]                           // room ID -> room NID
+	RoomServerRoomIDs  Cache[int64, string]                                   // room NID -> room ID
+	RoomServerEvents   Cache[int64, *gomatrixserverlib.Event]                 // event NID -> event
+	RoomInfos          Cache[string, types.RoomInfo]                          // room ID -> room info
+	FederationPDUs     Cache[int64, *gomatrixserverlib.HeaderedEvent]         // queue NID -> PDU
+	FederationEDUs     Cache[int64, *gomatrixserverlib.EDU]                   // queue NID -> EDU
+	SpaceSummaryRooms  Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response
+	LazyLoading        Cache[lazyLoadingCacheKey, string]                     // composite key -> event ID
 }
 
 // Cache is the interface that an implementation must satisfy.
-type Cache interface {
-	Get(key string) (value interface{}, ok bool)
-	Set(key string, value interface{})
-	Unset(key string)
+type Cache[K keyable, T any] interface {
+	Get(key K) (value T, ok bool)
+	Set(key K, value T)
+	Unset(key K)
 }
 
-const CacheNoMaxAge = time.Duration(0)
+type keyable interface {
+	// from https://github.com/dgraph-io/ristretto/blob/8e850b710d6df0383c375ec6a7beae4ce48fc8d5/z/z.go#L34
+	uint64 | string | []byte | byte | int | int32 | uint32 | int64 | lazyLoadingCacheKey
+}
+
+type costable interface {
+	CacheCost() int
+}
diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go
deleted file mode 100644
index 594760892..000000000
--- a/internal/caching/impl_inmemorylru.go
+++ /dev/null
@@ -1,189 +0,0 @@
-package caching
-
-import (
-	"fmt"
-	"time"
-
-	lru "github.com/hashicorp/golang-lru"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
-	roomVersions, err := NewInMemoryLRUCachePartition(
-		RoomVersionCacheName,
-		RoomVersionCacheMutable,
-		RoomVersionCacheMaxEntries,
-		RoomVersionCacheMaxAge,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-	serverKeys, err := NewInMemoryLRUCachePartition(
-		ServerKeyCacheName,
-		ServerKeyCacheMutable,
-		ServerKeyCacheMaxEntries,
-		ServerKeyCacheMaxAge,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-	roomServerRoomIDs, err := NewInMemoryLRUCachePartition(
-		RoomServerRoomIDsCacheName,
-		RoomServerRoomIDsCacheMutable,
-		RoomServerRoomIDsCacheMaxEntries,
-		RoomServerRoomIDsCacheMaxAge,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-	roomInfos, err := NewInMemoryLRUCachePartition(
-		RoomInfoCacheName,
-		RoomInfoCacheMutable,
-		RoomInfoCacheMaxEntries,
-		RoomInfoCacheMaxAge,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-	federationEvents, err := NewInMemoryLRUCachePartition(
-		FederationEventCacheName,
-		FederationEventCacheMutable,
-		FederationEventCacheMaxEntries,
-		FederationEventCacheMaxAge,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-	spaceRooms, err := NewInMemoryLRUCachePartition(
-		SpaceSummaryRoomsCacheName,
-		SpaceSummaryRoomsCacheMutable,
-		SpaceSummaryRoomsCacheMaxEntries,
-		SpaceSummaryRoomsCacheMaxAge,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	lazyLoadCache, err := NewInMemoryLRUCachePartition(
-		LazyLoadCacheName,
-		LazyLoadCacheMutable,
-		LazyLoadCacheMaxEntries,
-		LazyLoadCacheMaxAge,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	go cacheCleaner(
-		roomVersions, serverKeys, roomServerRoomIDs,
-		roomInfos, federationEvents, spaceRooms, lazyLoadCache,
-	)
-	return &Caches{
-		RoomVersions:      roomVersions,
-		ServerKeys:        serverKeys,
-		RoomServerRoomIDs: roomServerRoomIDs,
-		RoomInfos:         roomInfos,
-		FederationEvents:  federationEvents,
-		SpaceSummaryRooms: spaceRooms,
-		LazyLoading:       lazyLoadCache,
-	}, nil
-}
-
-func cacheCleaner(caches ...*InMemoryLRUCachePartition) {
-	for {
-		time.Sleep(time.Minute)
-		for _, cache := range caches {
-			// Hold onto the last 10% of the cache entries, since
-			// otherwise a quiet period might cause us to evict all
-			// cache entries entirely.
-			if cache.lru.Len() > cache.maxEntries/10 {
-				cache.lru.RemoveOldest()
-			}
-		}
-	}
-}
-
-type InMemoryLRUCachePartition struct {
-	name       string
-	mutable    bool
-	maxEntries int
-	maxAge     time.Duration
-	lru        *lru.Cache
-}
-
-type inMemoryLRUCacheEntry struct {
-	value   interface{}
-	created time.Time
-}
-
-func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int, maxAge time.Duration, enablePrometheus bool) (*InMemoryLRUCachePartition, error) {
-	var err error
-	cache := InMemoryLRUCachePartition{
-		name:       name,
-		mutable:    mutable,
-		maxEntries: maxEntries,
-		maxAge:     maxAge,
-	}
-	cache.lru, err = lru.New(maxEntries)
-	if err != nil {
-		return nil, err
-	}
-	if enablePrometheus {
-		promauto.NewGaugeFunc(prometheus.GaugeOpts{
-			Namespace: "dendrite",
-			Subsystem: "caching_in_memory_lru",
-			Name:      name,
-		}, func() float64 {
-			return float64(cache.lru.Len())
-		})
-	}
-	return &cache, nil
-}
-
-func (c *InMemoryLRUCachePartition) Set(key string, value interface{}) {
-	if !c.mutable {
-		if peek, ok := c.lru.Peek(key); ok {
-			if entry, ok := peek.(*inMemoryLRUCacheEntry); ok && entry.value != value {
-				panic(fmt.Sprintf("invalid use of immutable cache tries to mutate existing value of %q", key))
-			}
-		}
-	}
-	c.lru.Add(key, &inMemoryLRUCacheEntry{
-		value:   value,
-		created: time.Now(),
-	})
-}
-
-func (c *InMemoryLRUCachePartition) Unset(key string) {
-	if !c.mutable {
-		panic(fmt.Sprintf("invalid use of immutable cache tries to unset value of %q", key))
-	}
-	c.lru.Remove(key)
-}
-
-func (c *InMemoryLRUCachePartition) Get(key string) (value interface{}, ok bool) {
-	v, ok := c.lru.Get(key)
-	if !ok {
-		return nil, false
-	}
-	entry, ok := v.(*inMemoryLRUCacheEntry)
-	switch {
-	case ok && c.maxAge == CacheNoMaxAge:
-		return entry.value, ok // There's no maximum age policy
-	case ok && time.Since(entry.created) < c.maxAge:
-		return entry.value, ok // The value for the key isn't stale
-	default:
-		// Either the key was found and it was stale, or the key
-		// wasn't found at all
-		c.lru.Remove(key)
-		return nil, false
-	}
-}
diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go
new file mode 100644
index 000000000..6d625b552
--- /dev/null
+++ b/internal/caching/impl_ristretto.go
@@ -0,0 +1,200 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caching
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+	"unsafe"
+
+	"github.com/dgraph-io/ristretto"
+	"github.com/dgraph-io/ristretto/z"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	roomVersionsCache byte = iota + 1
+	serverKeysCache
+	roomNIDsCache
+	roomIDsCache
+	roomEventsCache
+	roomInfosCache
+	federationPDUsCache
+	federationEDUsCache
+	spaceSummaryRoomsCache
+	lazyLoadingCache
+)
+
+func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enablePrometheus bool) *Caches {
+	cache, err := ristretto.NewCache(&ristretto.Config{
+		NumCounters: 1e5, // 10x number of expected cache items, affects bloom filter size, gives us room for 10,000 currently
+		BufferItems: 64,  // recommended by the ristretto godocs as a sane buffer size value
+		MaxCost:     int64(maxCost),
+		Metrics:     true,
+		KeyToHash: func(key interface{}) (uint64, uint64) {
+			return z.KeyToHash(key)
+		},
+	})
+	if err != nil {
+		panic(err)
+	}
+	if enablePrometheus {
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "dendrite",
+			Subsystem: "caching_ristretto",
+			Name:      "ratio",
+		}, func() float64 {
+			return float64(cache.Metrics.Ratio())
+		})
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "dendrite",
+			Subsystem: "caching_ristretto",
+			Name:      "cost",
+		}, func() float64 {
+			return float64(cache.Metrics.CostAdded() - cache.Metrics.CostEvicted())
+		})
+	}
+	return &Caches{
+		RoomVersions: &RistrettoCachePartition[string, gomatrixserverlib.RoomVersion]{ // room ID -> room version
+			cache:  cache,
+			Prefix: roomVersionsCache,
+			MaxAge: maxAge,
+		},
+		ServerKeys: &RistrettoCachePartition[string, gomatrixserverlib.PublicKeyLookupResult]{ // server name -> server keys
+			cache:   cache,
+			Prefix:  serverKeysCache,
+			Mutable: true,
+			MaxAge:  maxAge,
+		},
+		RoomServerRoomNIDs: &RistrettoCachePartition[string, types.RoomNID]{ // room ID -> room NID
+			cache:  cache,
+			Prefix: roomNIDsCache,
+			MaxAge: maxAge,
+		},
+		RoomServerRoomIDs: &RistrettoCachePartition[int64, string]{ // room NID -> room ID
+			cache:  cache,
+			Prefix: roomIDsCache,
+			MaxAge: maxAge,
+		},
+		RoomServerEvents: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.Event]{ // event NID -> event
+			&RistrettoCachePartition[int64, *gomatrixserverlib.Event]{
+				cache:  cache,
+				Prefix: roomEventsCache,
+				MaxAge: maxAge,
+			},
+		},
+		RoomInfos: &RistrettoCachePartition[string, types.RoomInfo]{ // room ID -> room info
+			cache:   cache,
+			Prefix:  roomInfosCache,
+			Mutable: true,
+			MaxAge:  maxAge,
+		},
+		FederationPDUs: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{ // queue NID -> PDU
+			&RistrettoCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{
+				cache:   cache,
+				Prefix:  federationPDUsCache,
+				Mutable: true,
+				MaxAge:  lesserOf(time.Hour/2, maxAge),
+			},
+		},
+		FederationEDUs: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.EDU]{ // queue NID -> EDU
+			&RistrettoCachePartition[int64, *gomatrixserverlib.EDU]{
+				cache:   cache,
+				Prefix:  federationEDUsCache,
+				Mutable: true,
+				MaxAge:  lesserOf(time.Hour/2, maxAge),
+			},
+		},
+		SpaceSummaryRooms: &RistrettoCachePartition[string, gomatrixserverlib.MSC2946SpacesResponse]{ // room ID -> space response
+			cache:   cache,
+			Prefix:  spaceSummaryRoomsCache,
+			Mutable: true,
+			MaxAge:  maxAge,
+		},
+		LazyLoading: &RistrettoCachePartition[lazyLoadingCacheKey, string]{ // composite key -> event ID
+			cache:   cache,
+			Prefix:  lazyLoadingCache,
+			Mutable: true,
+			MaxAge:  maxAge,
+		},
+	}
+}
+
+type RistrettoCostedCachePartition[k keyable, v costable] struct {
+	*RistrettoCachePartition[k, v]
+}
+
+func (c *RistrettoCostedCachePartition[K, V]) Set(key K, value V) {
+	cost := value.CacheCost()
+	c.setWithCost(key, value, int64(cost))
+}
+
+type RistrettoCachePartition[K keyable, V any] struct {
+	cache   *ristretto.Cache
+	Prefix  byte
+	Mutable bool
+	MaxAge  time.Duration
+}
+
+func (c *RistrettoCachePartition[K, V]) setWithCost(key K, value V, cost int64) {
+	bkey := fmt.Sprintf("%c%v", c.Prefix, key)
+	if !c.Mutable {
+		if v, ok := c.cache.Get(bkey); ok && v != nil && !reflect.DeepEqual(v, value) {
+			panic(fmt.Sprintf("invalid use of immutable cache tries to change value of %v from %v to %v", key, v, value))
+		}
+	}
+	c.cache.SetWithTTL(bkey, value, int64(len(bkey))+cost, c.MaxAge)
+}
+
+func (c *RistrettoCachePartition[K, V]) Set(key K, value V) {
+	var cost int64
+	if cv, ok := any(value).(string); ok {
+		cost = int64(len(cv))
+	} else {
+		cost = int64(unsafe.Sizeof(value))
+	}
+	c.setWithCost(key, value, cost)
+}
+
+func (c *RistrettoCachePartition[K, V]) Unset(key K) {
+	bkey := fmt.Sprintf("%c%v", c.Prefix, key)
+	if !c.Mutable {
+		panic(fmt.Sprintf("invalid use of immutable cache tries to unset value of %v", key))
+	}
+	c.cache.Del(bkey)
+}
+
+func (c *RistrettoCachePartition[K, V]) Get(key K) (value V, ok bool) {
+	bkey := fmt.Sprintf("%c%v", c.Prefix, key)
+	v, ok := c.cache.Get(bkey)
+	if !ok || v == nil {
+		var empty V
+		return empty, false
+	}
+	value, ok = v.(V)
+	return
+}
+
+func lesserOf(a, b time.Duration) time.Duration {
+	if a < b {
+		return a
+	}
+	return b
+}
diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go
index ee67a6daf..d96231963 100644
--- a/internal/eventutil/events.go
+++ b/internal/eventutil/events.go
@@ -170,20 +170,18 @@ func truncateAuthAndPrevEvents(auth, prev []gomatrixserverlib.EventReference) (
 
 // RedactEvent redacts the given event and sets the unsigned field appropriately. This should be used by
 // downstream components to the roomserver when an OutputTypeRedactedEvent occurs.
-func RedactEvent(redactionEvent, redactedEvent *gomatrixserverlib.Event) (*gomatrixserverlib.Event, error) {
+func RedactEvent(redactionEvent, redactedEvent *gomatrixserverlib.Event) error {
 	// sanity check
 	if redactionEvent.Type() != gomatrixserverlib.MRoomRedaction {
-		return nil, fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type())
+		return fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type())
 	}
-	r := redactedEvent.Redact()
-	err := r.SetUnsignedField("redacted_because", redactionEvent)
-	if err != nil {
-		return nil, err
+	redactedEvent.Redact()
+	if err := redactedEvent.SetUnsignedField("redacted_because", redactionEvent); err != nil {
+		return err
 	}
 	// NOTSPEC: sytest relies on this unspecced field existing :(
-	err = r.SetUnsignedField("redacted_by", redactionEvent.EventID())
-	if err != nil {
-		return nil, err
+	if err := redactedEvent.SetUnsignedField("redacted_by", redactionEvent.EventID()); err != nil {
+		return err
 	}
-	return r, nil
+	return nil
 }
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 743b1efe6..866670d7a 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -319,11 +319,9 @@ func (r *Inputer) processRoomEvent(
 
 	// if storing this event results in it being redacted then do so.
 	if !isRejected && redactedEventID == event.EventID() {
-		r, rerr := eventutil.RedactEvent(redactionEvent, event)
-		if rerr != nil {
+		if err = eventutil.RedactEvent(redactionEvent, event); err != nil {
 			return fmt.Errorf("eventutil.RedactEvent: %w", rerr)
 		}
-		event = r
 	}
 
 	// For outliers we can stop after we've stored the event itself as it
diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go
index 7c65f9eac..4708560ac 100644
--- a/roomserver/internal/input/input_test.go
+++ b/roomserver/internal/input/input_test.go
@@ -48,10 +48,6 @@ func TestSingleTransactionOnInput(t *testing.T) {
 		Kind:  api.KindOutlier, // don't panic if we generate an output event
 		Event: event.Headered(gomatrixserverlib.RoomVersionV6),
 	}
-	cache, err := caching.NewInMemoryLRUCache(false)
-	if err != nil {
-		t.Fatal(err)
-	}
 	db, err := storage.Open(
 		nil,
 		&config.DatabaseOptions{
@@ -59,7 +55,7 @@ func TestSingleTransactionOnInput(t *testing.T) {
 			MaxOpenConnections: 1,
 			MaxIdleConnections: 1,
 		},
-		cache,
+		caching.NewRistrettoCache(8*1024*1024, time.Hour, false),
 	)
 	if err != nil {
 		t.Logf("PostgreSQL not available (%s), skipping", err)
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 9eddca733..3f98fbc24 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -593,12 +593,11 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs
 		// redacted, which we don't care about since we aren't returning it in this backfill.
 		if redactedEventID == ev.EventID() {
 			eventToRedact := ev.Unwrap()
-			redactedEvent, err := eventutil.RedactEvent(redactionEvent, eventToRedact)
-			if err != nil {
+			if err := eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil {
 				logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to redact event")
 				continue
 			}
-			ev = redactedEvent.Headered(ev.RoomVersion)
+			ev = eventToRedact.Headered(ev.RoomVersion)
 			events[j] = ev
 		}
 		backfilledEventMap[ev.EventID()] = types.Event{
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index 91f271652..d1d24b099 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -1027,7 +1027,7 @@ func (v *StateResolution) loadStateEvents(
 
 	result := make([]*gomatrixserverlib.Event, 0, len(entries))
 	eventEntries := make([]types.StateEntry, 0, len(entries))
-	eventNIDs := make([]types.EventNID, 0, len(entries))
+	eventNIDs := make(types.EventNIDs, 0, len(entries))
 	for _, entry := range entries {
 		if e, ok := v.events[entry.EventNID]; ok {
 			result = append(result, e)
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index ba937ba30..692af1f6c 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -439,8 +439,18 @@ func (d *Database) Events(
 }
 
 func (d *Database) events(
-	ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID,
+	ctx context.Context, txn *sql.Tx, inputEventNIDs types.EventNIDs,
 ) ([]types.Event, error) {
+	sort.Sort(inputEventNIDs)
+	events := make(map[types.EventNID]*gomatrixserverlib.Event, len(inputEventNIDs))
+	eventNIDs := make([]types.EventNID, 0, len(inputEventNIDs))
+	for _, nid := range inputEventNIDs {
+		if event, ok := d.Cache.GetRoomServerEvent(nid); ok && event != nil {
+			events[nid] = event
+		} else {
+			eventNIDs = append(eventNIDs, nid)
+		}
+	}
 	eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, txn, eventNIDs)
 	if err != nil {
 		return nil, err
@@ -476,18 +486,29 @@ func (d *Database) events(
 	for n, v := range dbRoomVersions {
 		roomVersions[n] = v
 	}
-	results := make([]types.Event, len(eventJSONs))
-	for i, eventJSON := range eventJSONs {
-		result := &results[i]
-		result.EventNID = eventJSON.EventNID
-		roomNID := roomNIDs[result.EventNID]
+	for _, eventJSON := range eventJSONs {
+		roomNID := roomNIDs[eventJSON.EventNID]
 		roomVersion := roomVersions[roomNID]
-		result.Event, err = gomatrixserverlib.NewEventFromTrustedJSONWithEventID(
+		events[eventJSON.EventNID], err = gomatrixserverlib.NewEventFromTrustedJSONWithEventID(
 			eventIDs[eventJSON.EventNID], eventJSON.EventJSON, false, roomVersion,
 		)
 		if err != nil {
 			return nil, err
 		}
+		if event := events[eventJSON.EventNID]; event != nil {
+			d.Cache.StoreRoomServerEvent(eventJSON.EventNID, event)
+		}
+	}
+	results := make([]types.Event, 0, len(inputEventNIDs))
+	for _, nid := range inputEventNIDs {
+		event, ok := events[nid]
+		if !ok || event == nil {
+			return nil, fmt.Errorf("event %d missing", nid)
+		}
+		results = append(results, types.Event{
+			EventNID: nid,
+			Event:    event,
+		})
 	}
 	if !redactionsArePermanent {
 		d.applyRedactions(results)
@@ -854,7 +875,7 @@ func (d *Database) handleRedactions(
 
 	// mark the event as redacted
 	if redactionsArePermanent {
-		redactedEvent.Event = redactedEvent.Redact()
+		redactedEvent.Redact()
 	}
 
 	err = redactedEvent.SetUnsignedField("redacted_because", redactionEvent)
@@ -926,7 +947,7 @@ func (d *Database) loadRedactionPair(
 func (d *Database) applyRedactions(events []types.Event) {
 	for i := range events {
 		if result := gjson.GetBytes(events[i].Unsigned(), "redacted_because"); result.Exists() {
-			events[i].Event = events[i].Redact()
+			events[i].Redact()
 		}
 	}
 }
diff --git a/setup/base/base.go b/setup/base/base.go
index 5cbd7da9c..93ab87de1 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -161,11 +161,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		}
 	}
 
-	cache, err := caching.NewInMemoryLRUCache(enableMetrics)
-	if err != nil {
-		logrus.WithError(err).Warnf("Failed to create cache")
-	}
-
 	var dnsCache *gomatrixserverlib.DNSCache
 	if cfg.Global.DNSCache.Enabled {
 		dnsCache = gomatrixserverlib.NewDNSCache(
@@ -233,7 +228,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		UseHTTPAPIs:            useHTTPAPIs,
 		tracerCloser:           closer,
 		Cfg:                    cfg,
-		Caches:                 cache,
+		Caches:                 caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, enableMetrics),
 		DNSCache:               dnsCache,
 		PublicClientAPIMux:     mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicClientPathPrefix).Subrouter().UseEncodedPath(),
 		PublicFederationAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath(),
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index 9d4c1485e..ac1380a4e 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -2,6 +2,8 @@ package config
 
 import (
 	"math/rand"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -73,6 +75,9 @@ type Global struct {
 
 	// ReportStats configures opt-in anonymous stats reporting.
 	ReportStats ReportStats `yaml:"report_stats"`
+
+	// Configuration for the caches.
+	Cache Cache `yaml:"cache"`
 }
 
 func (c *Global) Defaults(generate bool) {
@@ -90,6 +95,7 @@ func (c *Global) Defaults(generate bool) {
 	c.Sentry.Defaults()
 	c.ServerNotices.Defaults(generate)
 	c.ReportStats.Defaults()
+	c.Cache.Defaults(generate)
 }
 
 func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
@@ -102,6 +108,7 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	c.DNSCache.Verify(configErrs, isMonolith)
 	c.ServerNotices.Verify(configErrs, isMonolith)
 	c.ReportStats.Verify(configErrs, isMonolith)
+	c.Cache.Verify(configErrs, isMonolith)
 }
 
 type OldVerifyKeys struct {
@@ -168,6 +175,20 @@ func (c *ServerNotices) Defaults(generate bool) {
 
 func (c *ServerNotices) Verify(errors *ConfigErrors, isMonolith bool) {}
 
+type Cache struct {
+	EstimatedMaxSize DataUnit      `yaml:"max_size_estimated"`
+	MaxAge           time.Duration `yaml:"max_age"`
+}
+
+func (c *Cache) Defaults(generate bool) {
+	c.EstimatedMaxSize = 1024 * 1024 * 1024 // 1GB
+	c.MaxAge = time.Hour
+}
+
+func (c *Cache) Verify(errors *ConfigErrors, isMonolith bool) {
+	checkPositive(errors, "max_size_estimated", int64(c.EstimatedMaxSize))
+}
+
 // ReportStats configures opt-in anonymous stats reporting.
 type ReportStats struct {
 	// Enabled configures anonymous usage stats of the server
@@ -268,3 +289,28 @@ type PresenceOptions struct {
 	// Whether outbound presence events are allowed
 	EnableOutbound bool `yaml:"enable_outbound"`
 }
+
+type DataUnit int64
+
+func (d *DataUnit) UnmarshalText(text []byte) error {
+	var magnitude float64
+	s := strings.ToLower(string(text))
+	switch {
+	case strings.HasSuffix(s, "tb"):
+		s, magnitude = s[:len(s)-2], 1024*1024*1024*1024
+	case strings.HasSuffix(s, "gb"):
+		s, magnitude = s[:len(s)-2], 1024*1024*1024
+	case strings.HasSuffix(s, "mb"):
+		s, magnitude = s[:len(s)-2], 1024*1024
+	case strings.HasSuffix(s, "kb"):
+		s, magnitude = s[:len(s)-2], 1024
+	default:
+		magnitude = 1
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return err
+	}
+	*d = DataUnit(v * magnitude)
+	return nil
+}
diff --git a/setup/config/config_test.go b/setup/config/config_test.go
index cbc57ad18..b9b1e7bb5 100644
--- a/setup/config/config_test.go
+++ b/setup/config/config_test.go
@@ -17,6 +17,8 @@ package config
 import (
 	"fmt"
 	"testing"
+
+	"gopkg.in/yaml.v2"
 )
 
 func TestLoadConfigRelative(t *testing.T) {
@@ -268,3 +270,22 @@ n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU
 ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4
 -----END CERTIFICATE-----
 `
+
+func TestUnmarshalDataUnit(t *testing.T) {
+	target := struct {
+		Got DataUnit `yaml:"value"`
+	}{}
+	for input, expect := range map[string]DataUnit{
+		"value: 0.6tb": 659706976665,
+		"value: 1.2gb": 1288490188,
+		"value: 256mb": 268435456,
+		"value: 128kb": 131072,
+		"value: 128":   128,
+	} {
+		if err := yaml.Unmarshal([]byte(input), &target); err != nil {
+			t.Fatal(err)
+		} else if target.Got != expect {
+			t.Fatalf("expected value %d but got %d", expect, target.Got)
+		}
+	}
+}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index ec5edd355..76114aff8 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -545,12 +545,11 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda
 	}
 	eventToRedact := redactedEvents[0].Unwrap()
 	redactionEvent := redactedBecause.Unwrap()
-	ev, err := eventutil.RedactEvent(redactionEvent, eventToRedact)
-	if err != nil {
+	if err = eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil {
 		return err
 	}
 
-	newEvent := ev.Headered(redactedBecause.RoomVersion)
+	newEvent := eventToRedact.Headered(redactedBecause.RoomVersion)
 	err = d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
 		return d.OutputEvents.UpdateEventJSON(ctx, newEvent)
 	})

From 09f0ff14c88fec2ae5b3db79054378a8ced255a9 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 12 Jul 2022 08:23:58 +0200
Subject: [PATCH 199/391] Minor SendToDevice fix (#2565)

* Avoid unnecessary marshalling if sending to the local server

* Fix ordering of ToDevice messages

* Revive SendToDevice test
---
 federationapi/consumers/sendtodevice.go       |   5 +
 .../storage/postgres/send_to_device_table.go  |  10 +-
 .../storage/sqlite3/send_to_device_table.go   |   8 +-
 syncapi/storage/storage_test.go               | 191 +++++++++++-------
 4 files changed, 129 insertions(+), 85 deletions(-)

diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go
index 84c9f620d..f99a895e0 100644
--- a/federationapi/consumers/sendtodevice.go
+++ b/federationapi/consumers/sendtodevice.go
@@ -95,6 +95,11 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msg *nats.Ms
 		return true
 	}
 
+	// The SyncAPI is already handling sendToDevice for the local server
+	if destServerName == t.ServerName {
+		return true
+	}
+
 	// Pack the EDU and marshal it
 	edu := &gomatrixserverlib.EDU{
 		Type:   gomatrixserverlib.MDirectToDevice,
diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go
index 47c1cdaed..96d6844fd 100644
--- a/syncapi/storage/postgres/send_to_device_table.go
+++ b/syncapi/storage/postgres/send_to_device_table.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/sirupsen/logrus"
 )
 
 const sendToDeviceSchema = `
@@ -51,7 +52,7 @@ const selectSendToDeviceMessagesSQL = `
 	SELECT id, user_id, device_id, content
 	  FROM syncapi_send_to_device
 	  WHERE user_id = $1 AND device_id = $2 AND id > $3 AND id <= $4
-	  ORDER BY id DESC
+	  ORDER BY id ASC
 `
 
 const deleteSendToDeviceMessagesSQL = `
@@ -112,17 +113,18 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages(
 		if err = rows.Scan(&id, &userID, &deviceID, &content); err != nil {
 			return
 		}
-		if id > lastPos {
-			lastPos = id
-		}
 		event := types.SendToDeviceEvent{
 			ID:       id,
 			UserID:   userID,
 			DeviceID: deviceID,
 		}
 		if err = json.Unmarshal([]byte(content), &event.SendToDeviceEvent); err != nil {
+			logrus.WithError(err).Errorf("Failed to unmarshal send-to-device message")
 			continue
 		}
+		if id > lastPos {
+			lastPos = id
+		}
 		events = append(events, event)
 	}
 	if lastPos == 0 {
diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go
index 0b1d5bbf2..5285acbe6 100644
--- a/syncapi/storage/sqlite3/send_to_device_table.go
+++ b/syncapi/storage/sqlite3/send_to_device_table.go
@@ -49,7 +49,7 @@ const selectSendToDeviceMessagesSQL = `
 	SELECT id, user_id, device_id, content
 	  FROM syncapi_send_to_device
 	  WHERE user_id = $1 AND device_id = $2 AND id > $3 AND id <= $4
-	  ORDER BY id DESC
+	  ORDER BY id ASC
 `
 
 const deleteSendToDeviceMessagesSQL = `
@@ -120,9 +120,6 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages(
 			logrus.WithError(err).Errorf("Failed to retrieve send-to-device message")
 			return
 		}
-		if id > lastPos {
-			lastPos = id
-		}
 		event := types.SendToDeviceEvent{
 			ID:       id,
 			UserID:   userID,
@@ -132,6 +129,9 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages(
 			logrus.WithError(err).Errorf("Failed to unmarshal send-to-device message")
 			continue
 		}
+		if id > lastPos {
+			lastPos = id
+		}
 		events = append(events, event)
 	}
 	if lastPos == 0 {
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index 563c92e34..c74151700 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -1,7 +1,9 @@
 package storage_test
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"testing"
@@ -394,90 +396,125 @@ func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) {
 		from = topologyTokenBefore(t, db, paginatedEvents[len(paginatedEvents)-1].EventID())
 	}
 }
+*/
 
 func TestSendToDeviceBehaviour(t *testing.T) {
-	//t.Parallel()
-	db := MustCreateDatabase(t)
+	t.Parallel()
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	deviceID := "one"
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := MustCreateDatabase(t, dbType)
+		defer close()
+		// At this point there should be no messages. We haven't sent anything
+		// yet.
+		_, events, err := db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(events) != 0 {
+			t.Fatal("first call should have no updates")
+		}
 
-	// At this point there should be no messages. We haven't sent anything
-	// yet.
-	_, events, updates, deletions, err := db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(events) != 0 || len(updates) != 0 || len(deletions) != 0 {
-		t.Fatal("first call should have no updates")
-	}
-	err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{})
-	if err != nil {
-		return
-	}
+		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, 100)
+		if err != nil {
+			return
+		}
 
-	// Try sending a message.
-	streamPos, err := db.StoreNewSendForDeviceMessage(ctx, "alice", "one", gomatrixserverlib.SendToDeviceEvent{
-		Sender:  "bob",
-		Type:    "m.type",
-		Content: json.RawMessage("{}"),
+		// Try sending a message.
+		streamPos, err := db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{
+			Sender:  bob.ID,
+			Type:    "m.type",
+			Content: json.RawMessage("{}"),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// At this point we should get exactly one message. We're sending the sync position
+		// that we were given from the update and the send-to-device update will be updated
+		// in the database to reflect that this was the sync position we sent the message at.
+		streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if count := len(events); count != 1 {
+			t.Fatalf("second call should have one update, got %d", count)
+		}
+		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos)
+		if err != nil {
+			return
+		}
+
+		// At this point we should still have one message because we haven't progressed the
+		// sync position yet. This is equivalent to the client failing to /sync and retrying
+		// with the same position.
+		streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(events) != 1 {
+			t.Fatal("third call should have one update still")
+		}
+		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos+1)
+		if err != nil {
+			return
+		}
+
+		// At this point we should now have no updates, because we've progressed the sync
+		// position. Therefore the update from before will not be sent again.
+		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos+1, streamPos+2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(events) != 0 {
+			t.Fatal("fourth call should have no updates")
+		}
+		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos+1)
+		if err != nil {
+			return
+		}
+
+		// At this point we should still have no updates, because no new updates have been
+		// sent.
+		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(events) != 0 {
+			t.Fatal("fifth call should have no updates")
+		}
+
+		// Send some more messages and verify the ordering is correct ("in order of arrival")
+		var lastPos types.StreamPosition = 0
+		for i := 0; i < 10; i++ {
+			streamPos, err = db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{
+				Sender:  bob.ID,
+				Type:    "m.type",
+				Content: json.RawMessage(fmt.Sprintf(`{ "count": %d }`, i)),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			lastPos = streamPos
+		}
+
+		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, lastPos)
+		if err != nil {
+			t.Fatalf("unable to get events: %v", err)
+		}
+
+		for i := 0; i < 10; i++ {
+			want := json.RawMessage(fmt.Sprintf(`{"count":%d}`, i))
+			got := events[i].Content
+			if !bytes.Equal(got, want) {
+				t.Fatalf("messages are out of order\nwant: %s\ngot: %s", string(want), string(got))
+			}
+		}
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// At this point we should get exactly one message. We're sending the sync position
-	// that we were given from the update and the send-to-device update will be updated
-	// in the database to reflect that this was the sync position we sent the message at.
-	_, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(events) != 1 || len(updates) != 1 || len(deletions) != 0 {
-		t.Fatal("second call should have one update")
-	}
-	err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{SendToDevicePosition: streamPos})
-	if err != nil {
-		return
-	}
-
-	// At this point we should still have one message because we haven't progressed the
-	// sync position yet. This is equivalent to the client failing to /sync and retrying
-	// with the same position.
-	_, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(events) != 1 || len(updates) != 0 || len(deletions) != 0 {
-		t.Fatal("third call should have one update still")
-	}
-	err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{SendToDevicePosition: streamPos})
-	if err != nil {
-		return
-	}
-
-	// At this point we should now have no updates, because we've progressed the sync
-	// position. Therefore the update from before will not be sent again.
-	_, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos + 1})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(events) != 0 || len(updates) != 0 || len(deletions) != 1 {
-		t.Fatal("fourth call should have no updates")
-	}
-	err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.StreamingToken{SendToDevicePosition: streamPos + 1})
-	if err != nil {
-		return
-	}
-
-	// At this point we should still have no updates, because no new updates have been
-	// sent.
-	_, events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.StreamingToken{SendToDevicePosition: streamPos + 2})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(events) != 0 || len(updates) != 0 || len(deletions) != 0 {
-		t.Fatal("fifth call should have no updates")
-	}
 }
 
+/*
 func TestInviteBehaviour(t *testing.T) {
 	db := MustCreateDatabase(t)
 	inviteRoom1 := "!inviteRoom1:somewhere"

From 9cd8e9d4b97c6cff66665f57065a5bb8b7144045 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 13 Jul 2022 09:32:03 +0100
Subject: [PATCH 200/391] Use `/v3` to request media from remote servers
 (update to matrix-org/gomatrixserverlib#324)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 2a2a037c1..1e62e1d93 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46
 	github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 98549f702..86656ccd5 100644
--- a/go.sum
+++ b/go.sum
@@ -341,8 +341,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c h1:mt30TDK8kXKV+nCmVfnqoXsh842N+74kvZw7DXuS/JQ=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220711125303-3bb2e997a44c/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46 h1:5X/kXY3nwqKOwwrE9tnMKrjbsi3PHigQYvrvDBSntO8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From a1f9b02edfd56809eaedd04fd233b1828085e190 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 13 Jul 2022 10:13:34 +0100
Subject: [PATCH 201/391] Pointerise `types.RoomInfo` in the cache so we can
 update it in-place in the latest events updater

---
 internal/caching/cache_roominfo.go        |  8 ++++----
 internal/caching/caches.go                |  2 +-
 internal/caching/impl_ristretto.go        |  2 +-
 roomserver/storage/shared/room_updater.go | 13 ++++++-------
 roomserver/storage/shared/storage.go      |  6 +++---
 5 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/internal/caching/cache_roominfo.go b/internal/caching/cache_roominfo.go
index d03a61077..5dfed3c85 100644
--- a/internal/caching/cache_roominfo.go
+++ b/internal/caching/cache_roominfo.go
@@ -16,18 +16,18 @@ import (
 // a room Info cache. It must only be used from the roomserver only
 // It is not safe for use from other components.
 type RoomInfoCache interface {
-	GetRoomInfo(roomID string) (roomInfo types.RoomInfo, ok bool)
-	StoreRoomInfo(roomID string, roomInfo types.RoomInfo)
+	GetRoomInfo(roomID string) (roomInfo *types.RoomInfo, ok bool)
+	StoreRoomInfo(roomID string, roomInfo *types.RoomInfo)
 }
 
 // GetRoomInfo must only be called from the roomserver only. It is not
 // safe for use from other components.
-func (c Caches) GetRoomInfo(roomID string) (types.RoomInfo, bool) {
+func (c Caches) GetRoomInfo(roomID string) (*types.RoomInfo, bool) {
 	return c.RoomInfos.Get(roomID)
 }
 
 // StoreRoomInfo must only be called from the roomserver only. It is not
 // safe for use from other components.
-func (c Caches) StoreRoomInfo(roomID string, roomInfo types.RoomInfo) {
+func (c Caches) StoreRoomInfo(roomID string, roomInfo *types.RoomInfo) {
 	c.RoomInfos.Set(roomID, roomInfo)
 }
diff --git a/internal/caching/caches.go b/internal/caching/caches.go
index 14b232dd0..e7914ce7d 100644
--- a/internal/caching/caches.go
+++ b/internal/caching/caches.go
@@ -28,7 +28,7 @@ type Caches struct {
 	RoomServerRoomNIDs Cache[string, types.RoomNID]                           // room ID -> room NID
 	RoomServerRoomIDs  Cache[int64, string]                                   // room NID -> room ID
 	RoomServerEvents   Cache[int64, *gomatrixserverlib.Event]                 // event NID -> event
-	RoomInfos          Cache[string, types.RoomInfo]                          // room ID -> room info
+	RoomInfos          Cache[string, *types.RoomInfo]                         // room ID -> room info
 	FederationPDUs     Cache[int64, *gomatrixserverlib.HeaderedEvent]         // queue NID -> PDU
 	FederationEDUs     Cache[int64, *gomatrixserverlib.EDU]                   // queue NID -> EDU
 	SpaceSummaryRooms  Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response
diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go
index 6d625b552..677218b5e 100644
--- a/internal/caching/impl_ristretto.go
+++ b/internal/caching/impl_ristretto.go
@@ -100,7 +100,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm
 				MaxAge: maxAge,
 			},
 		},
-		RoomInfos: &RistrettoCachePartition[string, types.RoomInfo]{ // room ID -> room info
+		RoomInfos: &RistrettoCachePartition[string, *types.RoomInfo]{ // room ID -> room info
 			cache:   cache,
 			Prefix:  roomInfosCache,
 			Mutable: true,
diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go
index 8f4e011bf..c35ac653c 100644
--- a/roomserver/storage/shared/room_updater.go
+++ b/roomserver/storage/shared/room_updater.go
@@ -225,13 +225,12 @@ func (u *RoomUpdater) SetLatestEvents(
 		if err := u.d.RoomsTable.UpdateLatestEventNIDs(u.ctx, txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID); err != nil {
 			return fmt.Errorf("u.d.RoomsTable.updateLatestEventNIDs: %w", err)
 		}
-		if roomID, ok := u.d.Cache.GetRoomServerRoomID(roomNID); ok {
-			if roomInfo, ok := u.d.Cache.GetRoomInfo(roomID); ok {
-				roomInfo.StateSnapshotNID = currentStateSnapshotNID
-				roomInfo.IsStub = false
-				u.d.Cache.StoreRoomInfo(roomID, roomInfo)
-			}
-		}
+
+		// Since it's entirely possible that this types.RoomInfo came from the
+		// cache, we should make sure to update that entry so that the next run
+		// works from live data.
+		u.roomInfo.StateSnapshotNID = currentStateSnapshotNID
+		u.roomInfo.IsStub = false
 		return nil
 	})
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 692af1f6c..d8d5f67c8 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -139,13 +139,13 @@ func (d *Database) RoomInfo(ctx context.Context, roomID string) (*types.RoomInfo
 }
 
 func (d *Database) roomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
-	if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok {
-		return &roomInfo, nil
+	if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok && roomInfo != nil {
+		return roomInfo, nil
 	}
 	roomInfo, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID)
 	if err == nil && roomInfo != nil {
 		d.Cache.StoreRoomServerRoomID(roomInfo.RoomNID, roomID)
-		d.Cache.StoreRoomInfo(roomID, *roomInfo)
+		d.Cache.StoreRoomInfo(roomID, roomInfo)
 	}
 	return roomInfo, err
 }

From 950659320eba4d196032a6d63bcefcfbe1d8eaf9 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 14 Jul 2022 10:09:26 +0100
Subject: [PATCH 202/391] Add a Troubleshooting page

---
 docs/administration/5_troubleshooting.md | 81 ++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 docs/administration/5_troubleshooting.md

diff --git a/docs/administration/5_troubleshooting.md b/docs/administration/5_troubleshooting.md
new file mode 100644
index 000000000..14df2e3fb
--- /dev/null
+++ b/docs/administration/5_troubleshooting.md
@@ -0,0 +1,81 @@
+---
+title: Troubleshooting
+parent: Administration
+permalink: /administration/troubleshooting
+---
+
+# Troubleshooting
+
+If your Dendrite installation is acting strangely, there are a few things you should
+check before seeking help.
+
+## 1. Logs
+
+Dendrite, by default, will log all warnings and errors to stdout, in addition to any
+other locations configured in the `dendrite.yaml` configuration file. Often there will
+be clues in the logs.
+
+You can increase this log level to the more verbose `debug` level if necessary by adding
+this to the config and restarting Dendrite:
+
+```
+logging:
+- type: std
+  level: debug
+```
+
+Look specifically for lines that contain `level=error` or `level=warning`.
+
+## 2. Federation tester
+
+If you are experiencing problems federating with other homeservers, you should check
+that the [Federation Tester](https://federationtester.matrix.org) is passing for your
+server.
+
+Common reasons that it may not pass include:
+
+1. Incorrect DNS configuration;
+2. Misconfigured DNS SRV entries or well-known files;
+3. Invalid TLS/SSL certificates;
+4. Reverse proxy configuration issues (if applicable).
+
+Correct any errors if shown and re-run the federation tester to check the results.
+
+## 3. System time
+
+Matrix relies heavily on TLS which requires the system time to be correct. If the clock
+drifts then you may find that federation no works reliably (or at all) and clients may
+struggle to connect to your Dendrite server.
+
+Ensure that your system time is correct and consider syncing to a reliable NTP source.
+
+## 4. Database connections
+
+If you are using the PostgreSQL database, you should ensure that Dendrite's configured
+number of database connections does not exceed the maximum allowed by PostgreSQL.
+
+Open your `postgresql.conf` configuration file and check the value of `max_connections`
+(which is typically `100` by default). Then open your `dendrite.yaml` configuration file
+and ensure that:
+
+1. If you are using the `global.database` section, that `max_open_conns` does not exceed
+   that number;
+2. If you are **not** using the `global.database` section, that the sum total of all
+   `max_open_conns` across all `database` blocks does not exceed that number.
+
+## 5. File descriptors
+
+Dendrite requires a sufficient number of file descriptors for every connection it makes
+to a remote server, every connection to the database engine and every file it is reading
+or writing to at a given time (media, logs etc). We recommend ensuring that the limit is
+no lower than 65535 for Dendrite.
+
+Dendrite will check at startup if there are a sufficient number of available descriptors.
+If there aren't, you will see a log lines like this:
+
+```
+level=warning msg="IMPORTANT: Process file descriptor limit is currently 65535, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
+```
+
+Follow the [Optimisation](../installation/10_optimisation.md) instructions to correct the
+available number of file descriptors.

From 69c86295f70b6756913df0bba37e30e0b3a6070e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 14 Jul 2022 13:51:00 +0100
Subject: [PATCH 203/391] Update `sytest-whitelist`

---
 sytest-whitelist | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sytest-whitelist b/sytest-whitelist
index ea25c75d0..2a145291f 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -719,4 +719,5 @@ registration is idempotent, with username specified
 Setting state twice is idempotent
 Joining room twice is idempotent
 Inbound federation can return missing events for shared visibility
-Inbound federation ignores redactions from invalid servers room > v3
\ No newline at end of file
+Inbound federation ignores redactions from invalid servers room > v3
+Newly joined room includes presence in incremental sync
\ No newline at end of file

From 90bf01d8b107375d45aee9de79c1760a0363b189 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 15 Jul 2022 16:25:26 +0100
Subject: [PATCH 204/391] Use sync API database in `filterSharedUsers` (#2572)

* Add function to the sync API storage package for filtering shared users

* Use the database instead of asking the RS API

* Fix unit tests

* Fix map handling in `filterSharedUsers`
---
 syncapi/internal/keychange.go                 | 33 +++++++++---------
 syncapi/internal/keychange_test.go            | 31 +++++++++++++----
 syncapi/storage/interface.go                  |  7 ++++
 .../postgres/current_room_state_table.go      | 30 ++++++++++++++++
 syncapi/storage/shared/syncserver.go          |  4 +++
 .../sqlite3/current_room_state_table.go       | 34 ++++++++++++++++++-
 syncapi/storage/tables/interface.go           |  2 ++
 syncapi/streams/stream_devicelist.go          |  2 +-
 syncapi/sync/requestpool.go                   |  2 +-
 9 files changed, 118 insertions(+), 27 deletions(-)

diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go
index d96718d20..03df9285c 100644
--- a/syncapi/internal/keychange.go
+++ b/syncapi/internal/keychange.go
@@ -21,6 +21,7 @@ import (
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	keytypes "github.com/matrix-org/dendrite/keyserver/types"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -46,7 +47,7 @@ func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, devi
 // was filled in, else false if there are no new device list changes because there is nothing to catch up on. The response MUST
 // be already filled in with join/leave information.
 func DeviceListCatchup(
-	ctx context.Context, keyAPI keyapi.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI,
+	ctx context.Context, db storage.SharedUsers, keyAPI keyapi.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI,
 	userID string, res *types.Response, from, to types.StreamPosition,
 ) (newPos types.StreamPosition, hasNew bool, err error) {
 
@@ -93,7 +94,7 @@ func DeviceListCatchup(
 	queryRes.UserIDs = append(queryRes.UserIDs, leaveUserIDs...)
 	queryRes.UserIDs = util.UniqueStrings(queryRes.UserIDs)
 	var sharedUsersMap map[string]int
-	sharedUsersMap, queryRes.UserIDs = filterSharedUsers(ctx, rsAPI, userID, queryRes.UserIDs)
+	sharedUsersMap, queryRes.UserIDs = filterSharedUsers(ctx, db, userID, queryRes.UserIDs)
 	util.GetLogger(ctx).Debugf(
 		"QueryKeyChanges request off=%d,to=%d response off=%d uids=%v",
 		offset, toOffset, queryRes.Offset, queryRes.UserIDs,
@@ -215,30 +216,28 @@ func TrackChangedUsers(
 	return changed, left, nil
 }
 
+// filterSharedUsers takes a list of remote users whose keys have changed and filters
+// it down to include only users who the requesting user shares a room with.
 func filterSharedUsers(
-	ctx context.Context, rsAPI roomserverAPI.SyncRoomserverAPI, userID string, usersWithChangedKeys []string,
+	ctx context.Context, db storage.SharedUsers, userID string, usersWithChangedKeys []string,
 ) (map[string]int, []string) {
-	var result []string
-	var sharedUsersRes roomserverAPI.QuerySharedUsersResponse
-	err := rsAPI.QuerySharedUsers(ctx, &roomserverAPI.QuerySharedUsersRequest{
-		UserID:       userID,
-		OtherUserIDs: usersWithChangedKeys,
-	}, &sharedUsersRes)
+	sharedUsersMap := make(map[string]int, len(usersWithChangedKeys))
+	for _, userID := range usersWithChangedKeys {
+		sharedUsersMap[userID] = 0
+	}
+	sharedUsers, err := db.SharedUsers(ctx, userID, usersWithChangedKeys)
 	if err != nil {
 		// default to all users so we do needless queries rather than miss some important device update
 		return nil, usersWithChangedKeys
 	}
+	for _, userID := range sharedUsers {
+		sharedUsersMap[userID]++
+	}
 	// We forcibly put ourselves in this list because we should be notified about our own device updates
 	// and if we are in 0 rooms then we don't technically share any room with ourselves so we wouldn't
 	// be notified about key changes.
-	sharedUsersRes.UserIDsToCount[userID] = 1
-
-	for _, uid := range usersWithChangedKeys {
-		if sharedUsersRes.UserIDsToCount[uid] > 0 {
-			result = append(result, uid)
-		}
-	}
-	return sharedUsersRes.UserIDsToCount, result
+	sharedUsersMap[userID] = 1
+	return sharedUsersMap, sharedUsers
 }
 
 func joinedRooms(res *types.Response, userID string) []string {
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 219b35e2c..79ed440e7 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 )
 
 var (
@@ -105,6 +106,22 @@ func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.Query
 	return nil
 }
 
+// This is actually a database function, but seeing as we track the state inside the
+// *mockRoomserverAPI, we'll just comply with the interface here instead.
+func (s *mockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) {
+	commonUsers := []string{}
+	for _, members := range s.roomIDToJoinedMembers {
+		for _, member := range members {
+			for _, userID := range otherUserIDs {
+				if member == userID {
+					commonUsers = append(commonUsers, userID)
+				}
+			}
+		}
+	}
+	return util.UniqueStrings(commonUsers), nil
+}
+
 type wantCatchup struct {
 	hasNew  bool
 	changed []string
@@ -178,7 +195,7 @@ func TestKeyChangeCatchupOnJoinShareNewUser(t *testing.T) {
 			"!another:room": {syncingUser},
 		},
 	}
-	_, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
+	_, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
 	if err != nil {
 		t.Fatalf("DeviceListCatchup returned an error: %s", err)
 	}
@@ -201,7 +218,7 @@ func TestKeyChangeCatchupOnLeaveShareLeftUser(t *testing.T) {
 			"!another:room": {syncingUser},
 		},
 	}
-	_, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
+	_, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
 	if err != nil {
 		t.Fatalf("DeviceListCatchup returned an error: %s", err)
 	}
@@ -224,7 +241,7 @@ func TestKeyChangeCatchupOnJoinShareNoNewUsers(t *testing.T) {
 			"!another:room": {syncingUser, existingUser},
 		},
 	}
-	_, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
+	_, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
 	if err != nil {
 		t.Fatalf("Catchup returned an error: %s", err)
 	}
@@ -246,7 +263,7 @@ func TestKeyChangeCatchupOnLeaveShareNoUsers(t *testing.T) {
 			"!another:room": {syncingUser, existingUser},
 		},
 	}
-	_, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
+	_, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
 	if err != nil {
 		t.Fatalf("DeviceListCatchup returned an error: %s", err)
 	}
@@ -305,7 +322,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) {
 			roomID: {syncingUser, existingUser},
 		},
 	}
-	_, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
+	_, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
 	if err != nil {
 		t.Fatalf("DeviceListCatchup returned an error: %s", err)
 	}
@@ -333,7 +350,7 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) {
 			"!another:room": {syncingUser},
 		},
 	}
-	_, hasNew, err := DeviceListCatchup(context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
+	_, hasNew, err := DeviceListCatchup(context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken)
 	if err != nil {
 		t.Fatalf("Catchup returned an error: %s", err)
 	}
@@ -419,7 +436,7 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) {
 		},
 	}
 	_, hasNew, err := DeviceListCatchup(
-		context.Background(), &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken,
+		context.Background(), rsAPI, &mockKeyAPI{}, rsAPI, syncingUser, syncResponse, emptyToken, emptyToken,
 	)
 	if err != nil {
 		t.Fatalf("DeviceListCatchup returned an error: %s", err)
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 5a036d889..055426030 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -27,6 +27,8 @@ import (
 
 type Database interface {
 	Presence
+	SharedUsers
+
 	MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error)
 	MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error)
 	MaxStreamPositionForInvites(ctx context.Context) (types.StreamPosition, error)
@@ -165,3 +167,8 @@ type Presence interface {
 	PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
 	MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
 }
+
+type SharedUsers interface {
+	// SharedUsers returns a subset of otherUserIDs that share a room with userID.
+	SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error)
+}
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 8ee387b39..c4667bafd 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -107,6 +107,11 @@ const selectEventsWithEventIDsSQL = "" +
 	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
 	" FROM syncapi_current_room_state WHERE event_id = ANY($1)"
 
+const selectSharedUsersSQL = "" +
+	"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
+	"	SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
+	") AND state_key = ANY($2) AND membership='join';"
+
 type currentRoomStateStatements struct {
 	upsertRoomStateStmt                *sql.Stmt
 	deleteRoomStateByEventIDStmt       *sql.Stmt
@@ -118,6 +123,7 @@ type currentRoomStateStatements struct {
 	selectJoinedUsersInRoomStmt        *sql.Stmt
 	selectEventsWithEventIDsStmt       *sql.Stmt
 	selectStateEventStmt               *sql.Stmt
+	selectSharedUsersStmt              *sql.Stmt
 }
 
 func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
@@ -156,6 +162,9 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
 	if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
 		return nil, err
 	}
+	if s.selectSharedUsersStmt, err = db.Prepare(selectSharedUsersSQL); err != nil {
+		return nil, err
+	}
 	return s, nil
 }
 
@@ -379,3 +388,24 @@ func (s *currentRoomStateStatements) SelectStateEvent(
 	}
 	return &ev, err
 }
+
+func (s *currentRoomStateStatements) SelectSharedUsers(
+	ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string,
+) ([]string, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectSharedUsersStmt)
+	rows, err := stmt.QueryContext(ctx, userID, otherUserIDs)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectSharedUsersStmt: rows.close() failed")
+
+	var stateKey string
+	result := make([]string, 0, len(otherUserIDs))
+	for rows.Next() {
+		if err := rows.Scan(&stateKey); err != nil {
+			return nil, err
+		}
+		result = append(result, stateKey)
+	}
+	return result, rows.Err()
+}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 76114aff8..d1c5e2d16 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -176,6 +176,10 @@ func (d *Database) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]t
 	return d.Peeks.SelectPeekingDevices(ctx)
 }
 
+func (d *Database) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) {
+	return d.CurrentRoomState.SelectSharedUsers(ctx, nil, userID, otherUserIDs)
+}
+
 func (d *Database) GetStateEvent(
 	ctx context.Context, roomID, evType, stateKey string,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index f0a1c7bb7..376c3a3d2 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -91,6 +91,11 @@ const selectEventsWithEventIDsSQL = "" +
 	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
 	" FROM syncapi_current_room_state WHERE event_id IN ($1)"
 
+const selectSharedUsersSQL = "" +
+	"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
+	"	SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
+	") AND state_key IN ($2) AND membership='join';"
+
 type currentRoomStateStatements struct {
 	db                                 *sql.DB
 	streamIDStatements                 *StreamIDStatements
@@ -100,8 +105,9 @@ type currentRoomStateStatements struct {
 	selectRoomIDsWithMembershipStmt    *sql.Stmt
 	selectRoomIDsWithAnyMembershipStmt *sql.Stmt
 	selectJoinedUsersStmt              *sql.Stmt
-	//selectJoinedUsersInRoomStmt        *sql.Stmt - prepared at runtime due to variadic
+	//selectJoinedUsersInRoomStmt      *sql.Stmt - prepared at runtime due to variadic
 	selectStateEventStmt *sql.Stmt
+	//selectSharedUsersSQL             *sql.Stmt - prepared at runtime due to variadic
 }
 
 func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (tables.CurrentRoomState, error) {
@@ -396,3 +402,29 @@ func (s *currentRoomStateStatements) SelectStateEvent(
 	}
 	return &ev, err
 }
+
+func (s *currentRoomStateStatements) SelectSharedUsers(
+	ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string,
+) ([]string, error) {
+	query := strings.Replace(selectSharedUsersSQL, "($2)", sqlutil.QueryVariadicOffset(len(otherUserIDs), 1), 1)
+	stmt, err := s.db.Prepare(query)
+	if err != nil {
+		return nil, fmt.Errorf("SelectSharedUsers s.db.Prepare: %w", err)
+	}
+	defer internal.CloseAndLogIfError(ctx, stmt, "SelectSharedUsers: stmt.close() failed")
+	rows, err := sqlutil.TxStmt(txn, stmt).QueryContext(ctx, userID, otherUserIDs)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectSharedUsersStmt: rows.close() failed")
+
+	var stateKey string
+	result := make([]string, 0, len(otherUserIDs))
+	for rows.Next() {
+		if err := rows.Scan(&stateKey); err != nil {
+			return nil, err
+		}
+		result = append(result, stateKey)
+	}
+	return result, rows.Err()
+}
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index ccdebfdbd..08568d9ac 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -104,6 +104,8 @@ type CurrentRoomState interface {
 	SelectJoinedUsers(ctx context.Context) (map[string][]string, error)
 	// SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
 	SelectJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error)
+	// SelectSharedUsers returns a subset of otherUserIDs that share a room with userID.
+	SelectSharedUsers(ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string) ([]string, error)
 }
 
 // BackwardsExtremities keeps track of backwards extremities for a room.
diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go
index f42099510..5448ee5bd 100644
--- a/syncapi/streams/stream_devicelist.go
+++ b/syncapi/streams/stream_devicelist.go
@@ -28,7 +28,7 @@ func (p *DeviceListStreamProvider) IncrementalSync(
 	from, to types.StreamPosition,
 ) types.StreamPosition {
 	var err error
-	to, _, err = internal.DeviceListCatchup(context.Background(), p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to)
+	to, _, err = internal.DeviceListCatchup(context.Background(), p.DB, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
 		return from
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 6f0849e08..b6b4779a5 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -429,7 +429,7 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use
 	}
 	rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), syncReq, fromToken.PDUPosition, toToken.PDUPosition)
 	_, _, err = internal.DeviceListCatchup(
-		req.Context(), rp.keyAPI, rp.rsAPI, syncReq.Device.UserID,
+		req.Context(), rp.db, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID,
 		syncReq.Response, fromToken.DeviceListPosition, toToken.DeviceListPosition,
 	)
 	if err != nil {

From 4ff57993abc7066c52d979617a81af45629dd611 Mon Sep 17 00:00:00 2001
From: andreever <52261463+andreever@users.noreply.github.com>
Date: Fri, 15 Jul 2022 17:33:05 +0200
Subject: [PATCH 205/391] Update 1_createusers.md (#2571)

* Update 1_createusers.md

Added description on how to create user accounts when running in docker.

* Update 1_createusers.md

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 docs/administration/1_createusers.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md
index f40b7f576..61ec2299b 100644
--- a/docs/administration/1_createusers.md
+++ b/docs/administration/1_createusers.md
@@ -32,6 +32,15 @@ To create a new **admin account**, add the `-admin` flag:
 ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
 ```
 
+An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`:
+
+```bash
+docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME
+```
+```bash
+docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
+```
+
 ## Using shared secret registration
 
 Dendrite supports the Synapse-compatible shared secret registration endpoint.

From c0c909d30642d03600fd48bd052a22040a67267b Mon Sep 17 00:00:00 2001
From: Maximilian Gaedig <38767445+MaximilianGaedig@users.noreply.github.com>
Date: Mon, 18 Jul 2022 00:01:52 +0200
Subject: [PATCH 206/391] Fix connection_string format in
 dendrite-sample.polylith.yaml (#2574)

---
 dendrite-sample.polylith.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index ea3d9d689..723547f55 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -125,7 +125,7 @@ app_service_api:
 
   # Database configuration for this component.
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_appservice?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_appservice?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -199,7 +199,7 @@ federation_api:
   external_api:
     listen: http://[::]:8072
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_federationapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_federationapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -236,7 +236,7 @@ key_server:
     listen: http://[::]:7779 # The listen address for incoming API requests
     connect: http://key_server:7779 # The connect address for other components to use
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_keyserver?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_keyserver?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -249,7 +249,7 @@ media_api:
   external_api:
     listen: http://[::]:8074
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_mediaapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_mediaapi?sslmode=disable
     max_open_conns: 5
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -286,7 +286,7 @@ mscs:
   #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
   #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_mscs?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_mscs?sslmode=disable
     max_open_conns: 5
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -297,7 +297,7 @@ room_server:
     listen: http://[::]:7770 # The listen address for incoming API requests
     connect: http://room_server:7770 # The connect address for other components to use
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_roomserver?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_roomserver?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -310,7 +310,7 @@ sync_api:
   external_api:
     listen: http://[::]:8073
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_syncapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_syncapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -326,7 +326,7 @@ user_api:
     listen: http://[::]:7781 # The listen address for incoming API requests
     connect: http://user_api:7781 # The connect address for other components to use
   account_database:
-    connection_string: postgresql://username@password:hostname/dendrite_userapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_userapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1

From a7e92f8cb99105c2a579f73fbc972b7fb0b35678 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 18 Jul 2022 14:46:15 +0200
Subject: [PATCH 207/391] History visibility database changes (#2533)

* Add new history_visibility column

* Update SQL queries to include history_visibility

* Store the history visibilty calculated by the roomserver

* Update GMSL

* Update migrations

* Fix migration

* Update GMSL

* Fix `go.sum`

* Update GMSL to use sql.Scanner & sql.Valuer

* Re-order migration/table creation

* Update gomatrixserverlib

* Add history_visibility column to current_room_state

* Fix migrations

* Return error instead of Fatal log

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 go.mod                                        |  2 +-
 go.sum                                        |  4 +-
 syncapi/consumers/roomserver.go               |  4 +-
 syncapi/routing/messages.go                   |  1 +
 syncapi/storage/interface.go                  |  4 +-
 .../postgres/current_room_state_table.go      | 12 +--
 ...2061412000000_history_visibility_column.go | 50 +++++++++++
 .../postgres/output_room_events_table.go      | 82 ++++++++++--------
 syncapi/storage/postgres/syncserver.go        | 24 ++++--
 syncapi/storage/shared/syncserver.go          |  7 +-
 .../sqlite3/current_room_state_table.go       | 12 +--
 ...2061412000000_history_visibility_column.go | 82 ++++++++++++++++++
 .../sqlite3/output_room_events_table.go       | 83 +++++++++++--------
 syncapi/storage/sqlite3/syncserver.go         | 24 ++++--
 syncapi/storage/storage_test.go               |  2 +-
 syncapi/storage/tables/interface.go           |  9 +-
 .../storage/tables/output_room_events_test.go |  4 +-
 17 files changed, 299 insertions(+), 107 deletions(-)
 create mode 100644 syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
 create mode 100644 syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go

diff --git a/go.mod b/go.mod
index 1e62e1d93..0a965ec8e 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d
 	github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 86656ccd5..df3feeb6e 100644
--- a/go.sum
+++ b/go.sum
@@ -341,8 +341,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46 h1:5X/kXY3nwqKOwwrE9tnMKrjbsi3PHigQYvrvDBSntO8=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220713083127-fc2ea1e62e46/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d h1:BWInUURXVOW+OiifMapoRIS7i122KWdEKj6fnDFXgBo=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index f0ca2106f..328e1e9dd 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -240,6 +240,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 		msg.RemovesStateEventIDs,
 		msg.TransactionID,
 		false,
+		msg.HistoryVisibility,
 	)
 	if err != nil {
 		// panic rather than continue with an inconsistent database
@@ -289,7 +290,8 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 		[]string{},           // adds no state
 		[]string{},           // removes no state
 		nil,                  // no transaction
-		ev.StateKey() != nil, // exclude from sync?
+		ev.StateKey() != nil, // exclude from sync?,
+		msg.HistoryVisibility,
 	)
 	if err != nil {
 		// panic rather than continue with an inconsistent database
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 24745cd55..990ca55b1 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -594,6 +594,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 			[]string{},
 			[]string{},
 			nil, true,
+			gomatrixserverlib.HistoryVisibilityShared,
 		)
 		if err != nil {
 			return nil, err
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 055426030..9751670b2 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -69,7 +69,9 @@ type Database interface {
 	// when generating the sync stream position for this event. Returns the sync stream position for the inserted event.
 	// Returns an error if there was a problem inserting this event.
 	WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []*gomatrixserverlib.HeaderedEvent,
-		addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error)
+		addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool,
+		historyVisibility gomatrixserverlib.HistoryVisibility,
+	) (types.StreamPosition, error)
 	// PurgeRoomState completely purges room state from the sync API. This is done when
 	// receiving an output event that completely resets the state.
 	PurgeRoomState(ctx context.Context, roomID string) error
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index c4667bafd..aae2d8c33 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -51,6 +51,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state (
     -- The serial ID of the output_room_events table when this event became
     -- part of the current state of the room.
     added_at BIGINT,
+    history_visibility SMALLINT NOT NULL DEFAULT 2,
     -- Clobber based on 3-uple of room_id, type and state_key
     CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key)
 );
@@ -63,8 +64,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON sync
 `
 
 const upsertRoomStateSQL = "" +
-	"INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" +
-	" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" +
+	"INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at, history_visibility)" +
+	" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" +
 	" ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" +
 	" DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9"
 
@@ -100,11 +101,11 @@ const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
 
 const selectEventsWithEventIDsSQL = "" +
-	// TODO: The session_id and transaction_id blanks are here because otherwise
-	// the rowsToStreamEvents expects there to be exactly six columns. We need to
+	// TODO: The session_id and transaction_id blanks are here because
+	// the rowsToStreamEvents expects there to be exactly seven columns. We need to
 	// figure out if these really need to be in the DB, and if so, we need a
 	// better permanent fix for this. - neilalexander, 2 Jan 2020
-	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
+	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" +
 	" FROM syncapi_current_room_state WHERE event_id = ANY($1)"
 
 const selectSharedUsersSQL = "" +
@@ -336,6 +337,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 		headeredJSON,
 		membership,
 		addedAt,
+		event.Visibility,
 	)
 	return err
 }
diff --git a/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
new file mode 100644
index 000000000..2b4b65287
--- /dev/null
+++ b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
@@ -0,0 +1,50 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadAddHistoryVisibilityColumn(m *sqlutil.Migrations) {
+	m.AddMigration(UpAddHistoryVisibilityColumn, DownAddHistoryVisibilityColumn)
+}
+
+func UpAddHistoryVisibilityColumn(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+		ALTER TABLE syncapi_output_room_events ADD COLUMN IF NOT EXISTS history_visibility SMALLINT NOT NULL DEFAULT 2;
+		UPDATE syncapi_output_room_events SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
+		ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS history_visibility SMALLINT NOT NULL DEFAULT 2;
+		UPDATE syncapi_current_room_state SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownAddHistoryVisibilityColumn(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+		ALTER TABLE syncapi_output_room_events DROP COLUMN IF EXISTS history_visibility;
+		ALTER TABLE syncapi_current_room_state DROP COLUMN IF EXISTS history_visibility;
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index d84d0cfa2..d0474d4c3 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -67,7 +67,9 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
   -- events retrieved through backfilling that have a position in the stream
   -- that relates to the moment these were retrieved rather than the moment these
   -- were emitted.
-  exclude_from_sync BOOL DEFAULT FALSE
+  exclude_from_sync BOOL DEFAULT FALSE,
+  -- The history visibility before this event (1 - world_readable; 2 - shared; 3 - invited; 4 - joined)
+  history_visibility SMALLINT NOT NULL DEFAULT 2
 );
 
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type);
@@ -78,16 +80,16 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON s
 
 const insertEventSQL = "" +
 	"INSERT INTO syncapi_output_room_events (" +
-	"room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" +
-	") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) " +
+	"room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync, history_visibility" +
+	") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " +
 	"ON CONFLICT ON CONSTRAINT syncapi_event_id_idx DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $11) " +
 	"RETURNING id"
 
 const selectEventsSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)"
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events WHERE event_id = ANY($1)"
 
 const selectEventsWithFilterSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events WHERE event_id = ANY($1)" +
 	" AND ( $2::text[] IS NULL OR     sender  = ANY($2)  )" +
 	" AND ( $3::text[] IS NULL OR NOT(sender  = ANY($3)) )" +
 	" AND ( $4::text[] IS NULL OR     type LIKE ANY($4)  )" +
@@ -96,7 +98,7 @@ const selectEventsWithFilterSQL = "" +
 	" LIMIT $7"
 
 const selectRecentEventsSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3" +
 	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
 	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
@@ -105,7 +107,7 @@ const selectRecentEventsSQL = "" +
 	" ORDER BY id DESC LIMIT $8"
 
 const selectRecentEventsForSyncSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" +
 	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
 	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
@@ -114,7 +116,7 @@ const selectRecentEventsForSyncSQL = "" +
 	" ORDER BY id DESC LIMIT $8"
 
 const selectEarlyEventsSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3" +
 	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
 	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
@@ -130,7 +132,7 @@ const updateEventJSONSQL = "" +
 
 // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id).
 const selectStateInRangeSQL = "" +
-	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
+	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" +
 	" FROM syncapi_output_room_events" +
 	" WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" +
 	" AND room_id = ANY($3)" +
@@ -146,10 +148,10 @@ const deleteEventsForRoomSQL = "" +
 	"DELETE FROM syncapi_output_room_events WHERE room_id = $1"
 
 const selectContextEventSQL = "" +
-	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
+	"SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
 
 const selectContextBeforeEventSQL = "" +
-	"SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" +
+	"SELECT headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2" +
 	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
 	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
 	" AND ( $6::text[] IS NULL OR     type LIKE ANY($6)  )" +
@@ -157,7 +159,7 @@ const selectContextBeforeEventSQL = "" +
 	" ORDER BY id DESC LIMIT $3"
 
 const selectContextAfterEventSQL = "" +
-	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" +
+	"SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2" +
 	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
 	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
 	" AND ( $6::text[] IS NULL OR     type LIKE ANY($6)  )" +
@@ -246,14 +248,15 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 
 	for rows.Next() {
 		var (
-			eventID         string
-			streamPos       types.StreamPosition
-			eventBytes      []byte
-			excludeFromSync bool
-			addIDs          pq.StringArray
-			delIDs          pq.StringArray
+			eventID           string
+			streamPos         types.StreamPosition
+			eventBytes        []byte
+			excludeFromSync   bool
+			addIDs            pq.StringArray
+			delIDs            pq.StringArray
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil {
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs, &historyVisibility); err != nil {
 			return nil, nil, err
 		}
 		// Sanity check for deleted state and whine if we see it. We don't need to do anything
@@ -283,6 +286,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 			needSet[id] = true
 		}
 		stateNeeded[ev.RoomID()] = needSet
+		ev.Visibility = historyVisibility
 
 		eventIDToEvent[eventID] = types.StreamEvent{
 			HeaderedEvent:   &ev,
@@ -314,7 +318,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID(
 func (s *outputRoomEventsStatements) InsertEvent(
 	ctx context.Context, txn *sql.Tx,
 	event *gomatrixserverlib.HeaderedEvent, addState, removeState []string,
-	transactionID *api.TransactionID, excludeFromSync bool,
+	transactionID *api.TransactionID, excludeFromSync bool, historyVisibility gomatrixserverlib.HistoryVisibility,
 ) (streamPos types.StreamPosition, err error) {
 	var txnID *string
 	var sessionID *int64
@@ -351,6 +355,7 @@ func (s *outputRoomEventsStatements) InsertEvent(
 		sessionID,
 		txnID,
 		excludeFromSync,
+		historyVisibility,
 	).Scan(&streamPos)
 	return
 }
@@ -504,13 +509,15 @@ func (s *outputRoomEventsStatements) SelectContextEvent(ctx context.Context, txn
 	row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID)
 
 	var eventAsString string
-	if err = row.Scan(&id, &eventAsString); err != nil {
+	var historyVisibility gomatrixserverlib.HistoryVisibility
+	if err = row.Scan(&id, &eventAsString, &historyVisibility); err != nil {
 		return 0, evt, err
 	}
 
 	if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil {
 		return 0, evt, err
 	}
+	evt.Visibility = historyVisibility
 	return id, evt, nil
 }
 
@@ -532,15 +539,17 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
 
 	for rows.Next() {
 		var (
-			eventBytes []byte
-			evt        *gomatrixserverlib.HeaderedEvent
+			eventBytes        []byte
+			evt               *gomatrixserverlib.HeaderedEvent
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err = rows.Scan(&eventBytes); err != nil {
+		if err = rows.Scan(&eventBytes, &historyVisibility); err != nil {
 			return evts, err
 		}
 		if err = json.Unmarshal(eventBytes, &evt); err != nil {
 			return evts, err
 		}
+		evt.Visibility = historyVisibility
 		evts = append(evts, evt)
 	}
 
@@ -565,15 +574,17 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent(
 
 	for rows.Next() {
 		var (
-			eventBytes []byte
-			evt        *gomatrixserverlib.HeaderedEvent
+			eventBytes        []byte
+			evt               *gomatrixserverlib.HeaderedEvent
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err = rows.Scan(&lastID, &eventBytes); err != nil {
+		if err = rows.Scan(&lastID, &eventBytes, &historyVisibility); err != nil {
 			return 0, evts, err
 		}
 		if err = json.Unmarshal(eventBytes, &evt); err != nil {
 			return 0, evts, err
 		}
+		evt.Visibility = historyVisibility
 		evts = append(evts, evt)
 	}
 
@@ -584,15 +595,16 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 	var result []types.StreamEvent
 	for rows.Next() {
 		var (
-			eventID         string
-			streamPos       types.StreamPosition
-			eventBytes      []byte
-			excludeFromSync bool
-			sessionID       *int64
-			txnID           *string
-			transactionID   *api.TransactionID
+			eventID           string
+			streamPos         types.StreamPosition
+			eventBytes        []byte
+			excludeFromSync   bool
+			sessionID         *int64
+			txnID             *string
+			transactionID     *api.TransactionID
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil {
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID, &historyVisibility); err != nil {
 			return nil, err
 		}
 		// TODO: Handle redacted events
diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go
index 9cfe7c070..c6121e419 100644
--- a/syncapi/storage/postgres/syncserver.go
+++ b/syncapi/storage/postgres/syncserver.go
@@ -42,18 +42,16 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
 		return nil, err
 	}
+	if _, err = d.db.Exec(outputRoomEventsSchema); err != nil {
+		return nil, err
+	}
+	if _, err = d.db.Exec(currentRoomStateSchema); err != nil {
+		return nil, err
+	}
 	accountData, err := NewPostgresAccountDataTable(d.db)
 	if err != nil {
 		return nil, err
 	}
-	events, err := NewPostgresEventsTable(d.db)
-	if err != nil {
-		return nil, err
-	}
-	currState, err := NewPostgresCurrentRoomStateTable(d.db)
-	if err != nil {
-		return nil, err
-	}
 	invites, err := NewPostgresInvitesTable(d.db)
 	if err != nil {
 		return nil, err
@@ -101,9 +99,19 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	m := sqlutil.NewMigrations()
 	deltas.LoadFixSequences(m)
 	deltas.LoadRemoveSendToDeviceSentColumn(m)
+	deltas.LoadAddHistoryVisibilityColumn(m)
 	if err = m.RunDeltas(d.db, dbProperties); err != nil {
 		return nil, err
 	}
+	// prepare statements after the migrations have run
+	events, err := NewPostgresEventsTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	currState, err := NewPostgresCurrentRoomStateTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Writer:              d.writer,
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index d1c5e2d16..932bda164 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -368,11 +368,12 @@ func (d *Database) WriteEvent(
 	addStateEvents []*gomatrixserverlib.HeaderedEvent,
 	addStateEventIDs, removeStateEventIDs []string,
 	transactionID *api.TransactionID, excludeFromSync bool,
+	historyVisibility gomatrixserverlib.HistoryVisibility,
 ) (pduPosition types.StreamPosition, returnErr error) {
 	returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		var err error
 		pos, err := d.OutputEvents.InsertEvent(
-			ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync,
+			ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, historyVisibility,
 		)
 		if err != nil {
 			return fmt.Errorf("d.OutputEvents.InsertEvent: %w", err)
@@ -391,7 +392,9 @@ func (d *Database) WriteEvent(
 			// Nothing to do, the event may have just been a message event.
 			return nil
 		}
-
+		for i := range addStateEvents {
+			addStateEvents[i].Visibility = historyVisibility
+		}
 		return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition)
 	})
 
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 376c3a3d2..501d0ee98 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -41,6 +41,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state (
     headered_event_json TEXT NOT NULL,
     membership TEXT,
     added_at BIGINT,
+    history_visibility SMALLINT NOT NULL DEFAULT 2, -- The history visibility before this event (1 - world_readable; 2 - shared; 3 - invited; 4 - joined)
     UNIQUE (room_id, type, state_key)
 );
 -- for event deletion
@@ -52,8 +53,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON sync
 `
 
 const upsertRoomStateSQL = "" +
-	"INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" +
-	" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" +
+	"INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at, history_visibility)" +
+	" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" +
 	" ON CONFLICT (room_id, type, state_key)" +
 	" DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9"
 
@@ -84,11 +85,11 @@ const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
 
 const selectEventsWithEventIDsSQL = "" +
-	// TODO: The session_id and transaction_id blanks are here because otherwise
-	// the rowsToStreamEvents expects there to be exactly six columns. We need to
+	// TODO: The session_id and transaction_id blanks are here because
+	// the rowsToStreamEvents expects there to be exactly seven columns. We need to
 	// figure out if these really need to be in the DB, and if so, we need a
 	// better permanent fix for this. - neilalexander, 2 Jan 2020
-	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
+	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" +
 	" FROM syncapi_current_room_state WHERE event_id IN ($1)"
 
 const selectSharedUsersSQL = "" +
@@ -328,6 +329,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
 		headeredJSON,
 		membership,
 		addedAt,
+		event.Visibility,
 	)
 	return err
 }
diff --git a/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
new file mode 100644
index 000000000..f6bcaddfb
--- /dev/null
+++ b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
@@ -0,0 +1,82 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadAddHistoryVisibilityColumn(m *sqlutil.Migrations) {
+	m.AddMigration(UpAddHistoryVisibilityColumn, DownAddHistoryVisibilityColumn)
+}
+
+func UpAddHistoryVisibilityColumn(tx *sql.Tx) error {
+	// SQLite doesn't have "if exists", so check if the column exists. If the query doesn't return an error, it already exists.
+	// Required for unit tests, as otherwise a duplicate column error will show up.
+	_, err := tx.Query("SELECT history_visibility FROM syncapi_output_room_events LIMIT 1")
+	if err == nil {
+		return nil
+	}
+	_, err = tx.Exec(`
+		ALTER TABLE syncapi_output_room_events ADD COLUMN history_visibility SMALLINT NOT NULL DEFAULT 2;
+		UPDATE syncapi_output_room_events SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+
+	_, err = tx.Query("SELECT history_visibility FROM syncapi_current_room_state LIMIT 1")
+	if err == nil {
+		return nil
+	}
+	_, err = tx.Exec(`
+		ALTER TABLE syncapi_current_room_state ADD COLUMN history_visibility SMALLINT NOT NULL DEFAULT 2;
+		UPDATE syncapi_current_room_state SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownAddHistoryVisibilityColumn(tx *sql.Tx) error {
+	// SQLite doesn't have "if exists", so check if the column exists.
+	_, err := tx.Query("SELECT history_visibility FROM syncapi_output_room_events LIMIT 1")
+	if err != nil {
+		// The column probably doesn't exist
+		return nil
+	}
+	_, err = tx.Exec(`
+		ALTER TABLE syncapi_output_room_events DROP COLUMN history_visibility;
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	_, err = tx.Query("SELECT history_visibility FROM syncapi_current_room_state LIMIT 1")
+	if err != nil {
+		// The column probably doesn't exist
+		return nil
+	}
+	_, err = tx.Exec(`
+		ALTER TABLE syncapi_current_room_state DROP COLUMN history_visibility;
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index f9961a9d1..b3dcb44cb 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -47,7 +47,8 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
   remove_state_ids TEXT, -- JSON encoded string array
   session_id BIGINT,
   transaction_id TEXT,
-  exclude_from_sync BOOL NOT NULL DEFAULT FALSE
+  exclude_from_sync BOOL NOT NULL DEFAULT FALSE,
+  history_visibility SMALLINT NOT NULL DEFAULT 2 -- The history visibility before this event (1 - world_readable; 2 - shared; 3 - invited; 4 - joined)
 );
 
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type);
@@ -58,27 +59,27 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON s
 
 const insertEventSQL = "" +
 	"INSERT INTO syncapi_output_room_events (" +
-	"id, room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" +
-	") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " +
-	"ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $13)"
+	"id, room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync, history_visibility" +
+	") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) " +
+	"ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $14)"
 
 const selectEventsSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id IN ($1)"
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events WHERE event_id IN ($1)"
 
 const selectRecentEventsSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3"
 
 // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
 const selectRecentEventsForSyncSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE"
 
 // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
 const selectEarlyEventsSQL = "" +
-	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	"SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" +
 	" WHERE room_id = $1 AND id > $2 AND id <= $3"
 
 // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
@@ -90,7 +91,7 @@ const updateEventJSONSQL = "" +
 	"UPDATE syncapi_output_room_events SET headered_event_json=$1 WHERE event_id=$2"
 
 const selectStateInRangeSQL = "" +
-	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
+	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" +
 	" FROM syncapi_output_room_events" +
 	" WHERE (id > $1 AND id <= $2)" +
 	" AND room_id IN ($3)" +
@@ -102,15 +103,15 @@ const deleteEventsForRoomSQL = "" +
 	"DELETE FROM syncapi_output_room_events WHERE room_id = $1"
 
 const selectContextEventSQL = "" +
-	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
+	"SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND event_id = $2"
 
 const selectContextBeforeEventSQL = "" +
-	"SELECT headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2"
+	"SELECT headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id < $2"
 
 // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
 const selectContextAfterEventSQL = "" +
-	"SELECT id, headered_event_json FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2"
+	"SELECT id, headered_event_json, history_visibility FROM syncapi_output_room_events WHERE room_id = $1 AND id > $2"
 
 // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
@@ -196,14 +197,15 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 
 	for rows.Next() {
 		var (
-			eventID         string
-			streamPos       types.StreamPosition
-			eventBytes      []byte
-			excludeFromSync bool
-			addIDsJSON      string
-			delIDsJSON      string
+			eventID           string
+			streamPos         types.StreamPosition
+			eventBytes        []byte
+			excludeFromSync   bool
+			addIDsJSON        string
+			delIDsJSON        string
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil {
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON, &historyVisibility); err != nil {
 			return nil, nil, err
 		}
 
@@ -239,6 +241,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 			needSet[id] = true
 		}
 		stateNeeded[ev.RoomID()] = needSet
+		ev.Visibility = historyVisibility
 
 		eventIDToEvent[eventID] = types.StreamEvent{
 			HeaderedEvent:   &ev,
@@ -270,7 +273,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID(
 func (s *outputRoomEventsStatements) InsertEvent(
 	ctx context.Context, txn *sql.Tx,
 	event *gomatrixserverlib.HeaderedEvent, addState, removeState []string,
-	transactionID *api.TransactionID, excludeFromSync bool,
+	transactionID *api.TransactionID, excludeFromSync bool, historyVisibility gomatrixserverlib.HistoryVisibility,
 ) (types.StreamPosition, error) {
 	var txnID *string
 	var sessionID *int64
@@ -326,6 +329,7 @@ func (s *outputRoomEventsStatements) InsertEvent(
 		sessionID,
 		txnID,
 		excludeFromSync,
+		historyVisibility,
 		excludeFromSync,
 	)
 	return streamPos, err
@@ -481,15 +485,16 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 	var result []types.StreamEvent
 	for rows.Next() {
 		var (
-			eventID         string
-			streamPos       types.StreamPosition
-			eventBytes      []byte
-			excludeFromSync bool
-			sessionID       *int64
-			txnID           *string
-			transactionID   *api.TransactionID
+			eventID           string
+			streamPos         types.StreamPosition
+			eventBytes        []byte
+			excludeFromSync   bool
+			sessionID         *int64
+			txnID             *string
+			transactionID     *api.TransactionID
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil {
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID, &historyVisibility); err != nil {
 			return nil, err
 		}
 		// TODO: Handle redacted events
@@ -505,6 +510,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 			}
 		}
 
+		ev.Visibility = historyVisibility
+
 		result = append(result, types.StreamEvent{
 			HeaderedEvent:   &ev,
 			StreamPosition:  streamPos,
@@ -519,13 +526,15 @@ func (s *outputRoomEventsStatements) SelectContextEvent(
 ) (id int, evt gomatrixserverlib.HeaderedEvent, err error) {
 	row := sqlutil.TxStmt(txn, s.selectContextEventStmt).QueryRowContext(ctx, roomID, eventID)
 	var eventAsString string
-	if err = row.Scan(&id, &eventAsString); err != nil {
+	var historyVisibility gomatrixserverlib.HistoryVisibility
+	if err = row.Scan(&id, &eventAsString, &historyVisibility); err != nil {
 		return 0, evt, err
 	}
 
 	if err = json.Unmarshal([]byte(eventAsString), &evt); err != nil {
 		return 0, evt, err
 	}
+	evt.Visibility = historyVisibility
 	return id, evt, nil
 }
 
@@ -550,15 +559,17 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent(
 
 	for rows.Next() {
 		var (
-			eventBytes []byte
-			evt        *gomatrixserverlib.HeaderedEvent
+			eventBytes        []byte
+			evt               *gomatrixserverlib.HeaderedEvent
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err = rows.Scan(&eventBytes); err != nil {
+		if err = rows.Scan(&eventBytes, &historyVisibility); err != nil {
 			return evts, err
 		}
 		if err = json.Unmarshal(eventBytes, &evt); err != nil {
 			return evts, err
 		}
+		evt.Visibility = historyVisibility
 		evts = append(evts, evt)
 	}
 
@@ -586,15 +597,17 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent(
 
 	for rows.Next() {
 		var (
-			eventBytes []byte
-			evt        *gomatrixserverlib.HeaderedEvent
+			eventBytes        []byte
+			evt               *gomatrixserverlib.HeaderedEvent
+			historyVisibility gomatrixserverlib.HistoryVisibility
 		)
-		if err = rows.Scan(&lastID, &eventBytes); err != nil {
+		if err = rows.Scan(&lastID, &eventBytes, &historyVisibility); err != nil {
 			return 0, evts, err
 		}
 		if err = json.Unmarshal(eventBytes, &evt); err != nil {
 			return 0, evts, err
 		}
+		evt.Visibility = historyVisibility
 		evts = append(evts, evt)
 	}
 	return lastID, evts, rows.Err()
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index e08a0ba82..39ceec81d 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -52,18 +52,16 @@ func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (er
 	if err = d.streamID.Prepare(d.db); err != nil {
 		return err
 	}
+	if _, err = d.db.Exec(outputRoomEventsSchema); err != nil {
+		return err
+	}
+	if _, err = d.db.Exec(currentRoomStateSchema); err != nil {
+		return err
+	}
 	accountData, err := NewSqliteAccountDataTable(d.db, &d.streamID)
 	if err != nil {
 		return err
 	}
-	events, err := NewSqliteEventsTable(d.db, &d.streamID)
-	if err != nil {
-		return err
-	}
-	roomState, err := NewSqliteCurrentRoomStateTable(d.db, &d.streamID)
-	if err != nil {
-		return err
-	}
 	invites, err := NewSqliteInvitesTable(d.db, &d.streamID)
 	if err != nil {
 		return err
@@ -111,9 +109,19 @@ func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (er
 	m := sqlutil.NewMigrations()
 	deltas.LoadFixSequences(m)
 	deltas.LoadRemoveSendToDeviceSentColumn(m)
+	deltas.LoadAddHistoryVisibilityColumn(m)
 	if err = m.RunDeltas(d.db, dbProperties); err != nil {
 		return err
 	}
+	// prepare statements after the migrations have run
+	events, err := NewSqliteEventsTable(d.db, &d.streamID)
+	if err != nil {
+		return err
+	}
+	roomState, err := NewSqliteCurrentRoomStateTable(d.db, &d.streamID)
+	if err != nil {
+		return err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Writer:              d.writer,
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index c74151700..df03a33c2 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -37,7 +37,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserver
 			addStateEvents = append(addStateEvents, ev)
 			addStateEventIDs = append(addStateEventIDs, ev.EventID())
 		}
-		pos, err := db.WriteEvent(ctx, ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false)
+		pos, err := db.WriteEvent(ctx, ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false, gomatrixserverlib.HistoryVisibilityShared)
 		if err != nil {
 			t.Fatalf("WriteEvent failed: %s", err)
 		}
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 08568d9ac..d8fc8f508 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -52,7 +52,14 @@ type Peeks interface {
 type Events interface {
 	SelectStateInRange(ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, roomIDs []string) (map[string]map[string]bool, map[string]types.StreamEvent, error)
 	SelectMaxEventID(ctx context.Context, txn *sql.Tx) (id int64, err error)
-	InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error)
+	InsertEvent(
+		ctx context.Context, txn *sql.Tx,
+		event *gomatrixserverlib.HeaderedEvent,
+		addState, removeState []string,
+		transactionID *api.TransactionID,
+		excludeFromSync bool,
+		historyVisibility gomatrixserverlib.HistoryVisibility,
+	) (streamPos types.StreamPosition, err error)
 	// SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high.
 	// If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync.
 	// Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`.
diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go
index 69bbd04c9..bdb17ae20 100644
--- a/syncapi/storage/tables/output_room_events_test.go
+++ b/syncapi/storage/tables/output_room_events_test.go
@@ -53,7 +53,7 @@ func TestOutputRoomEventsTable(t *testing.T) {
 		events := room.Events()
 		err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error {
 			for _, ev := range events {
-				_, err := tab.InsertEvent(ctx, txn, ev, nil, nil, nil, false)
+				_, err := tab.InsertEvent(ctx, txn, ev, nil, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared)
 				if err != nil {
 					return fmt.Errorf("failed to InsertEvent: %s", err)
 				}
@@ -79,7 +79,7 @@ func TestOutputRoomEventsTable(t *testing.T) {
 				"body": "test.txt",
 				"url":  "mxc://test.txt",
 			})
-			if _, err = tab.InsertEvent(ctx, txn, urlEv, nil, nil, nil, false); err != nil {
+			if _, err = tab.InsertEvent(ctx, txn, urlEv, nil, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared); err != nil {
 				return fmt.Errorf("failed to InsertEvent: %s", err)
 			}
 			wantEventID := []string{urlEv.EventID()}

From 99b696e775f13eb876f68195acdbffc2e05764b2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 18 Jul 2022 14:49:41 +0100
Subject: [PATCH 208/391] Tweak cache counters (#2575)

* Tweak cache counters

This makes the number of counters relative to the
maximum cache size. Since the counters
effectively manage the size of the bloom filter,
larger caches need more counters and smaller
caches need less.

10 counters per 1KB data means that the default
cache size of 1GB should result in a bloom filter
and TinyLRU admission set of about 16MB
estimated.

* Remove line left by accident
---
 internal/caching/impl_ristretto.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go
index 677218b5e..01c97ad55 100644
--- a/internal/caching/impl_ristretto.go
+++ b/internal/caching/impl_ristretto.go
@@ -44,9 +44,9 @@ const (
 
 func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enablePrometheus bool) *Caches {
 	cache, err := ristretto.NewCache(&ristretto.Config{
-		NumCounters: 1e5, // 10x number of expected cache items, affects bloom filter size, gives us room for 10,000 currently
-		BufferItems: 64,  // recommended by the ristretto godocs as a sane buffer size value
-		MaxCost:     int64(maxCost),
+		NumCounters: int64((maxCost / 1024) * 10), // 10 counters per 1KB data, affects bloom filter size
+		BufferItems: 64,                           // recommended by the ristretto godocs as a sane buffer size value
+		MaxCost:     int64(maxCost),               // max cost is in bytes, as per the Dendrite config
 		Metrics:     true,
 		KeyToHash: func(key interface{}) (uint64, uint64) {
 			return z.KeyToHash(key)

From bcff14adea471cbb496924622295c62c02a82720 Mon Sep 17 00:00:00 2001
From: Till Faelligen <davidf@element.io>
Date: Mon, 18 Jul 2022 18:19:44 +0200
Subject: [PATCH 209/391] Set historyVisibility in rowsToStreamEvents

---
 syncapi/storage/postgres/output_room_events_table.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index d0474d4c3..ddef27383 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -619,7 +619,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 				TransactionID: *txnID,
 			}
 		}
-
+		ev.Visibility = historyVisibility
 		result = append(result, types.StreamEvent{
 			HeaderedEvent:   &ev,
 			StreamPosition:  streamPos,

From 583b8ea273be82ebdf2a9f81fd51a150d910b349 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 19 Jul 2022 11:51:46 +0100
Subject: [PATCH 210/391] Update FAQ

---
 docs/FAQ.md                    | 9 ++++++---
 setup/config/config_global.go  | 6 +++---
 userapi/util/phonehomestats.go | 6 +++---
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index 47f39b9e6..f8255684e 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -86,9 +86,12 @@ would be a huge help too, as that will help us to understand where the memory us
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!
 
-## What is being reported when enabling anonymous stats?
+## What is being reported when enabling phone-home statistics?
 
-If anonymous stats reporting is enabled, the following data is send to the defined endpoint.
+Phone-home statistics contain your server's domain name, some configuration information about
+your deployment and aggregated information about active users on your deployment. They are sent
+to the endpoint URL configured in your Dendrite configuration file only. The following is an
+example of the data that is sent:
 
 ```json
 {
@@ -106,7 +109,7 @@ If anonymous stats reporting is enabled, the following data is send to the defin
     "go_arch": "amd64",
     "go_os": "linux",
     "go_version": "go1.16.13",
-    "homeserver": "localhost:8800",
+    "homeserver": "my.domain.com",
     "log_level": "trace",
     "memory_rss": 93452,
     "monolith": true,
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index ac1380a4e..10827bb21 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -73,7 +73,7 @@ type Global struct {
 	// ServerNotices configuration used for sending server notices
 	ServerNotices ServerNotices `yaml:"server_notices"`
 
-	// ReportStats configures opt-in anonymous stats reporting.
+	// ReportStats configures opt-in phone-home statistics reporting.
 	ReportStats ReportStats `yaml:"report_stats"`
 
 	// Configuration for the caches.
@@ -189,9 +189,9 @@ func (c *Cache) Verify(errors *ConfigErrors, isMonolith bool) {
 	checkPositive(errors, "max_size_estimated", int64(c.EstimatedMaxSize))
 }
 
-// ReportStats configures opt-in anonymous stats reporting.
+// ReportStats configures opt-in phone-home statistics reporting.
 type ReportStats struct {
-	// Enabled configures anonymous usage stats of the server
+	// Enabled configures phone-home statistics of the server
 	Enabled bool `yaml:"enabled"`
 
 	// Endpoint the endpoint to report stats to
diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go
index ad93a50e3..b17f62060 100644
--- a/userapi/util/phonehomestats.go
+++ b/userapi/util/phonehomestats.go
@@ -139,7 +139,7 @@ func (p *phoneHomeStats) collect() {
 
 	output := bytes.Buffer{}
 	if err = json.NewEncoder(&output).Encode(p.stats); err != nil {
-		logrus.WithError(err).Error("unable to encode anonymous stats")
+		logrus.WithError(err).Error("Unable to encode phone-home statistics")
 		return
 	}
 
@@ -147,14 +147,14 @@ func (p *phoneHomeStats) collect() {
 
 	request, err := http.NewRequestWithContext(ctx, http.MethodPost, p.cfg.Global.ReportStats.Endpoint, &output)
 	if err != nil {
-		logrus.WithError(err).Error("unable to create anonymous stats request")
+		logrus.WithError(err).Error("Unable to create phone-home statistics request")
 		return
 	}
 	request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
 
 	_, err = p.client.Do(request)
 	if err != nil {
-		logrus.WithError(err).Error("unable to send anonymous stats")
+		logrus.WithError(err).Error("Unable to send phone-home statistics")
 		return
 	}
 }

From 5c01306bb5add6f53907950982fc468c00ae266d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 19 Jul 2022 12:15:48 +0100
Subject: [PATCH 211/391] Add event state key cache (#2576)

---
 internal/caching/cache_eventstatekeys.go | 18 ++++++++++++++++++
 internal/caching/cache_roomservernids.go |  5 +++--
 internal/caching/caches.go               | 23 ++++++++++++-----------
 internal/caching/impl_ristretto.go       |  8 +++++++-
 roomserver/storage/shared/storage.go     | 19 ++++++++++++++++++-
 5 files changed, 58 insertions(+), 15 deletions(-)
 create mode 100644 internal/caching/cache_eventstatekeys.go

diff --git a/internal/caching/cache_eventstatekeys.go b/internal/caching/cache_eventstatekeys.go
new file mode 100644
index 000000000..05580ab05
--- /dev/null
+++ b/internal/caching/cache_eventstatekeys.go
@@ -0,0 +1,18 @@
+package caching
+
+import "github.com/matrix-org/dendrite/roomserver/types"
+
+// EventStateKeyCache contains the subset of functions needed for
+// a room event state key cache.
+type EventStateKeyCache interface {
+	GetEventStateKey(eventStateKeyNID types.EventStateKeyNID) (string, bool)
+	StoreEventStateKey(eventStateKeyNID types.EventStateKeyNID, eventStateKey string)
+}
+
+func (c Caches) GetEventStateKey(eventStateKeyNID types.EventStateKeyNID) (string, bool) {
+	return c.RoomServerStateKeys.Get(eventStateKeyNID)
+}
+
+func (c Caches) StoreEventStateKey(eventStateKeyNID types.EventStateKeyNID, eventStateKey string) {
+	c.RoomServerStateKeys.Set(eventStateKeyNID, eventStateKey)
+}
diff --git a/internal/caching/cache_roomservernids.go b/internal/caching/cache_roomservernids.go
index b409aeef2..f27154f19 100644
--- a/internal/caching/cache_roomservernids.go
+++ b/internal/caching/cache_roomservernids.go
@@ -9,6 +9,7 @@ type RoomServerCaches interface {
 	RoomVersionCache
 	RoomInfoCache
 	RoomServerEventsCache
+	EventStateKeyCache
 }
 
 // RoomServerNIDsCache contains the subset of functions needed for
@@ -19,9 +20,9 @@ type RoomServerNIDsCache interface {
 }
 
 func (c Caches) GetRoomServerRoomID(roomNID types.RoomNID) (string, bool) {
-	return c.RoomServerRoomIDs.Get(int64(roomNID))
+	return c.RoomServerRoomIDs.Get(roomNID)
 }
 
 func (c Caches) StoreRoomServerRoomID(roomNID types.RoomNID, roomID string) {
-	c.RoomServerRoomIDs.Set(int64(roomNID), roomID)
+	c.RoomServerRoomIDs.Set(roomNID, roomID)
 }
diff --git a/internal/caching/caches.go b/internal/caching/caches.go
index e7914ce7d..f13f743d3 100644
--- a/internal/caching/caches.go
+++ b/internal/caching/caches.go
@@ -23,16 +23,17 @@ import (
 // different implementations as long as they satisfy the Cache
 // interface.
 type Caches struct {
-	RoomVersions       Cache[string, gomatrixserverlib.RoomVersion]           // room ID -> room version
-	ServerKeys         Cache[string, gomatrixserverlib.PublicKeyLookupResult] // server name -> server keys
-	RoomServerRoomNIDs Cache[string, types.RoomNID]                           // room ID -> room NID
-	RoomServerRoomIDs  Cache[int64, string]                                   // room NID -> room ID
-	RoomServerEvents   Cache[int64, *gomatrixserverlib.Event]                 // event NID -> event
-	RoomInfos          Cache[string, *types.RoomInfo]                         // room ID -> room info
-	FederationPDUs     Cache[int64, *gomatrixserverlib.HeaderedEvent]         // queue NID -> PDU
-	FederationEDUs     Cache[int64, *gomatrixserverlib.EDU]                   // queue NID -> EDU
-	SpaceSummaryRooms  Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response
-	LazyLoading        Cache[lazyLoadingCacheKey, string]                     // composite key -> event ID
+	RoomVersions        Cache[string, gomatrixserverlib.RoomVersion]           // room ID -> room version
+	ServerKeys          Cache[string, gomatrixserverlib.PublicKeyLookupResult] // server name -> server keys
+	RoomServerRoomNIDs  Cache[string, types.RoomNID]                           // room ID -> room NID
+	RoomServerRoomIDs   Cache[types.RoomNID, string]                           // room NID -> room ID
+	RoomServerEvents    Cache[int64, *gomatrixserverlib.Event]                 // event NID -> event
+	RoomServerStateKeys Cache[types.EventStateKeyNID, string]                  // event NID -> event state key
+	RoomInfos           Cache[string, *types.RoomInfo]                         // room ID -> room info
+	FederationPDUs      Cache[int64, *gomatrixserverlib.HeaderedEvent]         // queue NID -> PDU
+	FederationEDUs      Cache[int64, *gomatrixserverlib.EDU]                   // queue NID -> EDU
+	SpaceSummaryRooms   Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response
+	LazyLoading         Cache[lazyLoadingCacheKey, string]                     // composite key -> event ID
 }
 
 // Cache is the interface that an implementation must satisfy.
@@ -44,7 +45,7 @@ type Cache[K keyable, T any] interface {
 
 type keyable interface {
 	// from https://github.com/dgraph-io/ristretto/blob/8e850b710d6df0383c375ec6a7beae4ce48fc8d5/z/z.go#L34
-	uint64 | string | []byte | byte | int | int32 | uint32 | int64 | lazyLoadingCacheKey
+	~uint64 | ~string | []byte | byte | ~int | ~int32 | ~uint32 | ~int64 | lazyLoadingCacheKey
 }
 
 type costable interface {
diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go
index 01c97ad55..fdbbb4d72 100644
--- a/internal/caching/impl_ristretto.go
+++ b/internal/caching/impl_ristretto.go
@@ -40,6 +40,7 @@ const (
 	federationEDUsCache
 	spaceSummaryRoomsCache
 	lazyLoadingCache
+	eventStateKeyCache
 )
 
 func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enablePrometheus bool) *Caches {
@@ -88,7 +89,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm
 			Prefix: roomNIDsCache,
 			MaxAge: maxAge,
 		},
-		RoomServerRoomIDs: &RistrettoCachePartition[int64, string]{ // room NID -> room ID
+		RoomServerRoomIDs: &RistrettoCachePartition[types.RoomNID, string]{ // room NID -> room ID
 			cache:  cache,
 			Prefix: roomIDsCache,
 			MaxAge: maxAge,
@@ -100,6 +101,11 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm
 				MaxAge: maxAge,
 			},
 		},
+		RoomServerStateKeys: &RistrettoCachePartition[types.EventStateKeyNID, string]{ // event NID -> event state key
+			cache:  cache,
+			Prefix: eventStateKeyCache,
+			MaxAge: maxAge,
+		},
 		RoomInfos: &RistrettoCachePartition[string, *types.RoomInfo]{ // room ID -> room info
 			cache:   cache,
 			Prefix:  roomInfosCache,
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index d8d5f67c8..2f9932ff8 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -72,7 +72,24 @@ func (d *Database) eventTypeNIDs(
 func (d *Database) EventStateKeys(
 	ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID,
 ) (map[types.EventStateKeyNID]string, error) {
-	return d.EventStateKeysTable.BulkSelectEventStateKey(ctx, nil, eventStateKeyNIDs)
+	result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs))
+	fetch := make([]types.EventStateKeyNID, 0, len(eventStateKeyNIDs))
+	for _, nid := range eventStateKeyNIDs {
+		if key, ok := d.Cache.GetEventStateKey(nid); ok {
+			result[nid] = key
+		} else {
+			fetch = append(fetch, nid)
+		}
+	}
+	fromDB, err := d.EventStateKeysTable.BulkSelectEventStateKey(ctx, nil, fetch)
+	if err != nil {
+		return nil, err
+	}
+	for nid, key := range fromDB {
+		result[nid] = key
+		d.Cache.StoreEventStateKey(nid, key)
+	}
+	return result, nil
 }
 
 func (d *Database) EventStateKeyNIDs(

From 84a779788360101a2353679cc2baa695fba7f9f2 Mon Sep 17 00:00:00 2001
From: Tulir Asokan <tulir@maunium.net>
Date: Tue, 19 Jul 2022 17:19:03 +0300
Subject: [PATCH 212/391] Explain how SRV works in Matrix and discourage using
 it (#2577)

* Explain how SRV works in Matrix and discourage using it

* Minor tweaks to formatting

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 docs/installation/2_domainname.md | 60 +++++++++++++++++++------------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md
index 0d4300eca..54064acb4 100644
--- a/docs/installation/2_domainname.md
+++ b/docs/installation/2_domainname.md
@@ -14,15 +14,18 @@ that take the format `@user:example.com`.
 For federation to work, the server name must be resolvable by other homeservers on the internet
 — that is, the domain must be registered and properly configured with the relevant DNS records.
 
-Matrix servers discover each other when federating using the following methods:
+Matrix servers usually discover each other when federating using the following methods:
 
-1. If a well-known delegation exists on `example.com`, use the path server from the
+1. If a well-known delegation exists on `example.com`, use the domain and port from the
    well-known file to connect to the remote homeserver;
-2. If a DNS SRV delegation exists on `example.com`, use the hostname and port from the DNS SRV
+2. If a DNS SRV delegation exists on `example.com`, use the IP address and port from the DNS SRV
    record to connect to the remote homeserver;
 3. If neither well-known or DNS SRV delegation are configured, attempt to connect to the remote
    homeserver by connecting to `example.com` port TCP/8448 using HTTPS.
 
+The exact details of how server name resolution works can be found in
+[the spec](https://spec.matrix.org/v1.3/server-server-api/#resolving-server-names).
+
 ## TLS certificates
 
 Matrix federation requires that valid TLS certificates are present on the domain. You must
@@ -51,17 +54,12 @@ you will be able to delegate from `example.com` to `matrix.example.com` so that
 
 Delegation can be performed in one of two ways:
 
-* **Well-known delegation**: A well-known text file is served over HTTPS on the domain name
-  that you want to use, pointing to your server on `matrix.example.com` port 8448;
-* **DNS SRV delegation**: A DNS SRV record is created on the domain name that you want to
-  use, pointing to your server on `matrix.example.com` port TCP/8448.
+* **Well-known delegation (preferred)**: A well-known text file is served over HTTPS on the domain
+  name that you want to use, pointing to your server on `matrix.example.com` port 8448;
+* **DNS SRV delegation (not recommended)**: See the SRV delegation section below for details.
 
-If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or DNS SRV
-delegation must refer to the hostname and port that the reverse proxy is listening on instead.
-
-Well-known delegation is typically easier to set up and usually preferred. However, you can use
-either or both methods to delegate. If you configure both methods of delegation, it is important
-that they both agree and refer to the same hostname and port.
+If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or delegation
+must refer to the hostname and port that the reverse proxy is listening on instead.
 
 ## Well-known delegation
 
@@ -74,20 +72,36 @@ and contain the following JSON document:
 
 ```json
 {
-    "m.server": "https://matrix.example.com:8448"
+    "m.server": "matrix.example.com:8448"
 }
 ```
 
+You can also serve `.well-known` with Dendrite itself by setting the `well_known_server_name` config
+option to the value you want for `m.server`. This is primarily useful if Dendrite is exposed on
+`example.com:443` and you don't want to set up a separate webserver just for serving the `.well-known`
+file.
+
+```yaml
+global:
+...
+   well_known_server_name: "example.com:443"
+```
+
 ## DNS SRV delegation
 
-Using DNS SRV delegation requires creating DNS SRV records on the `example.com` zone which
-refer to your Dendrite installation.
+This method is not recommended, as the behavior of SRV records in Matrix is rather unintuitive:
+SRV records will only change the IP address and port that other servers connect to, they won't
+affect the domain name. In technical terms, the `Host` header and TLS SNI of federation requests
+will still be `example.com` even if the SRV record points at `matrix.example.com`.
 
-Assuming that your Dendrite installation is listening for HTTPS connections at `matrix.example.com`
-port 8448, the DNS SRV record must have the following fields:
+In practice, this means that the server must be configured with valid TLS certificates for
+`example.com`, rather than `matrix.example.com` as one might intuitively expect. If there's a
+reverse proxy in between, the proxy configuration must be written as if it's `example.com`, as the
+proxy will never see the name `matrix.example.com` in incoming requests.
 
-* Name: `@` (or whichever term your DNS provider uses to signal the root)
-* Service: `_matrix`
-* Protocol: `_tcp`
-* Port: `8448`
-* Target: `matrix.example.com`
+This behavior also means that if `example.com` and `matrix.example.com` point at the same IP
+address, there is no reason to have a SRV record pointing at `matrix.example.com`. It can still
+be used to change the port number, but it won't do anything else.
+
+If you understand how SRV records work and still want to use them, the service name is `_matrix` and
+the protocol is `_tcp`.

From 9507966ebdbe0615e8d27be4b7a1d4440ada3e73 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 20 Jul 2022 12:39:06 +0200
Subject: [PATCH 213/391] Fix issue with membership event_nid being 0 (#2580)

---
 roomserver/storage/shared/membership_updater.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go
index ebfcef569..c79ca8995 100644
--- a/roomserver/storage/shared/membership_updater.go
+++ b/roomserver/storage/shared/membership_updater.go
@@ -105,8 +105,15 @@ func (u *MembershipUpdater) SetToInvite(event *gomatrixserverlib.Event) (bool, e
 		if err != nil {
 			return fmt.Errorf("u.d.InvitesTable.InsertInviteEvent: %w", err)
 		}
+
+		// Look up the NID of the invite event
+		nIDs, err := u.d.eventNIDs(u.ctx, u.txn, []string{event.EventID()}, false)
+		if err != nil {
+			return fmt.Errorf("u.d.EventNIDs: %w", err)
+		}
+
 		if u.membership != tables.MembershipStateInvite {
-			if inserted, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, 0, false); err != nil {
+			if inserted, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, nIDs[event.EventID()], false); err != nil {
 				return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err)
 			}
 		}

From 35ce551c8ffaef4d4bbfc5a4a8fd41a8b8ef2b27 Mon Sep 17 00:00:00 2001
From: Matt Holt <mholt@users.noreply.github.com>
Date: Wed, 20 Jul 2022 23:08:17 -0600
Subject: [PATCH 214/391] docs: Add build page; correct proxy info; fix Caddy
 example (#2579)

* Add build page; correct proxy info; fix Caddy example

* Improve Caddyfile example

* Apply review comments; add polylith Caddyfile
---
 docs/CONTRIBUTING.md                          |  2 +-
 docs/caddy/monolith/CaddyFile                 | 68 -------------------
 docs/caddy/monolith/Caddyfile                 | 57 ++++++++++++++++
 docs/caddy/polylith/Caddyfile                 | 66 ++++++++++++++++++
 ...ng_polylith.md => 10_starting_polylith.md} |  2 +-
 ...{10_optimisation.md => 11_optimisation.md} |  2 +-
 docs/installation/1_planning.md               | 11 +--
 docs/installation/2_domainname.md             | 34 +++++++---
 docs/installation/3_build.md                  | 38 +++++++++++
 .../{3_database.md => 4_database.md}          |  4 +-
 docs/installation/6_install_polylith.md       |  5 +-
 docs/installation/7_configuration.md          |  6 +-
 .../{4_signingkey.md => 8_signingkey.md}      |  2 +-
 ...ing_monolith.md => 9_starting_monolith.md} |  5 +-
 14 files changed, 209 insertions(+), 93 deletions(-)
 delete mode 100644 docs/caddy/monolith/CaddyFile
 create mode 100644 docs/caddy/monolith/Caddyfile
 create mode 100644 docs/caddy/polylith/Caddyfile
 rename docs/installation/{9_starting_polylith.md => 10_starting_polylith.md} (99%)
 rename docs/installation/{10_optimisation.md => 11_optimisation.md} (99%)
 create mode 100644 docs/installation/3_build.md
 rename docs/installation/{3_database.md => 4_database.md} (96%)
 rename docs/installation/{4_signingkey.md => 8_signingkey.md} (99%)
 rename docs/installation/{8_starting_monolith.md => 9_starting_monolith.md} (83%)

diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 5a89e6841..169224b9e 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -24,7 +24,7 @@ Unfortunately we can't accept contributions without it.
 
 ## Getting up and running
 
-See the [Installation](INSTALL.md) section for information on how to build an
+See the [Installation](installation) section for information on how to build an
 instance of Dendrite. You will likely need this in order to test your changes.
 
 ## Code style
diff --git a/docs/caddy/monolith/CaddyFile b/docs/caddy/monolith/CaddyFile
deleted file mode 100644
index cd93f9e10..000000000
--- a/docs/caddy/monolith/CaddyFile
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-    # debug
-    admin off
-    email example@example.com
-    default_sni example.com
-    # Debug endpoint
-    # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory	
-}
-
-#######################################################################
-#   Snippets
-#______________________________________________________________________
-
-(handle_errors_maintenance) {
-	handle_errors {
-		@maintenance expression {http.error.status_code} == 502
-		rewrite @maintenance maintenance.html
-		root * "/path/to/service/pages"
-		file_server
-	}
-}
-
-(matrix-well-known-header) {
-    # Headers
-    header Access-Control-Allow-Origin "*"
-    header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
-    header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
-    header Content-Type "application/json"
-}
-
-#######################################################################
-
-example.com {
-
-	# ...
-
-	handle /.well-known/matrix/server {
-		import matrix-well-known-header
-		respond `{ "m.server": "matrix.example.com:443" }` 200
-	}
-	
-	handle /.well-known/matrix/client {
-		import matrix-well-known-header
-		respond `{ "m.homeserver": { "base_url": "https://matrix.example.com" } }` 200
-	}
-
-	import handle_errors_maintenance
-}
-
-example.com:8448 {
-	# server<->server HTTPS traffic
-    reverse_proxy http://dendrite-host:8008
-}
-
-matrix.example.com {
-
-	handle /_matrix/* {
-		# client<->server HTTPS traffic
-		reverse_proxy  http://dendrite-host:8008
-	}
-
-	handle_path /* {
-		# Client webapp (Element SPA or ...)
-		file_server {
-			root /path/to/www/example.com/matrix-web-client/
-		}	
-	}
-}
diff --git a/docs/caddy/monolith/Caddyfile b/docs/caddy/monolith/Caddyfile
new file mode 100644
index 000000000..82567c4a6
--- /dev/null
+++ b/docs/caddy/monolith/Caddyfile
@@ -0,0 +1,57 @@
+# Sample Caddyfile for using Caddy in front of Dendrite.
+#
+# Customize email address and domain names.
+# Optional settings commented out.
+#
+# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST.
+# Documentation: https://caddyserver.com/docs/
+#
+# Bonus tip: If your IP address changes, use Caddy's
+# dynamic DNS plugin to update your DNS records to
+# point to your new IP automatically:
+# https://github.com/mholt/caddy-dynamicdns
+#
+
+
+# Global options block
+{
+	# In case there is a problem with your certificates.
+	# email example@example.com
+
+	# Turn off the admin endpoint if you don't need graceful config
+	# changes and/or are running untrusted code on your machine.
+	# admin off
+
+	# Enable this if your clients don't send ServerName in TLS handshakes.
+	# default_sni example.com
+
+	# Enable debug mode for verbose logging.
+	# debug
+
+	# Use Let's Encrypt's staging endpoint for testing.
+	# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
+
+	# If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
+	# else, enable these and put the alternate port numbers here.
+	# http_port  8080
+	# https_port 8443
+}
+
+# The server name of your matrix homeserver. This example shows
+# "well-known delegation" from the registered domain to a subdomain,
+# which is only needed if your server_name doesn't match your Matrix
+# homeserver URL (i.e. you can show users a vanity domain that looks
+# nice and is easy to remember but still have your Matrix server on
+# its own subdomain or hosted service).
+example.com {
+	header /.well-known/matrix/* Content-Type application/json
+	header /.well-known/matrix/* Access-Control-Allow-Origin *
+	respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
+	respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
+}
+
+# The actual domain name whereby your Matrix server is accessed.
+matrix.example.com {
+	# Set localhost:8008 to the address of your Dendrite server, if different
+	reverse_proxy /_matrix/* localhost:8008
+}
diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile
new file mode 100644
index 000000000..244e50e7e
--- /dev/null
+++ b/docs/caddy/polylith/Caddyfile
@@ -0,0 +1,66 @@
+# Sample Caddyfile for using Caddy in front of Dendrite.
+#
+# Customize email address and domain names.
+# Optional settings commented out.
+#
+# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST.
+# Documentation: https://caddyserver.com/docs/
+#
+# Bonus tip: If your IP address changes, use Caddy's
+# dynamic DNS plugin to update your DNS records to
+# point to your new IP automatically:
+# https://github.com/mholt/caddy-dynamicdns
+#
+
+
+# Global options block
+{
+	# In case there is a problem with your certificates.
+	# email example@example.com
+
+	# Turn off the admin endpoint if you don't need graceful config
+	# changes and/or are running untrusted code on your machine.
+	# admin off
+
+	# Enable this if your clients don't send ServerName in TLS handshakes.
+	# default_sni example.com
+
+	# Enable debug mode for verbose logging.
+	# debug
+
+	# Use Let's Encrypt's staging endpoint for testing.
+	# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
+
+	# If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
+	# else, enable these and put the alternate port numbers here.
+	# http_port  8080
+	# https_port 8443
+}
+
+# The server name of your matrix homeserver. This example shows
+# "well-known delegation" from the registered domain to a subdomain,
+# which is only needed if your server_name doesn't match your Matrix
+# homeserver URL (i.e. you can show users a vanity domain that looks
+# nice and is easy to remember but still have your Matrix server on
+# its own subdomain or hosted service).
+example.com {
+	header /.well-known/matrix/* Content-Type application/json
+	header /.well-known/matrix/* Access-Control-Allow-Origin *
+	respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
+	respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
+}
+
+# The actual domain name whereby your Matrix server is accessed.
+matrix.example.com {
+	# Change the end of each reverse_proxy line to the correct
+	# address for your various services.
+	@sync_api {
+		path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$
+	}
+	reverse_proxy @sync_api sync_api:8073
+
+	reverse_proxy /_matrix/client* client_api:8071
+	reverse_proxy /_matrix/federation* federation_api:8071
+	reverse_proxy /_matrix/key* federation_api:8071
+	reverse_proxy /_matrix/media* media_api:8071
+}
diff --git a/docs/installation/9_starting_polylith.md b/docs/installation/10_starting_polylith.md
similarity index 99%
rename from docs/installation/9_starting_polylith.md
rename to docs/installation/10_starting_polylith.md
index 228e52e85..0c2e2af2b 100644
--- a/docs/installation/9_starting_polylith.md
+++ b/docs/installation/10_starting_polylith.md
@@ -2,7 +2,7 @@
 title: Starting the polylith
 parent: Installation
 has_toc: true
-nav_order: 9
+nav_order: 10
 permalink: /installation/start/polylith
 ---
 
diff --git a/docs/installation/10_optimisation.md b/docs/installation/11_optimisation.md
similarity index 99%
rename from docs/installation/10_optimisation.md
rename to docs/installation/11_optimisation.md
index c19b7a75e..f2f67c947 100644
--- a/docs/installation/10_optimisation.md
+++ b/docs/installation/11_optimisation.md
@@ -2,7 +2,7 @@
 title: Optimise your installation
 parent: Installation
 has_toc: true
-nav_order: 10
+nav_order: 11
 permalink: /installation/start/optimisation
 ---
 
diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md
index d4f3d7052..3aa5b4d85 100644
--- a/docs/installation/1_planning.md
+++ b/docs/installation/1_planning.md
@@ -95,12 +95,13 @@ enabled.
 To do so, follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). JetStream must be enabled, either by passing the `-js` flag to `nats-server`,
 or by specifying the `store_dir` option in the the `jetstream` configuration.
 
-### Reverse proxy (polylith deployments)
+### Reverse proxy
 
-Polylith deployments require a reverse proxy, such as [NGINX](https://www.nginx.com) or
-[HAProxy](http://www.haproxy.org). Configuring those is not covered in this documentation,
-although a [sample configuration for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
-is provided.
+A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or
+[HAProxy](http://www.haproxy.org) is required for polylith deployments and is useful for monolith
+deployments. Configuring those is not covered in this documentation, although sample configurations
+for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and
+[NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided.
 
 ### Windows
 
diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md
index 54064acb4..7d7fc86bd 100644
--- a/docs/installation/2_domainname.md
+++ b/docs/installation/2_domainname.md
@@ -29,15 +29,23 @@ The exact details of how server name resolution works can be found in
 ## TLS certificates
 
 Matrix federation requires that valid TLS certificates are present on the domain. You must
-obtain certificates from a publicly accepted Certificate Authority (CA). [LetsEncrypt](https://letsencrypt.org)
-is an example of such a CA that can be used. Self-signed certificates are not suitable for
-federation and will typically not be accepted by other homeservers.
+obtain certificates from a publicly-trusted certificate authority (CA). [Let's Encrypt](https://letsencrypt.org)
+is a popular choice of CA because the certificates are publicly-trusted, free, and automated
+via the ACME protocol. (Self-signed certificates are not suitable for federation and will typically
+not be accepted by other homeservers.)
 
-A common practice to help ease the management of certificates is to install a reverse proxy in
-front of Dendrite which manages the TLS certificates and HTTPS proxying itself. Software such as
-[NGINX](https://www.nginx.com) and [HAProxy](http://www.haproxy.org) can be used for the task.
-Although the finer details of configuring these are not described here, you must reverse proxy
-all `/_matrix` paths to your Dendrite server.
+Automating the renewal of TLS certificates is best practice. There are many tools for this,
+but the simplest way to achieve TLS automation is to have your reverse proxy do it for you.
+[Caddy](https://caddyserver.com) is recommended as a production-grade reverse proxy with
+automatic TLS which is commonly used in front of Dendrite. It obtains and renews TLS certificates
+automatically and by default as long as your domain name is pointed at your server first.
+Although the finer details of [configuring Caddy](https://caddyserver.com/docs/) is not described
+here, in general, you must reverse proxy all `/_matrix` paths to your Dendrite server. For example,
+with Caddy:
+
+```
+reverse_proxy /_matrix/* localhost:8008
+```
 
 It is possible for the reverse proxy to listen on the standard HTTPS port TCP/443 so long as your
 domain delegation is configured to point to port TCP/443.
@@ -76,6 +84,16 @@ and contain the following JSON document:
 }
 ```
 
+For example, this can be done with the following Caddy config:
+
+```
+handle /.well-known/matrix/client {
+	header Content-Type application/json
+	header Access-Control-Allow-Origin *
+	respond `{"m.homeserver": {"base_url": "https://matrix.example.com:8448"}}`
+}
+```
+
 You can also serve `.well-known` with Dendrite itself by setting the `well_known_server_name` config
 option to the value you want for `m.server`. This is primarily useful if Dendrite is exposed on
 `example.com:443` and you don't want to set up a separate webserver just for serving the `.well-known`
diff --git a/docs/installation/3_build.md b/docs/installation/3_build.md
new file mode 100644
index 000000000..aed2080db
--- /dev/null
+++ b/docs/installation/3_build.md
@@ -0,0 +1,38 @@
+---
+title: Building Dendrite
+parent: Installation
+has_toc: true
+nav_order: 3
+permalink: /installation/build
+---
+
+# Build all Dendrite commands
+
+Dendrite has numerous utility commands in addition to the actual server binaries.
+Build them all from the root of the source repo with `build.sh` (Linux/Mac):
+
+```sh
+./build.sh
+```
+
+or `build.cmd` (Windows):
+
+```powershell
+build.cmd
+```
+
+The resulting binaries will be placed in the `bin` subfolder.
+
+# Installing as a monolith
+
+You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`:
+
+```sh
+go install ./cmd/dendrite-monolith-server
+```
+
+Alternatively, you can specify a custom path for the binary to be written to using `go build`:
+
+```sh
+go build -o /usr/local/bin/ ./cmd/dendrite-monolith-server
+```
diff --git a/docs/installation/3_database.md b/docs/installation/4_database.md
similarity index 96%
rename from docs/installation/3_database.md
rename to docs/installation/4_database.md
index f64fe9150..f6222a8d2 100644
--- a/docs/installation/3_database.md
+++ b/docs/installation/4_database.md
@@ -17,7 +17,9 @@ filenames in the Dendrite configuration file and start Dendrite. The databases w
 and populated automatically.
 
 Note that Dendrite **cannot share a single SQLite database across multiple components**. Each
-component must be configured with its own SQLite database filename.
+component must be configured with its own SQLite database filename. You will have to remove
+the `global.database` section from your Dendrite config and add it to each individual section
+instead in order to use SQLite.
 
 ### Connection strings
 
diff --git a/docs/installation/6_install_polylith.md b/docs/installation/6_install_polylith.md
index 375512f8f..ec4a77628 100644
--- a/docs/installation/6_install_polylith.md
+++ b/docs/installation/6_install_polylith.md
@@ -29,5 +29,6 @@ Polylith deployments require a reverse proxy in order to ensure that requests ar
 sent to the correct endpoint. You must ensure that a suitable reverse proxy is installed
 and configured.
 
-A [sample configuration file](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
-is provided for [NGINX](https://www.nginx.com).
+Sample configurations are provided
+for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy/polylith/Caddyfile)
+and [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf).
\ No newline at end of file
diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md
index e676afbe6..b1c747414 100644
--- a/docs/installation/7_configuration.md
+++ b/docs/installation/7_configuration.md
@@ -1,13 +1,13 @@
 ---
-title: Populate the configuration
+title: Configuring Dendrite
 parent: Installation
 nav_order: 7
 permalink: /installation/configuration
 ---
 
-# Populate the configuration
+# Configuring Dendrite
 
-The configuration file is used to configure Dendrite. Sample configuration files are
+A YAML configuration file is used to configure Dendrite. Sample configuration files are
 present in the top level of the Dendrite repository:
 
 * [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml)
diff --git a/docs/installation/4_signingkey.md b/docs/installation/8_signingkey.md
similarity index 99%
rename from docs/installation/4_signingkey.md
rename to docs/installation/8_signingkey.md
index 07dc485ff..323759a88 100644
--- a/docs/installation/4_signingkey.md
+++ b/docs/installation/8_signingkey.md
@@ -1,7 +1,7 @@
 ---
 title: Generating signing keys
 parent: Installation
-nav_order: 4
+nav_order: 8
 permalink: /installation/signingkeys
 ---
 
diff --git a/docs/installation/8_starting_monolith.md b/docs/installation/9_starting_monolith.md
similarity index 83%
rename from docs/installation/8_starting_monolith.md
rename to docs/installation/9_starting_monolith.md
index e0e7309d2..124477e73 100644
--- a/docs/installation/8_starting_monolith.md
+++ b/docs/installation/9_starting_monolith.md
@@ -15,8 +15,9 @@ you can start your Dendrite monolith deployment by starting the `dendrite-monoli
 ./dendrite-monolith-server -config /path/to/dendrite.yaml
 ```
 
-If you want to change the addresses or ports that Dendrite listens on, you
-can use the `-http-bind-address` and `-https-bind-address` command line arguments:
+By default, Dendrite will listen HTTP on port 8008. If you want to change the addresses
+or ports that Dendrite listens on, you can use the `-http-bind-address` and
+`-https-bind-address` command line arguments:
 
 ```bash
 ./dendrite-monolith-server -config /path/to/dendrite.yaml \

From a201b4400dd4fe99b4c391f94ac31884deccc12c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 22 Jul 2022 10:36:10 +0100
Subject: [PATCH 215/391] Bump tzinfo from 1.2.9 to 1.2.10 in /docs (#2584)

Bumps [tzinfo](https://github.com/tzinfo/tzinfo) from 1.2.9 to 1.2.10.
- [Release notes](https://github.com/tzinfo/tzinfo/releases)
- [Changelog](https://github.com/tzinfo/tzinfo/blob/master/CHANGES.md)
- [Commits](https://github.com/tzinfo/tzinfo/compare/v1.2.9...v1.2.10)

---
updated-dependencies:
- dependency-name: tzinfo
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 docs/Gemfile.lock | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
index e62aa4ce3..88d33ae84 100644
--- a/docs/Gemfile.lock
+++ b/docs/Gemfile.lock
@@ -233,6 +233,8 @@ GEM
     multipart-post (2.1.1)
     nokogiri (1.13.6-arm64-darwin)
       racc (~> 1.4)
+    nokogiri (1.13.6-x86_64-linux)
+      racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
@@ -263,7 +265,7 @@ GEM
     thread_safe (0.3.6)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
-    tzinfo (1.2.9)
+    tzinfo (1.2.10)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
@@ -273,11 +275,11 @@ GEM
 
 PLATFORMS
   arm64-darwin-21
+  x86_64-linux
 
 DEPENDENCIES
   github-pages (~> 226)
   jekyll-feed (~> 0.15.1)
-  minima (~> 2.5.1)
 
 BUNDLED WITH
    2.3.7

From f0c8a03649ab270bb15156a3cc47cae2aee3f36a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 22 Jul 2022 14:44:04 +0100
Subject: [PATCH 216/391] Membership updater refactoring (#2541)

* Membership updater refactoring

* Pass in membership state

* Use membership check rather than referring to state directly

* Delete irrelevant membership states

* We don't need the leave event after all

* Tweaks

* Put a log entry in that I might stand a chance of finding

* Be less panicky

* Tweak invite handling

* Don't freak if we can't find the event NID

* Use event NID from `types.Event`

* Clean up

* Better invite handling

* Placate the almighty linter

* Blacklist a Sytest which is otherwise fine under Complement for reasons I don't understand

* Fix the sytest after all (thanks @S7evinK for the spot)
---
 federationapi/routing/invite.go               |   2 -
 roomserver/internal/helpers/helpers.go        |  27 +++-
 roomserver/internal/input/input_membership.go | 117 +++++---------
 roomserver/internal/perform/perform_invite.go |  70 +++++----
 roomserver/internal/perform/perform_join.go   |  20 ++-
 roomserver/internal/perform/perform_leave.go  |   8 +-
 .../storage/postgres/membership_table.go      |  15 ++
 .../storage/shared/membership_updater.go      | 143 ++++--------------
 .../storage/sqlite3/membership_table.go       |  15 ++
 roomserver/storage/tables/interface.go        |   1 +
 syncapi/consumers/roomserver.go               |   8 +-
 11 files changed, 181 insertions(+), 245 deletions(-)

diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go
index cde87a0ac..4b795018c 100644
--- a/federationapi/routing/invite.go
+++ b/federationapi/routing/invite.go
@@ -26,7 +26,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
@@ -144,7 +143,6 @@ func processInvite(
 	// Check that the event is signed by the server sending the request.
 	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
 	if err != nil {
-		logrus.WithError(err).Errorf("XXX: invite.go")
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index e67bbfcaa..2653027e9 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -12,6 +12,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -21,14 +22,14 @@ import (
 // Move these to a more sensible place.
 
 func UpdateToInviteMembership(
-	mu *shared.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent,
+	mu *shared.MembershipUpdater, add *types.Event, updates []api.OutputEvent,
 	roomVersion gomatrixserverlib.RoomVersion,
 ) ([]api.OutputEvent, error) {
 	// We may have already sent the invite to the user, either because we are
 	// reprocessing this event, or because the we received this invite from a
 	// remote server via the federation invite API. In those cases we don't need
 	// to send the event.
-	needsSending, err := mu.SetToInvite(add)
+	needsSending, retired, err := mu.Update(tables.MembershipStateInvite, add)
 	if err != nil {
 		return nil, err
 	}
@@ -38,13 +39,23 @@ func UpdateToInviteMembership(
 		// room event stream. This ensures that the consumers only have to
 		// consider a single stream of events when determining whether a user
 		// is invited, rather than having to combine multiple streams themselves.
-		onie := api.OutputNewInviteEvent{
-			Event:       add.Headered(roomVersion),
-			RoomVersion: roomVersion,
-		}
 		updates = append(updates, api.OutputEvent{
-			Type:           api.OutputTypeNewInviteEvent,
-			NewInviteEvent: &onie,
+			Type: api.OutputTypeNewInviteEvent,
+			NewInviteEvent: &api.OutputNewInviteEvent{
+				Event:       add.Headered(roomVersion),
+				RoomVersion: roomVersion,
+			},
+		})
+	}
+	for _, eventID := range retired {
+		updates = append(updates, api.OutputEvent{
+			Type: api.OutputTypeRetireInviteEvent,
+			RetireInviteEvent: &api.OutputRetireInviteEvent{
+				EventID:          eventID,
+				Membership:       gomatrixserverlib.Join,
+				RetiredByEventID: add.EventID(),
+				TargetUserID:     *add.StateKey(),
+			},
 		})
 	}
 	return updates, nil
diff --git a/roomserver/internal/input/input_membership.go b/roomserver/internal/input/input_membership.go
index 3ce8791a3..28a54623b 100644
--- a/roomserver/internal/input/input_membership.go
+++ b/roomserver/internal/input/input_membership.go
@@ -21,6 +21,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/opentracing/opentracing-go"
@@ -60,20 +61,14 @@ func (r *Inputer) updateMemberships(
 	var updates []api.OutputEvent
 
 	for _, change := range changes {
-		var ae *gomatrixserverlib.Event
-		var re *gomatrixserverlib.Event
+		var ae *types.Event
+		var re *types.Event
 		targetUserNID := change.EventStateKeyNID
 		if change.removedEventNID != 0 {
-			ev, _ := helpers.EventMap(events).Lookup(change.removedEventNID)
-			if ev != nil {
-				re = ev.Event
-			}
+			re, _ = helpers.EventMap(events).Lookup(change.removedEventNID)
 		}
 		if change.addedEventNID != 0 {
-			ev, _ := helpers.EventMap(events).Lookup(change.addedEventNID)
-			if ev != nil {
-				ae = ev.Event
-			}
+			ae, _ = helpers.EventMap(events).Lookup(change.addedEventNID)
 		}
 		if updates, err = r.updateMembership(updater, targetUserNID, re, ae, updates); err != nil {
 			return nil, err
@@ -85,30 +80,27 @@ func (r *Inputer) updateMemberships(
 func (r *Inputer) updateMembership(
 	updater *shared.RoomUpdater,
 	targetUserNID types.EventStateKeyNID,
-	remove, add *gomatrixserverlib.Event,
+	remove, add *types.Event,
 	updates []api.OutputEvent,
 ) ([]api.OutputEvent, error) {
 	var err error
 	// Default the membership to Leave if no event was added or removed.
-	oldMembership := gomatrixserverlib.Leave
 	newMembership := gomatrixserverlib.Leave
-
-	if remove != nil {
-		oldMembership, err = remove.Membership()
-		if err != nil {
-			return nil, err
-		}
-	}
 	if add != nil {
 		newMembership, err = add.Membership()
 		if err != nil {
 			return nil, err
 		}
 	}
-	if oldMembership == newMembership && newMembership != gomatrixserverlib.Join {
-		// If the membership is the same then nothing changed and we can return
-		// immediately, unless it's a Join update (e.g. profile update).
-		return updates, nil
+
+	var targetLocal bool
+	if add != nil {
+		targetLocal = r.isLocalTarget(add)
+	}
+
+	mu, err := updater.MembershipUpdater(targetUserNID, targetLocal)
+	if err != nil {
+		return nil, err
 	}
 
 	// In an ideal world, we shouldn't ever have "add" be nil and "remove" be
@@ -120,17 +112,10 @@ func (r *Inputer) updateMembership(
 	// after a state reset, often thinking that the user was still joined to
 	// the room even though the room state said otherwise, and this would prevent
 	// the user from being able to attempt to rejoin the room without modifying
-	// the database. So instead what we'll do is we'll just update the membership
-	// table to say that the user is "leave" and we'll use the old event to
-	// avoid nil pointer exceptions on the code path that follows.
-	if add == nil {
-		add = remove
-		newMembership = gomatrixserverlib.Leave
-	}
-
-	mu, err := updater.MembershipUpdater(targetUserNID, r.isLocalTarget(add))
-	if err != nil {
-		return nil, err
+	// the database. So instead we're going to remove the membership from the
+	// database altogether, so that it doesn't create future problems.
+	if add == nil && remove != nil {
+		return nil, mu.Delete()
 	}
 
 	switch newMembership {
@@ -149,7 +134,7 @@ func (r *Inputer) updateMembership(
 	}
 }
 
-func (r *Inputer) isLocalTarget(event *gomatrixserverlib.Event) bool {
+func (r *Inputer) isLocalTarget(event *types.Event) bool {
 	isTargetLocalUser := false
 	if statekey := event.StateKey(); statekey != nil {
 		_, domain, _ := gomatrixserverlib.SplitID('@', *statekey)
@@ -159,81 +144,61 @@ func (r *Inputer) isLocalTarget(event *gomatrixserverlib.Event) bool {
 }
 
 func updateToJoinMembership(
-	mu *shared.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent,
+	mu *shared.MembershipUpdater, add *types.Event, updates []api.OutputEvent,
 ) ([]api.OutputEvent, error) {
-	// If the user is already marked as being joined, we call SetToJoin to update
-	// the event ID then we can return immediately. Retired is ignored as there
-	// is no invite event to retire.
-	if mu.IsJoin() {
-		_, err := mu.SetToJoin(add.Sender(), add.EventID(), true)
-		if err != nil {
-			return nil, err
-		}
-		return updates, nil
-	}
 	// When we mark a user as being joined we will invalidate any invites that
 	// are active for that user. We notify the consumers that the invites have
 	// been retired using a special event, even though they could infer this
 	// by studying the state changes in the room event stream.
-	retired, err := mu.SetToJoin(add.Sender(), add.EventID(), false)
+	_, retired, err := mu.Update(tables.MembershipStateJoin, add)
 	if err != nil {
 		return nil, err
 	}
 	for _, eventID := range retired {
-		orie := api.OutputRetireInviteEvent{
-			EventID:          eventID,
-			Membership:       gomatrixserverlib.Join,
-			RetiredByEventID: add.EventID(),
-			TargetUserID:     *add.StateKey(),
-		}
 		updates = append(updates, api.OutputEvent{
-			Type:              api.OutputTypeRetireInviteEvent,
-			RetireInviteEvent: &orie,
+			Type: api.OutputTypeRetireInviteEvent,
+			RetireInviteEvent: &api.OutputRetireInviteEvent{
+				EventID:          eventID,
+				Membership:       gomatrixserverlib.Join,
+				RetiredByEventID: add.EventID(),
+				TargetUserID:     *add.StateKey(),
+			},
 		})
 	}
 	return updates, nil
 }
 
 func updateToLeaveMembership(
-	mu *shared.MembershipUpdater, add *gomatrixserverlib.Event,
+	mu *shared.MembershipUpdater, add *types.Event,
 	newMembership string, updates []api.OutputEvent,
 ) ([]api.OutputEvent, error) {
-	// If the user is already neither joined, nor invited to the room then we
-	// can return immediately.
-	if mu.IsLeave() {
-		return updates, nil
-	}
 	// When we mark a user as having left we will invalidate any invites that
 	// are active for that user. We notify the consumers that the invites have
 	// been retired using a special event, even though they could infer this
 	// by studying the state changes in the room event stream.
-	retired, err := mu.SetToLeave(add.Sender(), add.EventID())
+	_, retired, err := mu.Update(tables.MembershipStateLeaveOrBan, add)
 	if err != nil {
 		return nil, err
 	}
 	for _, eventID := range retired {
-		orie := api.OutputRetireInviteEvent{
-			EventID:          eventID,
-			Membership:       newMembership,
-			RetiredByEventID: add.EventID(),
-			TargetUserID:     *add.StateKey(),
-		}
 		updates = append(updates, api.OutputEvent{
-			Type:              api.OutputTypeRetireInviteEvent,
-			RetireInviteEvent: &orie,
+			Type: api.OutputTypeRetireInviteEvent,
+			RetireInviteEvent: &api.OutputRetireInviteEvent{
+				EventID:          eventID,
+				Membership:       newMembership,
+				RetiredByEventID: add.EventID(),
+				TargetUserID:     *add.StateKey(),
+			},
 		})
 	}
 	return updates, nil
 }
 
 func updateToKnockMembership(
-	mu *shared.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent,
+	mu *shared.MembershipUpdater, add *types.Event, updates []api.OutputEvent,
 ) ([]api.OutputEvent, error) {
-	if mu.IsLeave() {
-		_, err := mu.SetToKnock(add)
-		if err != nil {
-			return nil, err
-		}
+	if _, _, err := mu.Update(tables.MembershipStateKnock, add); err != nil {
+		return nil, err
 	}
 	return updates, nil
 }
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index 644c954b6..fce529305 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -39,11 +39,13 @@ type Inviter struct {
 	Inputer *input.Inputer
 }
 
+// nolint:gocyclo
 func (r *Inviter) PerformInvite(
 	ctx context.Context,
 	req *api.PerformInviteRequest,
 	res *api.PerformInviteResponse,
 ) ([]api.OutputEvent, error) {
+	var outputUpdates []api.OutputEvent
 	event := req.Event
 	if event.StateKey() == nil {
 		return nil, fmt.Errorf("invite must be a state event")
@@ -66,6 +68,13 @@ func (r *Inviter) PerformInvite(
 	}
 	isTargetLocal := domain == r.Cfg.Matrix.ServerName
 	isOriginLocal := event.Origin() == r.Cfg.Matrix.ServerName
+	if !isOriginLocal && !isTargetLocal {
+		res.Error = &api.PerformError{
+			Code: api.PerformErrorBadRequest,
+			Msg:  "The invite must be either from or to a local user",
+		}
+		return nil, nil
+	}
 
 	logger := util.GetLogger(ctx).WithFields(map[string]interface{}{
 		"inviter":  event.Sender(),
@@ -97,6 +106,34 @@ func (r *Inviter) PerformInvite(
 		}
 	}
 
+	updateMembershipTableManually := func() ([]api.OutputEvent, error) {
+		var updater *shared.MembershipUpdater
+		if updater, err = r.DB.MembershipUpdater(ctx, roomID, targetUserID, isTargetLocal, req.RoomVersion); err != nil {
+			return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err)
+		}
+		outputUpdates, err = helpers.UpdateToInviteMembership(updater, &types.Event{
+			EventNID: 0,
+			Event:    event.Unwrap(),
+		}, outputUpdates, req.Event.RoomVersion)
+		if err != nil {
+			return nil, fmt.Errorf("updateToInviteMembership: %w", err)
+		}
+		if err = updater.Commit(); err != nil {
+			return nil, fmt.Errorf("updater.Commit: %w", err)
+		}
+		logger.Debugf("updated membership to invite and sending invite OutputEvent")
+		return outputUpdates, nil
+	}
+
+	if (info == nil || info.IsStub) && !isOriginLocal && isTargetLocal {
+		// The invite came in over federation for a room that we don't know about
+		// yet. We need to handle this a bit differently to most invites because
+		// we don't know the room state, therefore the roomserver can't process
+		// an input event. Instead we will update the membership table with the
+		// new invite and generate an output event.
+		return updateMembershipTableManually()
+	}
+
 	var isAlreadyJoined bool
 	if info != nil {
 		_, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
@@ -140,31 +177,13 @@ func (r *Inviter) PerformInvite(
 		return nil, nil
 	}
 
+	// If the invite originated remotely then we can't send an
+	// InputRoomEvent for the invite as it will never pass auth checks
+	// due to lacking room state, but we still need to tell the client
+	// about the invite so we can accept it, hence we return an output
+	// event to send to the Sync API.
 	if !isOriginLocal {
-		// The invite originated over federation. Process the membership
-		// update, which will notify the sync API etc about the incoming
-		// invite. We do NOT send an InputRoomEvent for the invite as it
-		// will never pass auth checks due to lacking room state, but we
-		// still need to tell the client about the invite so we can accept
-		// it, hence we return an output event to send to the sync api.
-		var updater *shared.MembershipUpdater
-		updater, err = r.DB.MembershipUpdater(ctx, roomID, targetUserID, isTargetLocal, req.RoomVersion)
-		if err != nil {
-			return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err)
-		}
-
-		unwrapped := event.Unwrap()
-		var outputUpdates []api.OutputEvent
-		outputUpdates, err = helpers.UpdateToInviteMembership(updater, unwrapped, nil, req.Event.RoomVersion)
-		if err != nil {
-			return nil, fmt.Errorf("updateToInviteMembership: %w", err)
-		}
-
-		if err = updater.Commit(); err != nil {
-			return nil, fmt.Errorf("updater.Commit: %w", err)
-		}
-		logger.Debugf("updated membership to invite and sending invite OutputEvent")
-		return outputUpdates, nil
+		return updateMembershipTableManually()
 	}
 
 	// The invite originated locally. Therefore we have a responsibility to
@@ -229,12 +248,11 @@ func (r *Inviter) PerformInvite(
 			Code: api.PerformErrorNotAllowed,
 		}
 		logger.WithError(err).WithField("event_id", event.EventID()).Error("r.InputRoomEvents failed")
-		return nil, nil
 	}
 
 	// Don't notify the sync api of this event in the same way as a federated invite so the invitee
 	// gets the invite, as the roomserver will do this when it processes the m.room.member invite.
-	return nil, nil
+	return outputUpdates, nil
 }
 
 func buildInviteStrippedState(
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index c9e839198..1445b4088 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -268,21 +268,19 @@ func (r *Joiner) performJoinRoomByID(
 	case nil:
 		// The room join is local. Send the new join event into the
 		// roomserver. First of all check that the user isn't already
-		// a member of the room.
-		alreadyJoined := false
-		for _, se := range buildRes.StateEvents {
-			if !se.StateKeyEquals(userID) {
-				continue
-			}
-			if membership, merr := se.Membership(); merr == nil {
-				alreadyJoined = (membership == gomatrixserverlib.Join)
-				break
-			}
+		// a member of the room. This is best-effort (as in we won't
+		// fail if we can't find the existing membership) because there
+		// is really no harm in just sending another membership event.
+		membershipReq := &api.QueryMembershipForUserRequest{
+			RoomID: req.RoomIDOrAlias,
+			UserID: userID,
 		}
+		membershipRes := &api.QueryMembershipForUserResponse{}
+		_ = r.Queryer.QueryMembershipForUser(ctx, membershipReq, membershipRes)
 
 		// If we haven't already joined the room then send an event
 		// into the room changing our membership status.
-		if !alreadyJoined {
+		if !membershipRes.RoomExists || !membershipRes.IsInRoom {
 			inputReq := rsAPI.InputRoomEventsRequest{
 				InputRoomEvents: []rsAPI.InputRoomEvent{
 					{
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index c5b62ac00..56e7240d0 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -228,14 +228,14 @@ func (r *Leaver) performFederatedRejectInvite(
 		util.GetLogger(ctx).WithError(err).Errorf("failed to get MembershipUpdater, still retiring invite event")
 	}
 	if updater != nil {
-		if _, err = updater.SetToLeave(req.UserID, eventID); err != nil {
-			util.GetLogger(ctx).WithError(err).Errorf("failed to set membership to leave, still retiring invite event")
+		if err = updater.Delete(); err != nil {
+			util.GetLogger(ctx).WithError(err).Errorf("failed to delete membership, still retiring invite event")
 			if err = updater.Rollback(); err != nil {
-				util.GetLogger(ctx).WithError(err).Errorf("failed to rollback membership leave, still retiring invite event")
+				util.GetLogger(ctx).WithError(err).Errorf("failed to rollback deleting membership, still retiring invite event")
 			}
 		} else {
 			if err = updater.Commit(); err != nil {
-				util.GetLogger(ctx).WithError(err).Errorf("failed to commit membership update, still retiring invite event")
+				util.GetLogger(ctx).WithError(err).Errorf("failed to commit deleting membership, still retiring invite event")
 			}
 		}
 	}
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index ce626ad1d..ee457393a 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -118,6 +118,9 @@ const updateMembershipForgetRoom = "" +
 	"UPDATE roomserver_membership SET forgotten = $3" +
 	" WHERE room_nid = $1 AND target_nid = $2"
 
+const deleteMembershipSQL = "" +
+	"DELETE FROM roomserver_membership WHERE room_nid = $1 AND target_nid = $2"
+
 const selectRoomsWithMembershipSQL = "" +
 	"SELECT room_nid FROM roomserver_membership WHERE membership_nid = $1 AND target_nid = $2 and forgotten = false"
 
@@ -165,6 +168,7 @@ type membershipStatements struct {
 	updateMembershipForgetRoomStmt                  *sql.Stmt
 	selectLocalServerInRoomStmt                     *sql.Stmt
 	selectServerInRoomStmt                          *sql.Stmt
+	deleteMembershipStmt                            *sql.Stmt
 }
 
 func CreateMembershipTable(db *sql.DB) error {
@@ -191,6 +195,7 @@ func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
 		{&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL},
 		{&s.selectServerInRoomStmt, selectServerInRoomSQL},
+		{&s.deleteMembershipStmt, deleteMembershipSQL},
 	}.Prepare(db)
 }
 
@@ -412,3 +417,13 @@ func (s *membershipStatements) SelectServerInRoom(
 	}
 	return roomNID == nid, nil
 }
+
+func (s *membershipStatements) DeleteMembership(
+	ctx context.Context, txn *sql.Tx,
+	roomNID types.RoomNID, targetUserNID types.EventStateKeyNID,
+) error {
+	_, err := sqlutil.TxStmt(txn, s.deleteMembershipStmt).ExecContext(
+		ctx, roomNID, targetUserNID,
+	)
+	return err
+}
diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go
index c79ca8995..07fb697f9 100644
--- a/roomserver/storage/shared/membership_updater.go
+++ b/roomserver/storage/shared/membership_updater.go
@@ -15,7 +15,7 @@ type MembershipUpdater struct {
 	d             *Database
 	roomNID       types.RoomNID
 	targetUserNID types.EventStateKeyNID
-	membership    tables.MembershipState
+	oldMembership tables.MembershipState
 }
 
 func NewMembershipUpdater(
@@ -30,7 +30,6 @@ func NewMembershipUpdater(
 		if err != nil {
 			return err
 		}
-
 		targetUserNID, err = d.assignStateKeyNID(ctx, targetUserID)
 		if err != nil {
 			return err
@@ -73,146 +72,62 @@ func (d *Database) membershipUpdaterTxn(
 
 // IsInvite implements types.MembershipUpdater
 func (u *MembershipUpdater) IsInvite() bool {
-	return u.membership == tables.MembershipStateInvite
+	return u.oldMembership == tables.MembershipStateInvite
 }
 
 // IsJoin implements types.MembershipUpdater
 func (u *MembershipUpdater) IsJoin() bool {
-	return u.membership == tables.MembershipStateJoin
+	return u.oldMembership == tables.MembershipStateJoin
 }
 
 // IsLeave implements types.MembershipUpdater
 func (u *MembershipUpdater) IsLeave() bool {
-	return u.membership == tables.MembershipStateLeaveOrBan
+	return u.oldMembership == tables.MembershipStateLeaveOrBan
 }
 
 // IsKnock implements types.MembershipUpdater
 func (u *MembershipUpdater) IsKnock() bool {
-	return u.membership == tables.MembershipStateKnock
+	return u.oldMembership == tables.MembershipStateKnock
 }
 
-// SetToInvite implements types.MembershipUpdater
-func (u *MembershipUpdater) SetToInvite(event *gomatrixserverlib.Event) (bool, error) {
-	var inserted bool
-	err := u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
+func (u *MembershipUpdater) Delete() error {
+	if _, err := u.d.InvitesTable.UpdateInviteRetired(u.ctx, u.txn, u.roomNID, u.targetUserNID); err != nil {
+		return err
+	}
+	return u.d.MembershipTable.DeleteMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID)
+}
+
+func (u *MembershipUpdater) Update(newMembership tables.MembershipState, event *types.Event) (bool, []string, error) {
+	var inserted bool    // Did the query result in a membership change?
+	var retired []string // Did we retire any updates in the process?
+	return inserted, retired, u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
 		senderUserNID, err := u.d.assignStateKeyNID(u.ctx, event.Sender())
 		if err != nil {
 			return fmt.Errorf("u.d.AssignStateKeyNID: %w", err)
 		}
-		inserted, err = u.d.InvitesTable.InsertInviteEvent(
-			u.ctx, u.txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(),
-		)
+		inserted, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, newMembership, event.EventNID, false)
 		if err != nil {
-			return fmt.Errorf("u.d.InvitesTable.InsertInviteEvent: %w", err)
+			return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err)
 		}
-
-		// Look up the NID of the invite event
-		nIDs, err := u.d.eventNIDs(u.ctx, u.txn, []string{event.EventID()}, false)
-		if err != nil {
-			return fmt.Errorf("u.d.EventNIDs: %w", err)
+		if !inserted {
+			return nil
 		}
-
-		if u.membership != tables.MembershipStateInvite {
-			if inserted, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, nIDs[event.EventID()], false); err != nil {
-				return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err)
+		switch {
+		case u.oldMembership != tables.MembershipStateInvite && newMembership == tables.MembershipStateInvite:
+			inserted, err = u.d.InvitesTable.InsertInviteEvent(
+				u.ctx, u.txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(),
+			)
+			if err != nil {
+				return fmt.Errorf("u.d.InvitesTable.InsertInviteEvent: %w", err)
 			}
-		}
-		return nil
-	})
-	return inserted, err
-}
-
-// SetToJoin implements types.MembershipUpdater
-func (u *MembershipUpdater) SetToJoin(senderUserID string, eventID string, isUpdate bool) ([]string, error) {
-	var inviteEventIDs []string
-
-	err := u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
-		senderUserNID, err := u.d.assignStateKeyNID(u.ctx, senderUserID)
-		if err != nil {
-			return fmt.Errorf("u.d.AssignStateKeyNID: %w", err)
-		}
-
-		// If this is a join event update, there is no invite to update
-		if !isUpdate {
-			inviteEventIDs, err = u.d.InvitesTable.UpdateInviteRetired(
+		case u.oldMembership == tables.MembershipStateInvite && newMembership != tables.MembershipStateInvite:
+			retired, err = u.d.InvitesTable.UpdateInviteRetired(
 				u.ctx, u.txn, u.roomNID, u.targetUserNID,
 			)
 			if err != nil {
 				return fmt.Errorf("u.d.InvitesTables.UpdateInviteRetired: %w", err)
 			}
 		}
-
-		// Look up the NID of the new join event
-		nIDs, err := u.d.eventNIDs(u.ctx, u.txn, []string{eventID}, false)
-		if err != nil {
-			return fmt.Errorf("u.d.EventNIDs: %w", err)
-		}
-
-		if u.membership != tables.MembershipStateJoin || isUpdate {
-			if _, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateJoin, nIDs[eventID], false); err != nil {
-				return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err)
-			}
-		}
-
 		return nil
 	})
-
-	return inviteEventIDs, err
-}
-
-// SetToLeave implements types.MembershipUpdater
-func (u *MembershipUpdater) SetToLeave(senderUserID string, eventID string) ([]string, error) {
-	var inviteEventIDs []string
-
-	err := u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
-		senderUserNID, err := u.d.assignStateKeyNID(u.ctx, senderUserID)
-		if err != nil {
-			return fmt.Errorf("u.d.AssignStateKeyNID: %w", err)
-		}
-		inviteEventIDs, err = u.d.InvitesTable.UpdateInviteRetired(
-			u.ctx, u.txn, u.roomNID, u.targetUserNID,
-		)
-		if err != nil {
-			return fmt.Errorf("u.d.InvitesTable.updateInviteRetired: %w", err)
-		}
-
-		// Look up the NID of the new leave event
-		nIDs, err := u.d.eventNIDs(u.ctx, u.txn, []string{eventID}, false)
-		if err != nil {
-			return fmt.Errorf("u.d.EventNIDs: %w", err)
-		}
-
-		if u.membership != tables.MembershipStateLeaveOrBan {
-			if _, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateLeaveOrBan, nIDs[eventID], false); err != nil {
-				return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err)
-			}
-		}
-
-		return nil
-	})
-	return inviteEventIDs, err
-}
-
-// SetToKnock implements types.MembershipUpdater
-func (u *MembershipUpdater) SetToKnock(event *gomatrixserverlib.Event) (bool, error) {
-	var inserted bool
-	err := u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
-		senderUserNID, err := u.d.assignStateKeyNID(u.ctx, event.Sender())
-		if err != nil {
-			return fmt.Errorf("u.d.AssignStateKeyNID: %w", err)
-		}
-		if u.membership != tables.MembershipStateKnock {
-			// Look up the NID of the new knock event
-			nIDs, err := u.d.eventNIDs(u.ctx, u.txn, []string{event.EventID()}, false)
-			if err != nil {
-				return fmt.Errorf("u.d.EventNIDs: %w", err)
-			}
-
-			if inserted, err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateKnock, nIDs[event.EventID()], false); err != nil {
-				return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err)
-			}
-		}
-		return nil
-	})
-	return inserted, err
 }
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index 570d3919c..00a9bd4f9 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -125,6 +125,9 @@ const selectServerInRoomSQL = "" +
 	" JOIN roomserver_event_state_keys ON roomserver_membership.target_nid = roomserver_event_state_keys.event_state_key_nid" +
 	" WHERE membership_nid = $1 AND room_nid = $2 AND event_state_key LIKE '%:' || $3 LIMIT 1"
 
+const deleteMembershipSQL = "" +
+	"DELETE FROM roomserver_membership WHERE room_nid = $1 AND target_nid = $2"
+
 type membershipStatements struct {
 	db                                              *sql.DB
 	insertMembershipStmt                            *sql.Stmt
@@ -140,6 +143,7 @@ type membershipStatements struct {
 	updateMembershipForgetRoomStmt                  *sql.Stmt
 	selectLocalServerInRoomStmt                     *sql.Stmt
 	selectServerInRoomStmt                          *sql.Stmt
+	deleteMembershipStmt                            *sql.Stmt
 }
 
 func CreateMembershipTable(db *sql.DB) error {
@@ -166,6 +170,7 @@ func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
 		{&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL},
 		{&s.selectServerInRoomStmt, selectServerInRoomSQL},
+		{&s.deleteMembershipStmt, deleteMembershipSQL},
 	}.Prepare(db)
 }
 
@@ -383,3 +388,13 @@ func (s *membershipStatements) SelectServerInRoom(ctx context.Context, txn *sql.
 	}
 	return roomNID == nid, nil
 }
+
+func (s *membershipStatements) DeleteMembership(
+	ctx context.Context, txn *sql.Tx,
+	roomNID types.RoomNID, targetUserNID types.EventStateKeyNID,
+) error {
+	_, err := sqlutil.TxStmt(txn, s.deleteMembershipStmt).ExecContext(
+		ctx, roomNID, targetUserNID,
+	)
+	return err
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 116e11c4e..d228257b7 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -133,6 +133,7 @@ type Membership interface {
 	UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error
 	SelectLocalServerInRoom(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) (bool, error)
 	SelectServerInRoom(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error)
+	DeleteMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) error
 }
 
 type Published interface {
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 328e1e9dd..f77b1673b 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -365,7 +365,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent(
 			"event":      string(msg.Event.JSON()),
 			"pdupos":     pduPos,
 			log.ErrorKey: err,
-		}).Panicf("roomserver output log: write invite failure")
+		}).Errorf("roomserver output log: write invite failure")
 		return
 	}
 
@@ -385,7 +385,7 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent(
 		log.WithFields(log.Fields{
 			"event_id":   msg.EventID,
 			log.ErrorKey: err,
-		}).Panicf("roomserver output log: remove invite failure")
+		}).Errorf("roomserver output log: remove invite failure")
 		return
 	}
 
@@ -403,7 +403,7 @@ func (s *OutputRoomEventConsumer) onNewPeek(
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
 			log.ErrorKey: err,
-		}).Panicf("roomserver output log: write peek failure")
+		}).Errorf("roomserver output log: write peek failure")
 		return
 	}
 
@@ -422,7 +422,7 @@ func (s *OutputRoomEventConsumer) onRetirePeek(
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
 			log.ErrorKey: err,
-		}).Panicf("roomserver output log: write peek failure")
+		}).Errorf("roomserver output log: write peek failure")
 		return
 	}
 

From c7d978274d1e9606574106a9fee63ff555604d1d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 22 Jul 2022 19:43:48 +0100
Subject: [PATCH 217/391] Try to fix HTTP 500s on `/members` (#2581)

---
 roomserver/internal/query/query.go                 |  7 +++++++
 roomserver/storage/postgres/membership_table.go    | 10 +++++-----
 roomserver/storage/sqlite3/membership_table.go     | 10 +++++-----
 roomserver/storage/tables/membership_table_test.go |  3 +++
 4 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index da1b32530..f652a8ea3 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -16,6 +16,7 @@ package query
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -225,6 +226,9 @@ func (r *Queryer) QueryMembershipsForRoom(
 		var eventNIDs []types.EventNID
 		eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, request.LocalOnly)
 		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil
+			}
 			return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err)
 		}
 		events, err = r.DB.Events(ctx, eventNIDs)
@@ -260,6 +264,9 @@ func (r *Queryer) QueryMembershipsForRoom(
 		var eventNIDs []types.EventNID
 		eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, request.JoinedOnly, false)
 		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil
+			}
 			return err
 		}
 
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index ee457393a..eaaea22a1 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -86,24 +86,24 @@ const insertMembershipSQL = "" +
 
 const selectMembershipFromRoomAndTargetSQL = "" +
 	"SELECT membership_nid, event_nid, forgotten FROM roomserver_membership" +
-	" WHERE room_nid = $1 AND target_nid = $2"
+	" WHERE room_nid = $1 AND event_nid != 0 AND target_nid = $2"
 
 const selectMembershipsFromRoomAndMembershipSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1 AND membership_nid = $2 and forgotten = false"
+	" WHERE room_nid = $1 AND event_nid != 0 AND membership_nid = $2 and forgotten = false"
 
 const selectLocalMembershipsFromRoomAndMembershipSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1 AND membership_nid = $2" +
+	" WHERE room_nid = $1 AND event_nid != 0 AND membership_nid = $2" +
 	" AND target_local = true and forgotten = false"
 
 const selectMembershipsFromRoomSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1 and forgotten = false"
+	" WHERE room_nid = $1 AND event_nid != 0 and forgotten = false"
 
 const selectLocalMembershipsFromRoomSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1" +
+	" WHERE room_nid = $1 AND event_nid != 0" +
 	" AND target_local = true and forgotten = false"
 
 const selectMembershipForUpdateSQL = "" +
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index 00a9bd4f9..9dda82fb2 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -62,24 +62,24 @@ const insertMembershipSQL = "" +
 
 const selectMembershipFromRoomAndTargetSQL = "" +
 	"SELECT membership_nid, event_nid, forgotten FROM roomserver_membership" +
-	" WHERE room_nid = $1 AND target_nid = $2"
+	" WHERE room_nid = $1 AND event_nid != 0 AND target_nid = $2"
 
 const selectMembershipsFromRoomAndMembershipSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1 AND membership_nid = $2 and forgotten = false"
+	" WHERE room_nid = $1 AND event_nid != 0 AND membership_nid = $2 and forgotten = false"
 
 const selectLocalMembershipsFromRoomAndMembershipSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1 AND membership_nid = $2" +
+	" WHERE room_nid = $1 AND event_nid != 0 AND membership_nid = $2" +
 	" AND target_local = true and forgotten = false"
 
 const selectMembershipsFromRoomSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1 and forgotten = false"
+	" WHERE room_nid = $1 AND event_nid != 0 and forgotten = false"
 
 const selectLocalMembershipsFromRoomSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1" +
+	" WHERE room_nid = $1 AND event_nid != 0" +
 	" AND target_local = true and forgotten = false"
 
 const selectMembershipForUpdateSQL = "" +
diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go
index 14e8ce50a..f789ef4ac 100644
--- a/roomserver/storage/tables/membership_table_test.go
+++ b/roomserver/storage/tables/membership_table_test.go
@@ -60,6 +60,9 @@ func TestMembershipTable(t *testing.T) {
 			// This inserts a left user to the room
 			err = tab.InsertMembership(ctx, nil, 1, stateKeyNID, true)
 			assert.NoError(t, err)
+			// We must update the membership with a non-zero event NID or it will get filtered out in later queries
+			_, err = tab.UpdateMembership(ctx, nil, 1, stateKeyNID, userNIDs[0], tables.MembershipStateLeaveOrBan, 1, false)
+			assert.NoError(t, err)
 		}
 
 		// ... so this should be false

From 081f5e722677fc0306934a814c557b57d56c2e0d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 25 Jul 2022 11:39:22 +0200
Subject: [PATCH 218/391] Update database migrations, remove goose (#2264)

* Add new db migration

* Update migrations
Remove goose

* Add possibility to test direct upgrades

* Try to fix WASM test

* Add checks for specific migrations

* Remove AddMigration
Use WithTransaction
Add Dendrite version to table

* Fix linter issues

* Update tests

* Update comments, outdent if

* Namespace migrations

* Add direct upgrade tests, skipping over one version

* Split migrations

* Update go version in CI

* Fix copy&paste mistake

* Use contexts in migrations

Co-authored-by: kegsay <kegan@matrix.org>
Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 .github/workflows/dendrite.yml                |  27 ++-
 cmd/dendrite-upgrade-tests/main.go            |   8 +-
 cmd/goose/README.md                           | 109 ---------
 cmd/goose/main.go                             | 154 -------------
 .../postgres/deltas/2021020411080000_rooms.go |  16 +-
 federationapi/storage/postgres/storage.go     |  10 +-
 .../sqlite3/deltas/2021020411080000_rooms.go  |  16 +-
 federationapi/storage/sqlite3/storage.go      |  10 +-
 go.mod                                        |   1 -
 go.sum                                        |   2 -
 internal/sqlutil/migrate.go                   | 218 +++++++++---------
 internal/sqlutil/migrate_test.go              | 112 +++++++++
 .../postgres/cross_signing_sigs_table.go      |  11 +
 .../deltas/2022012016470000_key_changes.go    |  26 +--
 .../deltas/2022042612000000_xsigning_idx.go   |  15 +-
 .../storage/postgres/key_changes_table.go     |  20 +-
 keyserver/storage/postgres/storage.go         |   7 -
 .../sqlite3/cross_signing_sigs_table.go       |  10 +
 .../deltas/2022012016470000_key_changes.go    |  24 +-
 .../deltas/2022042612000000_xsigning_idx.go   |  15 +-
 .../storage/sqlite3/key_changes_table.go      |  20 +-
 keyserver/storage/sqlite3/storage.go          |   7 -
 .../20201028212440_add_forgotten_column.go    |  21 +-
 .../2021041615092700_state_blocks_refactor.go |  74 +++---
 .../storage/postgres/membership_table.go      |  11 +-
 roomserver/storage/postgres/storage.go        |  25 +-
 .../20201028212440_add_forgotten_column.go    |  21 +-
 .../2021041615092700_state_blocks_refactor.go |  69 +++---
 .../storage/sqlite3/membership_table.go       |  11 +-
 roomserver/storage/sqlite3/storage.go         |  24 +-
 .../postgres/current_room_state_table.go      |  12 +
 .../deltas/20201211125500_sequences.go        |  21 +-
 .../20210112130000_sendtodevice_sentcolumn.go |  15 +-
 ...2061412000000_history_visibility_column.go |  24 +-
 .../postgres/output_room_events_table.go      |  12 +
 syncapi/storage/postgres/receipt_table.go     |  10 +
 .../storage/postgres/send_to_device_table.go  |  10 +
 syncapi/storage/postgres/syncserver.go        |  31 +--
 .../sqlite3/current_room_state_table.go       |  12 +
 .../deltas/20201211125500_sequences.go        |  21 +-
 .../20210112130000_sendtodevice_sentcolumn.go |  15 +-
 ...2061412000000_history_visibility_column.go |  32 +--
 .../sqlite3/output_room_events_table.go       |  12 +
 syncapi/storage/sqlite3/receipt_table.go      |  10 +
 .../storage/sqlite3/send_to_device_table.go   |  10 +
 syncapi/storage/sqlite3/syncserver.go         |  35 +--
 userapi/storage/postgres/accounts_table.go    |  18 ++
 .../deltas/20200929203058_is_active.go        |  22 +-
 .../deltas/20201001204705_last_seen_ts_ip.go  |  15 +-
 .../2022021013023800_add_account_type.go      |  15 +-
 userapi/storage/postgres/devices_table.go     |  10 +
 userapi/storage/postgres/storage.go           |  14 --
 userapi/storage/sqlite3/accounts_table.go     |  18 ++
 .../deltas/20200929203058_is_active.go        |  22 +-
 .../deltas/20201001204705_last_seen_ts_ip.go  |  15 +-
 .../2022021012490600_add_account_type.go      |  21 +-
 userapi/storage/sqlite3/devices_table.go      |  10 +
 userapi/storage/sqlite3/storage.go            |  17 --
 58 files changed, 734 insertions(+), 839 deletions(-)
 delete mode 100644 cmd/goose/README.md
 delete mode 100644 cmd/goose/main.go
 create mode 100644 internal/sqlutil/migrate_test.go

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 9b452d849..4cbfb380f 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -223,6 +223,31 @@ jobs:
       - name: Test upgrade
         run: ./dendrite-upgrade-tests --head .
 
+  # run database upgrade tests, skipping over one version
+  upgrade_test_direct:
+    name: Upgrade tests from HEAD-2
+    timeout-minutes: 20
+    needs: initial-tests-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.18"
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade
+      - name: Build upgrade-tests
+        run: go build ./cmd/dendrite-upgrade-tests
+      - name: Test upgrade
+        run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
+
   # run Sytest in different variations
   sytest:
     timeout-minutes: 20
@@ -359,7 +384,7 @@ jobs:
 
   integration-tests-done:
     name: Integration tests passed
-    needs: [initial-tests-done, upgrade_test, sytest, complement]
+    needs: [initial-tests-done, upgrade_test, upgrade_test_direct, sytest, complement]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
     steps:
diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index cabd07e70..39843dccb 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -37,6 +37,7 @@ var (
 	flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images")
 	flagHead             = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
 	flagDockerHost       = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
+	flagDirect           = flag.Bool("direct", false, "If a direct upgrade from the defined FROM version to TO should be done")
 	alphaNumerics        = regexp.MustCompile("[^a-zA-Z0-9]+")
 )
 
@@ -229,7 +230,7 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve
 	return semVers, nil
 }
 
-func calculateVersions(cli *http.Client, from, to string) []string {
+func calculateVersions(cli *http.Client, from, to string, direct bool) []string {
 	semvers, err := getAndSortVersionsFromGithub(cli)
 	if err != nil {
 		log.Fatalf("failed to collect semvers from github: %s", err)
@@ -284,6 +285,9 @@ func calculateVersions(cli *http.Client, from, to string) []string {
 	if to == HEAD {
 		versions = append(versions, HEAD)
 	}
+	if direct {
+		versions = []string{versions[0], versions[len(versions)-1]}
+	}
 	return versions
 }
 
@@ -461,7 +465,7 @@ func main() {
 		os.Exit(1)
 	}
 	cleanup(dockerClient)
-	versions := calculateVersions(httpClient, *flagFrom, *flagTo)
+	versions := calculateVersions(httpClient, *flagFrom, *flagTo, *flagDirect)
 	log.Printf("Testing dendrite versions: %v\n", versions)
 
 	branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions)
diff --git a/cmd/goose/README.md b/cmd/goose/README.md
deleted file mode 100644
index 725c6a586..000000000
--- a/cmd/goose/README.md
+++ /dev/null
@@ -1,109 +0,0 @@
-## Database migrations
-
-We use [goose](https://github.com/pressly/goose) to handle database migrations. This allows us to execute
-both SQL deltas (e.g `ALTER TABLE ...`) as well as manipulate data in the database in Go using Go functions.
-
-To run a migration, the `goose` binary in this directory needs to be built:
-```
-$ go build ./cmd/goose
-```
-
-This binary allows Dendrite databases to be upgraded and downgraded. Sample usage for upgrading the roomserver database:
-
-```
-# for sqlite
-$ ./goose -dir roomserver/storage/sqlite3/deltas sqlite3 ./roomserver.db up
-
-# for postgres
-$ ./goose -dir roomserver/storage/postgres/deltas postgres "user=dendrite dbname=dendrite sslmode=disable" up
-```
-
-For a full list of options, including rollbacks, see https://github.com/pressly/goose or use `goose` with no args.
-
-
-### Rationale
-
-Dendrite creates tables on startup using `CREATE TABLE IF NOT EXISTS`, so you might think that we should also
-apply version upgrades on startup as well. This is convenient and doesn't involve an additional binary to run
-which complicates upgrades. However, combining the upgrade mechanism and the server binary makes it difficult
-to handle rollbacks. Firstly, how do you specify you wish to rollback? We would have to add additional flags
-to the main server binary to say "rollback to version X". Secondly, if you roll back the server binary from
-version 5 to version 4, the version 4 binary doesn't know how to rollback the database from version 5 to
-version 4! For these reasons, we prefer to have a separate "upgrade" binary which is run for database upgrades.
-Rather than roll-our-own migration tool, we decided to use [goose](https://github.com/pressly/goose) as it supports
-complex migrations in Go code in addition to just executing SQL deltas. Other alternatives like
-`github.com/golang-migrate/migrate` [do not support](https://github.com/golang-migrate/migrate/issues/15) these
-kinds of complex migrations.
-
-### Adding new deltas
-
-You can add `.sql` or `.go` files manually or you can use goose to create them for you.
-
-If you only want to add a SQL delta then run:
-
-```
-$ ./goose -dir serverkeyapi/storage/sqlite3/deltas sqlite3 ./foo.db create new_col sql
-2020/09/09 14:37:43 Created new file: serverkeyapi/storage/sqlite3/deltas/20200909143743_new_col.sql
-```
-
-In this case, the version number is `20200909143743`. The important thing is that it is always increasing.
-
-Then add up/downgrade SQL commands to the created file which looks like:
-```sql
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-SELECT 'down SQL query';
--- +goose StatementEnd
-
-```
-You __must__ keep the `+goose` annotations. You'll need to repeat this process for Postgres.
-
-For complex Go migrations:
-
-```
-$ ./goose -dir serverkeyapi/storage/sqlite3/deltas sqlite3 ./foo.db create complex_update go
-2020/09/09 14:40:38 Created new file: serverkeyapi/storage/sqlite3/deltas/20200909144038_complex_update.go
-```
-
-Then modify the created `.go` file which looks like:
-
-```go
-package migrations
-
-import (
-	"database/sql"
-	"fmt"
-
-	"github.com/pressly/goose"
-)
-
-func init() {
-	goose.AddMigration(upComplexUpdate, downComplexUpdate)
-}
-
-func upComplexUpdate(tx *sql.Tx) error {
-	// This code is executed when the migration is applied.
-	return nil
-}
-
-func downComplexUpdate(tx *sql.Tx) error {
-	// This code is executed when the migration is rolled back.
-	return nil
-}
-
-```
-
-You __must__ import the package in `/cmd/goose/main.go` so `func init()` gets called.
-
-
-#### Database limitations
-
-- SQLite3 does NOT support `ALTER TABLE table_name DROP COLUMN` - you would have to rename the column or drop the table
-  entirely and recreate it. ([example](https://github.com/matrix-org/dendrite/blob/master/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql))
-  
-  More information: [sqlite.org](https://www.sqlite.org/lang_altertable.html)
diff --git a/cmd/goose/main.go b/cmd/goose/main.go
deleted file mode 100644
index 31a5b0050..000000000
--- a/cmd/goose/main.go
+++ /dev/null
@@ -1,154 +0,0 @@
-// This is custom goose binary
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-
-	"github.com/pressly/goose"
-
-	pgusers "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
-	slusers "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
-
-	_ "github.com/lib/pq"
-	_ "github.com/mattn/go-sqlite3"
-)
-
-const (
-	AppService       = "appservice"
-	FederationSender = "federationapi"
-	KeyServer        = "keyserver"
-	MediaAPI         = "mediaapi"
-	RoomServer       = "roomserver"
-	SigningKeyServer = "signingkeyserver"
-	SyncAPI          = "syncapi"
-	UserAPI          = "userapi"
-)
-
-var (
-	dir       = flags.String("dir", "", "directory with migration files")
-	flags     = flag.NewFlagSet("goose", flag.ExitOnError)
-	component = flags.String("component", "", "dendrite component name")
-	knownDBs  = []string{
-		AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPI,
-	}
-)
-
-// nolint: gocyclo
-func main() {
-	err := flags.Parse(os.Args[1:])
-	if err != nil {
-		panic(err.Error())
-	}
-	args := flags.Args()
-
-	if len(args) < 3 {
-		fmt.Println(
-			`Usage: goose [OPTIONS] DRIVER DBSTRING COMMAND
-
-Drivers:
-    postgres
-    sqlite3
-
-Examples:
-    goose -component roomserver sqlite3 ./roomserver.db status
-    goose -component roomserver sqlite3 ./roomserver.db up
-
-    goose -component roomserver postgres "user=dendrite dbname=dendrite sslmode=disable" status
-
-Options:
-  -component string
-    	Dendrite component name e.g roomserver, signingkeyserver, clientapi, syncapi
-  -table string
-    	migrations table name (default "goose_db_version")
-  -h	print help
-  -v	enable verbose mode
-  -dir string
-        directory with migration files, only relevant when creating new migrations.
-  -version
-    	print version
-
-Commands:
-    up                   Migrate the DB to the most recent version available
-    up-by-one            Migrate the DB up by 1
-    up-to VERSION        Migrate the DB to a specific VERSION
-    down                 Roll back the version by 1
-    down-to VERSION      Roll back to a specific VERSION
-    redo                 Re-run the latest migration
-    reset                Roll back all migrations
-    status               Dump the migration status for the current DB
-    version              Print the current version of the database
-    create NAME [sql|go] Creates new migration file with the current timestamp
-    fix                  Apply sequential ordering to migrations`,
-		)
-		return
-	}
-
-	engine := args[0]
-	if engine != "sqlite3" && engine != "postgres" {
-		fmt.Println("engine must be one of 'sqlite3' or 'postgres'")
-		return
-	}
-
-	knownComponent := false
-	for _, c := range knownDBs {
-		if c == *component {
-			knownComponent = true
-			break
-		}
-	}
-	if !knownComponent {
-		fmt.Printf("component must be one of %v\n", knownDBs)
-		return
-	}
-
-	if engine == "sqlite3" {
-		loadSQLiteDeltas(*component)
-	} else {
-		loadPostgresDeltas(*component)
-	}
-
-	dbstring, command := args[1], args[2]
-
-	db, err := goose.OpenDBWithDriver(engine, dbstring)
-	if err != nil {
-		log.Fatalf("goose: failed to open DB: %v\n", err)
-	}
-
-	defer func() {
-		if err := db.Close(); err != nil {
-			log.Fatalf("goose: failed to close DB: %v\n", err)
-		}
-	}()
-
-	arguments := []string{}
-	if len(args) > 3 {
-		arguments = append(arguments, args[3:]...)
-	}
-
-	// goose demands a directory even though we don't use it for upgrades
-	d := *dir
-	if d == "" {
-		d = os.TempDir()
-	}
-	if err := goose.Run(command, db, d, arguments...); err != nil {
-		log.Fatalf("goose %v: %v", command, err)
-	}
-}
-
-func loadSQLiteDeltas(component string) {
-	switch component {
-	case UserAPI:
-		slusers.LoadFromGoose()
-	}
-}
-
-func loadPostgresDeltas(component string) {
-	switch component {
-	case UserAPI:
-		pgusers.LoadFromGoose()
-	}
-}
diff --git a/federationapi/storage/postgres/deltas/2021020411080000_rooms.go b/federationapi/storage/postgres/deltas/2021020411080000_rooms.go
index cc4bdadfd..fc894846d 100644
--- a/federationapi/storage/postgres/deltas/2021020411080000_rooms.go
+++ b/federationapi/storage/postgres/deltas/2021020411080000_rooms.go
@@ -15,23 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
-}
-
-func LoadRemoveRoomsTable(m *sqlutil.Migrations) {
-	m.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
-}
-
-func UpRemoveRoomsTable(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpRemoveRoomsTable(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		DROP TABLE IF EXISTS federationsender_rooms;
 	`)
 	if err != nil {
diff --git a/federationapi/storage/postgres/storage.go b/federationapi/storage/postgres/storage.go
index 9863afb2b..7c2883c1b 100644
--- a/federationapi/storage/postgres/storage.go
+++ b/federationapi/storage/postgres/storage.go
@@ -82,9 +82,13 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, err
 	}
-	m := sqlutil.NewMigrations()
-	deltas.LoadRemoveRoomsTable(m)
-	if err = m.RunDeltas(d.db, dbProperties); err != nil {
+	m := sqlutil.NewMigrator(d.db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "federationsender: drop federationsender_rooms",
+		Up:      deltas.UpRemoveRoomsTable,
+	})
+	err = m.Up(base.Context())
+	if err != nil {
 		return nil, err
 	}
 	d.Database = shared.Database{
diff --git a/federationapi/storage/sqlite3/deltas/2021020411080000_rooms.go b/federationapi/storage/sqlite3/deltas/2021020411080000_rooms.go
index cc4bdadfd..fc894846d 100644
--- a/federationapi/storage/sqlite3/deltas/2021020411080000_rooms.go
+++ b/federationapi/storage/sqlite3/deltas/2021020411080000_rooms.go
@@ -15,23 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
-}
-
-func LoadRemoveRoomsTable(m *sqlutil.Migrations) {
-	m.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
-}
-
-func UpRemoveRoomsTable(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpRemoveRoomsTable(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		DROP TABLE IF EXISTS federationsender_rooms;
 	`)
 	if err != nil {
diff --git a/federationapi/storage/sqlite3/storage.go b/federationapi/storage/sqlite3/storage.go
index 7d0cee90e..9594aaec6 100644
--- a/federationapi/storage/sqlite3/storage.go
+++ b/federationapi/storage/sqlite3/storage.go
@@ -81,9 +81,13 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, err
 	}
-	m := sqlutil.NewMigrations()
-	deltas.LoadRemoveRoomsTable(m)
-	if err = m.RunDeltas(d.db, dbProperties); err != nil {
+	m := sqlutil.NewMigrator(d.db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "federationsender: drop federationsender_rooms",
+		Up:      deltas.UpRemoveRoomsTable,
+	})
+	err = m.Up(base.Context())
+	if err != nil {
 		return nil, err
 	}
 	d.Database = shared.Database{
diff --git a/go.mod b/go.mod
index 0a965ec8e..79216f0fa 100644
--- a/go.mod
+++ b/go.mod
@@ -37,7 +37,6 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/pressly/goose v2.7.0+incompatible
 	github.com/prometheus/client_golang v1.12.2
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.1
diff --git a/go.sum b/go.sum
index df3feeb6e..7107d4ff6 100644
--- a/go.sum
+++ b/go.sum
@@ -432,8 +432,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ=
-github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go
index 7518df3c8..18020a902 100644
--- a/internal/sqlutil/migrate.go
+++ b/internal/sqlutil/migrate.go
@@ -1,130 +1,142 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package sqlutil
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-	"runtime"
-	"sort"
+	"sync"
+	"time"
 
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/pressly/goose"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/sirupsen/logrus"
 )
 
-type Migrations struct {
-	registeredGoMigrations map[int64]*goose.Migration
+const createDBMigrationsSQL = "" +
+	"CREATE TABLE IF NOT EXISTS db_migrations (" +
+	" version TEXT PRIMARY KEY NOT NULL," +
+	" time TEXT NOT NULL," +
+	" dendrite_version TEXT NOT NULL" +
+	");"
+
+const insertVersionSQL = "" +
+	"INSERT INTO db_migrations (version, time, dendrite_version)" +
+	" VALUES ($1, $2, $3)"
+
+const selectDBMigrationsSQL = "SELECT version FROM db_migrations"
+
+// Migration defines a migration to be run.
+type Migration struct {
+	// Version is a simple description/name of this migration.
+	Version string
+	// Up defines the function to execute for an upgrade.
+	Up func(ctx context.Context, txn *sql.Tx) error
+	// Down defines the function to execute for a downgrade (not implemented yet).
+	Down func(ctx context.Context, txn *sql.Tx) error
 }
 
-func NewMigrations() *Migrations {
-	return &Migrations{
-		registeredGoMigrations: make(map[int64]*goose.Migration),
+// Migrator
+type Migrator struct {
+	db              *sql.DB
+	migrations      []Migration
+	knownMigrations map[string]struct{}
+	mutex           *sync.Mutex
+}
+
+// NewMigrator creates a new DB migrator.
+func NewMigrator(db *sql.DB) *Migrator {
+	return &Migrator{
+		db:              db,
+		migrations:      []Migration{},
+		knownMigrations: make(map[string]struct{}),
+		mutex:           &sync.Mutex{},
 	}
 }
 
-// Copy-pasted from goose directly to store migrations into a map we control
-
-// AddMigration adds a migration.
-func (m *Migrations) AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) {
-	_, filename, _, _ := runtime.Caller(1)
-	m.AddNamedMigration(filename, up, down)
-}
-
-// AddNamedMigration : Add a named migration.
-func (m *Migrations) AddNamedMigration(filename string, up func(*sql.Tx) error, down func(*sql.Tx) error) {
-	v, _ := goose.NumericComponent(filename)
-	migration := &goose.Migration{Version: v, Next: -1, Previous: -1, Registered: true, UpFn: up, DownFn: down, Source: filename}
-
-	if existing, ok := m.registeredGoMigrations[v]; ok {
-		panic(fmt.Sprintf("failed to add migration %q: version conflicts with %q", filename, existing.Source))
+// AddMigrations appends migrations to the list of migrations. Migrations are executed
+// in the order they are added to the list. De-duplicates migrations using their Version field.
+func (m *Migrator) AddMigrations(migrations ...Migration) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	for _, mig := range migrations {
+		if _, ok := m.knownMigrations[mig.Version]; !ok {
+			m.migrations = append(m.migrations, mig)
+			m.knownMigrations[mig.Version] = struct{}{}
+		}
 	}
-
-	m.registeredGoMigrations[v] = migration
 }
 
-// RunDeltas up to the latest version.
-func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error {
-	maxVer := goose.MaxVersion
-	minVer := int64(0)
-	migrations, err := m.collect(minVer, maxVer)
+// Up executes all migrations in order they were added.
+func (m *Migrator) Up(ctx context.Context) error {
+	var (
+		err             error
+		dendriteVersion = internal.VersionString()
+	)
+	// ensure there is a table for known migrations
+	executedMigrations, err := m.ExecutedMigrations(ctx)
 	if err != nil {
-		return fmt.Errorf("runDeltas: Failed to collect migrations: %w", err)
+		return fmt.Errorf("unable to create/get migrations: %w", err)
 	}
-	if props.ConnectionString.IsPostgres() {
-		if err = goose.SetDialect("postgres"); err != nil {
-			return err
-		}
-	} else if props.ConnectionString.IsSQLite() {
-		if err = goose.SetDialect("sqlite3"); err != nil {
-			return err
-		}
-	} else {
-		return fmt.Errorf("unknown connection string: %s", props.ConnectionString)
-	}
-	for {
-		current, err := goose.EnsureDBVersion(db)
-		if err != nil {
-			return fmt.Errorf("runDeltas: Failed to EnsureDBVersion: %w", err)
-		}
 
-		next, err := migrations.Next(current)
-		if err != nil {
-			if err == goose.ErrNoNextVersion {
-				return nil
+	return WithTransaction(m.db, func(txn *sql.Tx) error {
+		for i := range m.migrations {
+			now := time.Now().UTC().Format(time.RFC3339)
+			migration := m.migrations[i]
+			logrus.Debugf("Executing database migration '%s'", migration.Version)
+			// Skip migration if it was already executed
+			if _, ok := executedMigrations[migration.Version]; ok {
+				continue
+			}
+			err = migration.Up(ctx, txn)
+			if err != nil {
+				return fmt.Errorf("unable to execute migration '%s': %w", migration.Version, err)
+			}
+			_, err = txn.ExecContext(ctx, insertVersionSQL,
+				migration.Version,
+				now,
+				dendriteVersion,
+			)
+			if err != nil {
+				return fmt.Errorf("unable to insert executed migrations: %w", err)
 			}
-
-			return fmt.Errorf("runDeltas: Failed to load next migration to %+v : %w", next, err)
 		}
-
-		if err = next.Up(db); err != nil {
-			return fmt.Errorf("runDeltas: Failed run migration: %w", err)
-		}
-	}
+		return nil
+	})
 }
 
-func (m *Migrations) collect(current, target int64) (goose.Migrations, error) {
-	var migrations goose.Migrations
-
-	// Go migrations registered via goose.AddMigration().
-	for _, migration := range m.registeredGoMigrations {
-		v, err := goose.NumericComponent(migration.Source)
-		if err != nil {
-			return nil, err
-		}
-		if versionFilter(v, current, target) {
-			migrations = append(migrations, migration)
+// ExecutedMigrations returns a map with already executed migrations in addition to creating the
+// migrations table, if it doesn't exist.
+func (m *Migrator) ExecutedMigrations(ctx context.Context) (map[string]struct{}, error) {
+	result := make(map[string]struct{})
+	_, err := m.db.ExecContext(ctx, createDBMigrationsSQL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create db_migrations: %w", err)
+	}
+	rows, err := m.db.QueryContext(ctx, selectDBMigrationsSQL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to query db_migrations: %w", err)
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "ExecutedMigrations: rows.close() failed")
+	var version string
+	for rows.Next() {
+		if err = rows.Scan(&version); err != nil {
+			return nil, fmt.Errorf("unable to scan version: %w", err)
 		}
+		result[version] = struct{}{}
 	}
 
-	migrations = sortAndConnectMigrations(migrations)
-
-	return migrations, nil
-}
-
-func sortAndConnectMigrations(migrations goose.Migrations) goose.Migrations {
-	sort.Sort(migrations)
-
-	// now that we're sorted in the appropriate direction,
-	// populate next and previous for each migration
-	for i, m := range migrations {
-		prev := int64(-1)
-		if i > 0 {
-			prev = migrations[i-1].Version
-			migrations[i-1].Next = m.Version
-		}
-		migrations[i].Previous = prev
-	}
-
-	return migrations
-}
-
-func versionFilter(v, current, target int64) bool {
-
-	if target > current {
-		return v > current && v <= target
-	}
-
-	if target < current {
-		return v <= current && v > target
-	}
-
-	return false
+	return result, rows.Err()
 }
diff --git a/internal/sqlutil/migrate_test.go b/internal/sqlutil/migrate_test.go
new file mode 100644
index 000000000..d8bcae196
--- /dev/null
+++ b/internal/sqlutil/migrate_test.go
@@ -0,0 +1,112 @@
+package sqlutil_test
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/test"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+var dummyMigrations = []sqlutil.Migration{
+	{
+		Version: "init",
+		Up: func(ctx context.Context, txn *sql.Tx) error {
+			_, err := txn.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS dummy ( test TEXT );")
+			return err
+		},
+	},
+	{
+		Version: "v2",
+		Up: func(ctx context.Context, txn *sql.Tx) error {
+			_, err := txn.ExecContext(ctx, "ALTER TABLE dummy ADD COLUMN test2 TEXT;")
+			return err
+		},
+	},
+	{
+		Version: "v2", // duplicate, this migration will be skipped
+		Up: func(ctx context.Context, txn *sql.Tx) error {
+			_, err := txn.ExecContext(ctx, "ALTER TABLE dummy ADD COLUMN test2 TEXT;")
+			return err
+		},
+	},
+	{
+		Version: "multiple execs",
+		Up: func(ctx context.Context, txn *sql.Tx) error {
+			_, err := txn.ExecContext(ctx, "ALTER TABLE dummy ADD COLUMN test3 TEXT;")
+			if err != nil {
+				return err
+			}
+			_, err = txn.ExecContext(ctx, "ALTER TABLE dummy ADD COLUMN test4 TEXT;")
+			return err
+		},
+	},
+}
+
+var failMigration = sqlutil.Migration{
+	Version: "iFail",
+	Up: func(ctx context.Context, txn *sql.Tx) error {
+		return fmt.Errorf("iFail")
+	},
+	Down: nil,
+}
+
+func Test_migrations_Up(t *testing.T) {
+	withFail := append(dummyMigrations, failMigration)
+
+	tests := []struct {
+		name       string
+		migrations []sqlutil.Migration
+		wantResult map[string]struct{}
+		wantErr    bool
+	}{
+		{
+			name:       "dummy migration",
+			migrations: dummyMigrations,
+			wantResult: map[string]struct{}{
+				"init":           {},
+				"v2":             {},
+				"multiple execs": {},
+			},
+		},
+		{
+			name:       "with fail",
+			migrations: withFail,
+			wantErr:    true,
+		},
+	}
+
+	ctx := context.Background()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+				conStr, close := test.PrepareDBConnectionString(t, dbType)
+				defer close()
+				driverName := "sqlite3"
+				if dbType == test.DBTypePostgres {
+					driverName = "postgres"
+				}
+				db, err := sql.Open(driverName, conStr)
+				if err != nil {
+					t.Errorf("unable to open database: %v", err)
+				}
+				m := sqlutil.NewMigrator(db)
+				m.AddMigrations(tt.migrations...)
+				if err = m.Up(ctx); (err != nil) != tt.wantErr {
+					t.Errorf("Up() error = %v, wantErr %v", err, tt.wantErr)
+				}
+				result, err := m.ExecutedMigrations(ctx)
+				if err != nil {
+					t.Errorf("unable to get executed migrations: %v", err)
+				}
+				if !tt.wantErr && !reflect.DeepEqual(result, tt.wantResult) {
+					t.Errorf("expected: %+v, got %v", tt.wantResult, result)
+				}
+			})
+		})
+	}
+}
diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go
index b101e7ce5..8b2a865b9 100644
--- a/keyserver/storage/postgres/cross_signing_sigs_table.go
+++ b/keyserver/storage/postgres/cross_signing_sigs_table.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/keyserver/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/keyserver/storage/tables"
 	"github.com/matrix-org/dendrite/keyserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -66,6 +67,16 @@ func NewPostgresCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, erro
 	if err != nil {
 		return nil, err
 	}
+
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "keyserver: cross signing signature indexes",
+		Up:      deltas.UpFixCrossSigningSignatureIndexes,
+	})
+	if err = m.Up(context.Background()); err != nil {
+		return nil, err
+	}
+
 	return s, sqlutil.StatementList{
 		{&s.selectCrossSigningSigsForTargetStmt, selectCrossSigningSigsForTargetSQL},
 		{&s.upsertCrossSigningSigsForTargetStmt, upsertCrossSigningSigsForTargetSQL},
diff --git a/keyserver/storage/postgres/deltas/2022012016470000_key_changes.go b/keyserver/storage/postgres/deltas/2022012016470000_key_changes.go
index e5bcf08d1..0cfe9e791 100644
--- a/keyserver/storage/postgres/deltas/2022012016470000_key_changes.go
+++ b/keyserver/storage/postgres/deltas/2022012016470000_key_changes.go
@@ -15,37 +15,27 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpRefactorKeyChanges, DownRefactorKeyChanges)
-}
-
-func LoadRefactorKeyChanges(m *sqlutil.Migrations) {
-	m.AddMigration(UpRefactorKeyChanges, DownRefactorKeyChanges)
-}
-
-func UpRefactorKeyChanges(tx *sql.Tx) error {
+func UpRefactorKeyChanges(ctx context.Context, tx *sql.Tx) error {
 	// start counting from the last max offset, else 0. We need to do a count(*) first to see if there
 	// even are entries in this table to know if we can query for log_offset. Without the count then
 	// the query to SELECT the max log offset fails on new Dendrite instances as log_offset doesn't
 	// exist on that table. Even though we discard the error, the txn is tainted and gets aborted :/
 	var count int
-	_ = tx.QueryRow(`SELECT count(*) FROM keyserver_key_changes`).Scan(&count)
+	_ = tx.QueryRowContext(ctx, `SELECT count(*) FROM keyserver_key_changes`).Scan(&count)
 	if count > 0 {
 		var maxOffset int64
-		_ = tx.QueryRow(`SELECT coalesce(MAX(log_offset), 0) AS offset FROM keyserver_key_changes`).Scan(&maxOffset)
-		if _, err := tx.Exec(fmt.Sprintf(`CREATE SEQUENCE IF NOT EXISTS keyserver_key_changes_seq START %d`, maxOffset)); err != nil {
+		_ = tx.QueryRowContext(ctx, `SELECT coalesce(MAX(log_offset), 0) AS offset FROM keyserver_key_changes`).Scan(&maxOffset)
+		if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE SEQUENCE IF NOT EXISTS keyserver_key_changes_seq START %d`, maxOffset)); err != nil {
 			return fmt.Errorf("failed to CREATE SEQUENCE for key changes, starting at %d: %s", maxOffset, err)
 		}
 	}
 
-	_, err := tx.Exec(`
+	_, err := tx.ExecContext(ctx, `
 		-- make the new table
 		DROP TABLE IF EXISTS keyserver_key_changes;
 		CREATE TABLE IF NOT EXISTS keyserver_key_changes (
@@ -60,8 +50,8 @@ func UpRefactorKeyChanges(tx *sql.Tx) error {
 	return nil
 }
 
-func DownRefactorKeyChanges(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownRefactorKeyChanges(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	-- Drop all data and revert back, we can't keep the data as Kafka offsets determine the numbers
 	DROP SEQUENCE IF EXISTS keyserver_key_changes_seq;
 	DROP TABLE IF EXISTS keyserver_key_changes;
diff --git a/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go b/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go
index 12956e3b4..1a3d4fee9 100644
--- a/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go
+++ b/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go
@@ -15,18 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadFixCrossSigningSignatureIndexes(m *sqlutil.Migrations) {
-	m.AddMigration(UpFixCrossSigningSignatureIndexes, DownFixCrossSigningSignatureIndexes)
-}
-
-func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpFixCrossSigningSignatureIndexes(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE keyserver_cross_signing_sigs DROP CONSTRAINT keyserver_cross_signing_sigs_pkey;
 		ALTER TABLE keyserver_cross_signing_sigs ADD PRIMARY KEY (origin_user_id, origin_key_id, target_user_id, target_key_id);
 
@@ -38,8 +33,8 @@ func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
 	return nil
 }
 
-func DownFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownFixCrossSigningSignatureIndexes(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE keyserver_cross_signing_sigs DROP CONSTRAINT keyserver_cross_signing_sigs_pkey;
 		ALTER TABLE keyserver_cross_signing_sigs ADD PRIMARY KEY (origin_user_id, target_user_id, target_key_id);
 
diff --git a/keyserver/storage/postgres/key_changes_table.go b/keyserver/storage/postgres/key_changes_table.go
index f93a94bd3..6894d7b7c 100644
--- a/keyserver/storage/postgres/key_changes_table.go
+++ b/keyserver/storage/postgres/key_changes_table.go
@@ -19,6 +19,8 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/keyserver/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/keyserver/storage/tables"
 )
 
@@ -55,7 +57,23 @@ func NewPostgresKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) {
 		db: db,
 	}
 	_, err := db.Exec(keyChangesSchema)
-	return s, err
+	if err != nil {
+		return s, err
+	}
+
+	// TODO: Remove when we are sure we are not having goose artefacts in the db
+	// This forces an error, which indicates the migration is already applied, since the
+	// column partition was removed from the table
+	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan()
+	if err == nil {
+		m := sqlutil.NewMigrator(db)
+		m.AddMigrations(sqlutil.Migration{
+			Version: "keyserver: refactor key changes",
+			Up:      deltas.UpRefactorKeyChanges,
+		})
+		return s, m.Up(context.Background())
+	}
+	return s, nil
 }
 
 func (s *keyChangesStatements) Prepare() (err error) {
diff --git a/keyserver/storage/postgres/storage.go b/keyserver/storage/postgres/storage.go
index b8f70acf8..35e630559 100644
--- a/keyserver/storage/postgres/storage.go
+++ b/keyserver/storage/postgres/storage.go
@@ -16,7 +16,6 @@ package postgres
 
 import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/keyserver/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/keyserver/storage/shared"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -53,12 +52,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if err != nil {
 		return nil, err
 	}
-	m := sqlutil.NewMigrations()
-	deltas.LoadRefactorKeyChanges(m)
-	deltas.LoadFixCrossSigningSignatureIndexes(m)
-	if err = m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
-	}
 	if err = kc.Prepare(); err != nil {
 		return nil, err
 	}
diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go
index 36d562b8a..ea431151e 100644
--- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go
+++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/keyserver/storage/tables"
 	"github.com/matrix-org/dendrite/keyserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -65,6 +66,15 @@ func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error)
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "keyserver: cross signing signature indexes",
+		Up:      deltas.UpFixCrossSigningSignatureIndexes,
+	})
+	if err = m.Up(context.Background()); err != nil {
+		return nil, err
+	}
+
 	return s, sqlutil.StatementList{
 		{&s.selectCrossSigningSigsForTargetStmt, selectCrossSigningSigsForTargetSQL},
 		{&s.upsertCrossSigningSigsForTargetStmt, upsertCrossSigningSigsForTargetSQL},
diff --git a/keyserver/storage/sqlite3/deltas/2022012016470000_key_changes.go b/keyserver/storage/sqlite3/deltas/2022012016470000_key_changes.go
index fbc548c38..cd0f19df9 100644
--- a/keyserver/storage/sqlite3/deltas/2022012016470000_key_changes.go
+++ b/keyserver/storage/sqlite3/deltas/2022012016470000_key_changes.go
@@ -15,28 +15,18 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpRefactorKeyChanges, DownRefactorKeyChanges)
-}
-
-func LoadRefactorKeyChanges(m *sqlutil.Migrations) {
-	m.AddMigration(UpRefactorKeyChanges, DownRefactorKeyChanges)
-}
-
-func UpRefactorKeyChanges(tx *sql.Tx) error {
+func UpRefactorKeyChanges(ctx context.Context, tx *sql.Tx) error {
 	// start counting from the last max offset, else 0.
 	var maxOffset int64
 	var userID string
-	_ = tx.QueryRow(`SELECT user_id, MAX(log_offset) FROM keyserver_key_changes GROUP BY user_id`).Scan(&userID, &maxOffset)
+	_ = tx.QueryRowContext(ctx, `SELECT user_id, MAX(log_offset) FROM keyserver_key_changes GROUP BY user_id`).Scan(&userID, &maxOffset)
 
-	_, err := tx.Exec(`
+	_, err := tx.ExecContext(ctx, `
 		-- make the new table
 		DROP TABLE IF EXISTS keyserver_key_changes;
 		CREATE TABLE IF NOT EXISTS keyserver_key_changes (
@@ -51,14 +41,14 @@ func UpRefactorKeyChanges(tx *sql.Tx) error {
 	}
 	// to start counting from maxOffset, insert a row with that value
 	if userID != "" {
-		_, err = tx.Exec(`INSERT INTO keyserver_key_changes(change_id, user_id) VALUES($1, $2)`, maxOffset, userID)
+		_, err = tx.ExecContext(ctx, `INSERT INTO keyserver_key_changes(change_id, user_id) VALUES($1, $2)`, maxOffset, userID)
 		return err
 	}
 	return nil
 }
 
-func DownRefactorKeyChanges(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownRefactorKeyChanges(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	-- Drop all data and revert back, we can't keep the data as Kafka offsets determine the numbers
 	DROP TABLE IF EXISTS keyserver_key_changes;
 	CREATE TABLE IF NOT EXISTS keyserver_key_changes (
diff --git a/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go b/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go
index 230e39fef..d4e38dea5 100644
--- a/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go
+++ b/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go
@@ -15,18 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadFixCrossSigningSignatureIndexes(m *sqlutil.Migrations) {
-	m.AddMigration(UpFixCrossSigningSignatureIndexes, DownFixCrossSigningSignatureIndexes)
-}
-
-func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpFixCrossSigningSignatureIndexes(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs_tmp (
 			origin_user_id TEXT NOT NULL,
 			origin_key_id TEXT NOT NULL,
@@ -50,8 +45,8 @@ func UpFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
 	return nil
 }
 
-func DownFixCrossSigningSignatureIndexes(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownFixCrossSigningSignatureIndexes(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs_tmp (
 			origin_user_id TEXT NOT NULL,
 			origin_key_id TEXT NOT NULL,
diff --git a/keyserver/storage/sqlite3/key_changes_table.go b/keyserver/storage/sqlite3/key_changes_table.go
index e035e8c9c..1b27c3d05 100644
--- a/keyserver/storage/sqlite3/key_changes_table.go
+++ b/keyserver/storage/sqlite3/key_changes_table.go
@@ -19,6 +19,8 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/keyserver/storage/tables"
 )
 
@@ -53,7 +55,23 @@ func NewSqliteKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) {
 		db: db,
 	}
 	_, err := db.Exec(keyChangesSchema)
-	return s, err
+	if err != nil {
+		return s, err
+	}
+	// TODO: Remove when we are sure we are not having goose artefacts in the db
+	// This forces an error, which indicates the migration is already applied, since the
+	// column partition was removed from the table
+	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan()
+	if err == nil {
+		m := sqlutil.NewMigrator(db)
+		m.AddMigrations(sqlutil.Migration{
+			Version: "keyserver: refactor key changes",
+			Up:      deltas.UpRefactorKeyChanges,
+		})
+		return s, m.Up(context.Background())
+	}
+
+	return s, nil
 }
 
 func (s *keyChangesStatements) Prepare() (err error) {
diff --git a/keyserver/storage/sqlite3/storage.go b/keyserver/storage/sqlite3/storage.go
index aeea9eac6..873fe3e24 100644
--- a/keyserver/storage/sqlite3/storage.go
+++ b/keyserver/storage/sqlite3/storage.go
@@ -17,7 +17,6 @@ package sqlite3
 import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/storage/shared"
-	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 )
@@ -52,12 +51,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 		return nil, err
 	}
 
-	m := sqlutil.NewMigrations()
-	deltas.LoadRefactorKeyChanges(m)
-	deltas.LoadFixCrossSigningSignatureIndexes(m)
-	if err = m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
-	}
 	if err = kc.Prepare(); err != nil {
 		return nil, err
 	}
diff --git a/roomserver/storage/postgres/deltas/20201028212440_add_forgotten_column.go b/roomserver/storage/postgres/deltas/20201028212440_add_forgotten_column.go
index f3bd8632f..61d4dba87 100644
--- a/roomserver/storage/postgres/deltas/20201028212440_add_forgotten_column.go
+++ b/roomserver/storage/postgres/deltas/20201028212440_add_forgotten_column.go
@@ -15,32 +15,21 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpAddForgottenColumn, DownAddForgottenColumn)
-	goose.AddMigration(UpStateBlocksRefactor, DownStateBlocksRefactor)
-}
-
-func LoadAddForgottenColumn(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddForgottenColumn, DownAddForgottenColumn)
-}
-
-func UpAddForgottenColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`ALTER TABLE roomserver_membership ADD COLUMN IF NOT EXISTS forgotten BOOLEAN NOT NULL DEFAULT false;`)
+func UpAddForgottenColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_membership ADD COLUMN IF NOT EXISTS forgotten BOOLEAN NOT NULL DEFAULT false;`)
 	if err != nil {
 		return fmt.Errorf("failed to execute upgrade: %w", err)
 	}
 	return nil
 }
 
-func DownAddForgottenColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`ALTER TABLE roomserver_membership DROP COLUMN IF EXISTS forgotten;`)
+func DownAddForgottenColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_membership DROP COLUMN IF EXISTS forgotten;`)
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
diff --git a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go
index 06442a4c3..355c49b14 100644
--- a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go
+++ b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go
@@ -15,11 +15,11 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 
 	"github.com/lib/pq"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -36,48 +36,44 @@ type stateBlockData struct {
 	EventNIDs     types.EventNIDs
 }
 
-func LoadStateBlocksRefactor(m *sqlutil.Migrations) {
-	m.AddMigration(UpStateBlocksRefactor, DownStateBlocksRefactor)
-}
-
 // nolint:gocyclo
-func UpStateBlocksRefactor(tx *sql.Tx) error {
+func UpStateBlocksRefactor(ctx context.Context, tx *sql.Tx) error {
 	logrus.Warn("Performing state storage upgrade. Please wait, this may take some time!")
 	defer logrus.Warn("State storage upgrade complete")
 
 	var snapshotcount int
 	var maxsnapshotid int
 	var maxblockid int
-	if err := tx.QueryRow(`SELECT COUNT(DISTINCT state_snapshot_nid) FROM roomserver_state_snapshots;`).Scan(&snapshotcount); err != nil {
-		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
+	if err := tx.QueryRowContext(ctx, `SELECT COUNT(DISTINCT state_snapshot_nid) FROM roomserver_state_snapshots;`).Scan(&snapshotcount); err != nil {
+		return fmt.Errorf("tx.QueryRowContext.Scan (count snapshots): %w", err)
 	}
-	if err := tx.QueryRow(`SELECT COALESCE(MAX(state_snapshot_nid),0) FROM roomserver_state_snapshots;`).Scan(&maxsnapshotid); err != nil {
-		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
+	if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(state_snapshot_nid),0) FROM roomserver_state_snapshots;`).Scan(&maxsnapshotid); err != nil {
+		return fmt.Errorf("tx.QueryRowContext.Scan (count snapshots): %w", err)
 	}
-	if err := tx.QueryRow(`SELECT COALESCE(MAX(state_block_nid),0) FROM roomserver_state_block;`).Scan(&maxblockid); err != nil {
-		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
+	if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(state_block_nid),0) FROM roomserver_state_block;`).Scan(&maxblockid); err != nil {
+		return fmt.Errorf("tx.QueryRowContext.Scan (count snapshots): %w", err)
 	}
 	maxsnapshotid++
 	maxblockid++
 
-	if _, err := tx.Exec(`ALTER TABLE roomserver_state_block RENAME TO _roomserver_state_block;`); err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+	if _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_state_block RENAME TO _roomserver_state_block;`); err != nil {
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
-	if _, err := tx.Exec(`ALTER TABLE roomserver_state_snapshots RENAME TO _roomserver_state_snapshots;`); err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+	if _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_state_snapshots RENAME TO _roomserver_state_snapshots;`); err != nil {
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
 	// We create new sequences starting with the maximum state snapshot and block NIDs.
 	// This means that all newly created snapshots and blocks by the migration will have
 	// NIDs higher than these values, so that when we come to update the references to
 	// these NIDs using UPDATE statements, we can guarantee we are only ever updating old
 	// values and not accidentally overwriting new ones.
-	if _, err := tx.Exec(fmt.Sprintf(`CREATE SEQUENCE roomserver_state_block_nid_sequence START WITH %d;`, maxblockid)); err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+	if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE SEQUENCE roomserver_state_block_nid_sequence START WITH %d;`, maxblockid)); err != nil {
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
-	if _, err := tx.Exec(fmt.Sprintf(`CREATE SEQUENCE roomserver_state_snapshot_nid_sequence START WITH %d;`, maxsnapshotid)); err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+	if _, err := tx.ExecContext(ctx, fmt.Sprintf(`CREATE SEQUENCE roomserver_state_snapshot_nid_sequence START WITH %d;`, maxsnapshotid)); err != nil {
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
-	_, err := tx.Exec(`
+	_, err := tx.ExecContext(ctx, `
 		CREATE TABLE IF NOT EXISTS roomserver_state_block (
 			state_block_nid bigint PRIMARY KEY DEFAULT nextval('roomserver_state_block_nid_sequence'),
 			state_block_hash BYTEA UNIQUE,
@@ -87,7 +83,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 	if err != nil {
 		return fmt.Errorf("tx.Exec (create blocks table): %w", err)
 	}
-	_, err = tx.Exec(`
+	_, err = tx.ExecContext(ctx, `
 		CREATE TABLE IF NOT EXISTS roomserver_state_snapshots (
 			state_snapshot_nid bigint PRIMARY KEY DEFAULT nextval('roomserver_state_snapshot_nid_sequence'),
 			state_snapshot_hash BYTEA UNIQUE,
@@ -104,7 +100,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 	// in question a state snapshot NID of 0 to indicate 'no snapshot'.
 	// If we don't do this, we'll fail the assertions later on which try to ensure we didn't forget
 	// any snapshots.
-	_, err = tx.Exec(
+	_, err = tx.ExecContext(ctx,
 		`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE event_type_nid = $1 AND event_state_key_nid = $2`,
 		types.MRoomCreateNID, types.EmptyStateKeyNID,
 	)
@@ -115,7 +111,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 	batchsize := 100
 	for batchoffset := 0; batchoffset < snapshotcount; batchoffset += batchsize {
 		var snapshotrows *sql.Rows
-		snapshotrows, err = tx.Query(`
+		snapshotrows, err = tx.QueryContext(ctx, `
 			SELECT
 				state_snapshot_nid,
 				room_nid,
@@ -146,7 +142,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 				state_block_nid;
 		`, batchsize, batchoffset)
 		if err != nil {
-			return fmt.Errorf("tx.Query: %w", err)
+			return fmt.Errorf("tx.QueryContext: %w", err)
 		}
 
 		logrus.Warnf("Rewriting snapshots %d-%d of %d...", batchoffset, batchoffset+batchsize, snapshotcount)
@@ -183,7 +179,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		// fill in bad create snapshots
 		for _, s := range badCreateSnapshots {
 			var createEventNID types.EventNID
-			err = tx.QueryRow(
+			err = tx.QueryRowContext(ctx,
 				`SELECT event_nid FROM roomserver_events WHERE state_snapshot_nid = $1 AND event_type_nid = 1`, s.StateSnapshotNID,
 			).Scan(&createEventNID)
 			if err == sql.ErrNoRows {
@@ -208,7 +204,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 			}
 
 			var blocknid types.StateBlockNID
-			err = tx.QueryRow(`
+			err = tx.QueryRowContext(ctx, `
 				INSERT INTO roomserver_state_block (state_block_hash, event_nids)
 					VALUES ($1, $2)
 					ON CONFLICT (state_block_hash) DO UPDATE SET event_nids=$2
@@ -227,7 +223,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 			}
 
 			var newNID types.StateSnapshotNID
-			err = tx.QueryRow(`
+			err = tx.QueryRowContext(ctx, `
 					INSERT INTO roomserver_state_snapshots (state_snapshot_hash, room_nid, state_block_nids)
 						VALUES ($1, $2, $3)
 						ON CONFLICT (state_snapshot_hash) DO UPDATE SET room_nid=$2
@@ -237,12 +233,12 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 				return fmt.Errorf("tx.QueryRow.Scan (insert new snapshot): %w", err)
 			}
 
-			if _, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
-				return fmt.Errorf("tx.Exec (update events): %w", err)
+			if _, err = tx.ExecContext(ctx, `UPDATE roomserver_events SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
+				return fmt.Errorf("tx.ExecContext (update events): %w", err)
 			}
 
-			if _, err = tx.Exec(`UPDATE roomserver_rooms SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
-				return fmt.Errorf("tx.Exec (update rooms): %w", err)
+			if _, err = tx.ExecContext(ctx, `UPDATE roomserver_rooms SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newNID, snapshotdata.StateSnapshotNID, maxsnapshotid); err != nil {
+				return fmt.Errorf("tx.ExecContext (update rooms): %w", err)
 			}
 		}
 	}
@@ -252,13 +248,13 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 	// in roomserver_state_snapshots
 	var count int64
 
-	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_events WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
+	if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM roomserver_events WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
 		var res sql.Result
 		var c int64
-		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid)
+		res, err = tx.ExecContext(ctx, `UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid)
 		if err != nil && err != sql.ErrNoRows {
 			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
 		}
@@ -268,13 +264,13 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
 		}
 	}
-	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
+	if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
 		var debugRoomID string
 		var debugSnapNID, debugLastEventNID int64
-		err = tx.QueryRow(
+		err = tx.QueryRowContext(ctx,
 			`SELECT room_id, state_snapshot_nid, last_event_sent_nid FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid,
 		).Scan(&debugRoomID, &debugSnapNID, &debugLastEventNID)
 		if err != nil {
@@ -291,13 +287,13 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		return fmt.Errorf("%d rooms exist in roomserver_rooms which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
 	}
 
-	if _, err = tx.Exec(`
+	if _, err = tx.ExecContext(ctx, `
 		DROP TABLE _roomserver_state_snapshots;
 		DROP SEQUENCE roomserver_state_snapshot_nid_seq;
 	`); err != nil {
 		return fmt.Errorf("tx.Exec (delete old snapshot table): %w", err)
 	}
-	if _, err = tx.Exec(`
+	if _, err = tx.ExecContext(ctx, `
 		DROP TABLE _roomserver_state_block;
 		DROP SEQUENCE roomserver_state_block_nid_seq;
 	`); err != nil {
@@ -307,6 +303,6 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 	return nil
 }
 
-func DownStateBlocksRefactor(tx *sql.Tx) error {
+func DownStateBlocksRefactor(ctx context.Context, tx *sql.Tx) error {
 	panic("Downgrading state storage is not supported")
 }
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index eaaea22a1..bd3fd5592 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -23,6 +23,7 @@ import (
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -173,7 +174,15 @@ type membershipStatements struct {
 
 func CreateMembershipTable(db *sql.DB) error {
 	_, err := db.Exec(membershipSchema)
-	return err
+	if err != nil {
+		return err
+	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "roomserver: add forgotten column",
+		Up:      deltas.UpAddForgottenColumn,
+	})
+	return m.Up(context.Background())
 }
 
 func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) {
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index 70ea4d8ba..4c271ea9b 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -21,7 +21,6 @@ import (
 
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
-
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas"
@@ -45,17 +44,25 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 	}
 
 	// Create the tables.
-	if err := d.create(db); err != nil {
+	if err = d.create(db); err != nil {
 		return nil, err
 	}
 
-	// Then execute the migrations. By this point the tables are created with the latest
-	// schemas.
-	m := sqlutil.NewMigrations()
-	deltas.LoadAddForgottenColumn(m)
-	deltas.LoadStateBlocksRefactor(m)
-	if err := m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
+	// Special case, since this migration uses several tables, so it needs to
+	// be sure that all tables are created first.
+	// TODO: Remove when we are sure we are not having goose artefacts in the db
+	// This forces an error, which indicates the migration is already applied, since the
+	// column event_nid was removed from the table
+	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan()
+	if err == nil {
+		m := sqlutil.NewMigrator(db)
+		m.AddMigrations(sqlutil.Migration{
+			Version: "roomserver: state blocks refactor",
+			Up:      deltas.UpStateBlocksRefactor,
+		})
+		if err := m.Up(base.Context()); err != nil {
+			return nil, err
+		}
 	}
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
diff --git a/roomserver/storage/sqlite3/deltas/20201028212440_add_forgotten_column.go b/roomserver/storage/sqlite3/deltas/20201028212440_add_forgotten_column.go
index d08ab02d5..4c002e33d 100644
--- a/roomserver/storage/sqlite3/deltas/20201028212440_add_forgotten_column.go
+++ b/roomserver/storage/sqlite3/deltas/20201028212440_add_forgotten_column.go
@@ -15,24 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpAddForgottenColumn, DownAddForgottenColumn)
-	goose.AddMigration(UpStateBlocksRefactor, DownStateBlocksRefactor)
-}
-
-func LoadAddForgottenColumn(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddForgottenColumn, DownAddForgottenColumn)
-}
-
-func UpAddForgottenColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`	ALTER TABLE roomserver_membership RENAME TO roomserver_membership_tmp;
+func UpAddForgottenColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_membership RENAME TO roomserver_membership_tmp;
 CREATE TABLE IF NOT EXISTS roomserver_membership (
 		room_nid INTEGER NOT NULL,
 		target_nid INTEGER NOT NULL,
@@ -57,8 +46,8 @@ DROP TABLE roomserver_membership_tmp;`)
 	return nil
 }
 
-func DownAddForgottenColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`	ALTER TABLE roomserver_membership RENAME TO roomserver_membership_tmp;
+func DownAddForgottenColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_membership RENAME TO roomserver_membership_tmp;
 CREATE TABLE IF NOT EXISTS roomserver_membership (
 		room_nid INTEGER NOT NULL,
 		target_nid INTEGER NOT NULL,
diff --git a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go
index 8f5ab8fc5..00978121f 100644
--- a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go
+++ b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go
@@ -21,40 +21,35 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
-func LoadStateBlocksRefactor(m *sqlutil.Migrations) {
-	m.AddMigration(UpStateBlocksRefactor, DownStateBlocksRefactor)
-}
-
 // nolint:gocyclo
-func UpStateBlocksRefactor(tx *sql.Tx) error {
+func UpStateBlocksRefactor(ctx context.Context, tx *sql.Tx) error {
 	logrus.Warn("Performing state storage upgrade. Please wait, this may take some time!")
 	defer logrus.Warn("State storage upgrade complete")
 
 	var maxsnapshotid int
 	var maxblockid int
-	if err := tx.QueryRow(`SELECT IFNULL(MAX(state_snapshot_nid),0) FROM roomserver_state_snapshots;`).Scan(&maxsnapshotid); err != nil {
-		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
+	if err := tx.QueryRowContext(ctx, `SELECT IFNULL(MAX(state_snapshot_nid),0) FROM roomserver_state_snapshots;`).Scan(&maxsnapshotid); err != nil {
+		return fmt.Errorf("tx.QueryRowContext.Scan (count snapshots): %w", err)
 	}
-	if err := tx.QueryRow(`SELECT IFNULL(MAX(state_block_nid),0) FROM roomserver_state_block;`).Scan(&maxblockid); err != nil {
-		return fmt.Errorf("tx.QueryRow.Scan (count snapshots): %w", err)
+	if err := tx.QueryRowContext(ctx, `SELECT IFNULL(MAX(state_block_nid),0) FROM roomserver_state_block;`).Scan(&maxblockid); err != nil {
+		return fmt.Errorf("tx.QueryRowContext.Scan (count snapshots): %w", err)
 	}
 	maxsnapshotid++
 	maxblockid++
 	oldMaxSnapshotID := maxsnapshotid
 
-	if _, err := tx.Exec(`ALTER TABLE roomserver_state_block RENAME TO _roomserver_state_block;`); err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+	if _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_state_block RENAME TO _roomserver_state_block;`); err != nil {
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
-	if _, err := tx.Exec(`ALTER TABLE roomserver_state_snapshots RENAME TO _roomserver_state_snapshots;`); err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+	if _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_state_snapshots RENAME TO _roomserver_state_snapshots;`); err != nil {
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
-	_, err := tx.Exec(`
+	_, err := tx.ExecContext(ctx, `
 		CREATE TABLE IF NOT EXISTS roomserver_state_block (
 			state_block_nid INTEGER PRIMARY KEY AUTOINCREMENT,
 			state_block_hash BLOB UNIQUE,
@@ -62,9 +57,9 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		);
 	`)
 	if err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
-	_, err = tx.Exec(`
+	_, err = tx.ExecContext(ctx, `
 		CREATE TABLE IF NOT EXISTS roomserver_state_snapshots (
 			state_snapshot_nid INTEGER PRIMARY KEY AUTOINCREMENT,
 			state_snapshot_hash BLOB UNIQUE,
@@ -73,11 +68,11 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 	  	);
 	`)
 	if err != nil {
-		return fmt.Errorf("tx.Exec: %w", err)
+		return fmt.Errorf("tx.ExecContext: %w", err)
 	}
-	snapshotrows, err := tx.Query(`SELECT state_snapshot_nid, room_nid, state_block_nids FROM _roomserver_state_snapshots;`)
+	snapshotrows, err := tx.QueryContext(ctx, `SELECT state_snapshot_nid, room_nid, state_block_nids FROM _roomserver_state_snapshots;`)
 	if err != nil {
-		return fmt.Errorf("tx.Query: %w", err)
+		return fmt.Errorf("tx.QueryContext: %w", err)
 	}
 	defer internal.CloseAndLogIfError(context.TODO(), snapshotrows, "rows.close() failed")
 	for snapshotrows.Next() {
@@ -99,7 +94,7 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 			// in question a state snapshot NID of 0 to indicate 'no snapshot'.
 			// If we don't do this, we'll fail the assertions later on which try to ensure we didn't forget
 			// any snapshots.
-			_, err = tx.Exec(
+			_, err = tx.ExecContext(ctx,
 				`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE event_type_nid = $1 AND event_state_key_nid = $2 AND state_snapshot_nid = $3`,
 				types.MRoomCreateNID, types.EmptyStateKeyNID, snapshot,
 			)
@@ -109,9 +104,9 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		}
 		for _, block := range blocks {
 			if err = func() error {
-				blockrows, berr := tx.Query(`SELECT event_nid FROM _roomserver_state_block WHERE state_block_nid = $1`, block)
+				blockrows, berr := tx.QueryContext(ctx, `SELECT event_nid FROM _roomserver_state_block WHERE state_block_nid = $1`, block)
 				if berr != nil {
-					return fmt.Errorf("tx.Query (event nids from old block): %w", berr)
+					return fmt.Errorf("tx.QueryContext (event nids from old block): %w", berr)
 				}
 				defer internal.CloseAndLogIfError(context.TODO(), blockrows, "rows.close() failed")
 				events := types.EventNIDs{}
@@ -129,14 +124,14 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 				}
 
 				var blocknid types.StateBlockNID
-				err = tx.QueryRow(`
+				err = tx.QueryRowContext(ctx, `
 					INSERT INTO roomserver_state_block (state_block_nid, state_block_hash, event_nids)
 						VALUES ($1, $2, $3)
 						ON CONFLICT (state_block_hash) DO UPDATE SET event_nids=$3
 						RETURNING state_block_nid
 				`, maxblockid, events.Hash(), eventjson).Scan(&blocknid)
 				if err != nil {
-					return fmt.Errorf("tx.QueryRow.Scan (insert new block): %w", err)
+					return fmt.Errorf("tx.QueryRowContext.Scan (insert new block): %w", err)
 				}
 				maxblockid++
 				newblocks = append(newblocks, blocknid)
@@ -151,22 +146,22 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 			}
 
 			var newsnapshot types.StateSnapshotNID
-			err = tx.QueryRow(`
+			err = tx.QueryRowContext(ctx, `
 				INSERT INTO roomserver_state_snapshots (state_snapshot_nid, state_snapshot_hash, room_nid, state_block_nids)
 					VALUES ($1, $2, $3, $4)
 					ON CONFLICT (state_snapshot_hash) DO UPDATE SET room_nid=$3
 					RETURNING state_snapshot_nid
 			`, maxsnapshotid, newblocks.Hash(), room, newblocksjson).Scan(&newsnapshot)
 			if err != nil {
-				return fmt.Errorf("tx.QueryRow.Scan (insert new snapshot): %w", err)
+				return fmt.Errorf("tx.QueryRowContext.Scan (insert new snapshot): %w", err)
 			}
 			maxsnapshotid++
-			_, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newsnapshot, snapshot, maxsnapshotid)
+			_, err = tx.ExecContext(ctx, `UPDATE roomserver_events SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newsnapshot, snapshot, maxsnapshotid)
 			if err != nil {
-				return fmt.Errorf("tx.Exec (update events): %w", err)
+				return fmt.Errorf("tx.ExecContext (update events): %w", err)
 			}
-			if _, err = tx.Exec(`UPDATE roomserver_rooms SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newsnapshot, snapshot, maxsnapshotid); err != nil {
-				return fmt.Errorf("tx.Exec (update rooms): %w", err)
+			if _, err = tx.ExecContext(ctx, `UPDATE roomserver_rooms SET state_snapshot_nid=$1 WHERE state_snapshot_nid=$2 AND state_snapshot_nid<$3`, newsnapshot, snapshot, maxsnapshotid); err != nil {
+				return fmt.Errorf("tx.ExecContext (update rooms): %w", err)
 			}
 		}
 	}
@@ -175,13 +170,13 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 	// If we do, this is a problem if Dendrite tries to load the snapshot as it will not exist
 	// in roomserver_state_snapshots
 	var count int64
-	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_events WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil {
+	if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM roomserver_events WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
 		var res sql.Result
 		var c int64
-		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID)
+		res, err = tx.ExecContext(ctx, `UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID)
 		if err != nil && err != sql.ErrNoRows {
 			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
 		}
@@ -191,23 +186,23 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
 		}
 	}
-	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil {
+	if err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
 		return fmt.Errorf("%d rooms exist in roomserver_rooms which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
 	}
 
-	if _, err = tx.Exec(`DROP TABLE _roomserver_state_snapshots;`); err != nil {
+	if _, err = tx.ExecContext(ctx, `DROP TABLE _roomserver_state_snapshots;`); err != nil {
 		return fmt.Errorf("tx.Exec (delete old snapshot table): %w", err)
 	}
-	if _, err = tx.Exec(`DROP TABLE _roomserver_state_block;`); err != nil {
+	if _, err = tx.ExecContext(ctx, `DROP TABLE _roomserver_state_block;`); err != nil {
 		return fmt.Errorf("tx.Exec (delete old block table): %w", err)
 	}
 
 	return nil
 }
 
-func DownStateBlocksRefactor(tx *sql.Tx) error {
+func DownStateBlocksRefactor(ctx context.Context, tx *sql.Tx) error {
 	panic("Downgrading state storage is not supported")
 }
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index 9dda82fb2..f3303eb0e 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -148,7 +149,15 @@ type membershipStatements struct {
 
 func CreateMembershipTable(db *sql.DB) error {
 	_, err := db.Exec(membershipSchema)
-	return err
+	if err != nil {
+		return err
+	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "roomserver: add forgotten column",
+		Up:      deltas.UpAddForgottenColumn,
+	})
+	return m.Up(context.Background())
 }
 
 func PrepareMembershipTable(db *sql.DB) (tables.Membership, error) {
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 8325fdad5..bb9c15b5a 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -54,17 +54,25 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 	// db.SetMaxOpenConns(20)
 
 	// Create the tables.
-	if err := d.create(db); err != nil {
+	if err = d.create(db); err != nil {
 		return nil, err
 	}
 
-	// Then execute the migrations. By this point the tables are created with the latest
-	// schemas.
-	m := sqlutil.NewMigrations()
-	deltas.LoadAddForgottenColumn(m)
-	deltas.LoadStateBlocksRefactor(m)
-	if err := m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
+	// Special case, since this migration uses several tables, so it needs to
+	// be sure that all tables are created first.
+	// TODO: Remove when we are sure we are not having goose artefacts in the db
+	// This forces an error, which indicates the migration is already applied, since the
+	// column event_nid was removed from the table
+	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan()
+	if err == nil {
+		m := sqlutil.NewMigrator(db)
+		m.AddMigrations(sqlutil.Migration{
+			Version: "roomserver: state blocks refactor",
+			Up:      deltas.UpStateBlocksRefactor,
+		})
+		if err := m.Up(base.Context()); err != nil {
+			return nil, err
+		}
 	}
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index aae2d8c33..d13b7be41 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -23,6 +23,7 @@ import (
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -133,6 +134,17 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
 	if err != nil {
 		return nil, err
 	}
+
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: add history visibility column (current_room_state)",
+		Up:      deltas.UpAddHistoryVisibilityColumnCurrentRoomState,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/postgres/deltas/20201211125500_sequences.go b/syncapi/storage/postgres/deltas/20201211125500_sequences.go
index 7db524da5..6303c9472 100644
--- a/syncapi/storage/postgres/deltas/20201211125500_sequences.go
+++ b/syncapi/storage/postgres/deltas/20201211125500_sequences.go
@@ -15,24 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpFixSequences, DownFixSequences)
-	goose.AddMigration(UpRemoveSendToDeviceSentColumn, DownRemoveSendToDeviceSentColumn)
-}
-
-func LoadFixSequences(m *sqlutil.Migrations) {
-	m.AddMigration(UpFixSequences, DownFixSequences)
-}
-
-func UpFixSequences(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpFixSequences(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		-- We need to delete all of the existing receipts because the indexes
 		-- will be wrong, and we'll get primary key violations if we try to
 		-- reuse existing stream IDs from a different sequence.
@@ -49,8 +38,8 @@ func UpFixSequences(tx *sql.Tx) error {
 	return nil
 }
 
-func DownFixSequences(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownFixSequences(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		-- We need to delete all of the existing receipts because the indexes
 		-- will be wrong, and we'll get primary key violations if we try to
 		-- reuse existing stream IDs from a different sequence.
diff --git a/syncapi/storage/postgres/deltas/20210112130000_sendtodevice_sentcolumn.go b/syncapi/storage/postgres/deltas/20210112130000_sendtodevice_sentcolumn.go
index 3690eca8e..77b083ae2 100644
--- a/syncapi/storage/postgres/deltas/20210112130000_sendtodevice_sentcolumn.go
+++ b/syncapi/storage/postgres/deltas/20210112130000_sendtodevice_sentcolumn.go
@@ -15,18 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadRemoveSendToDeviceSentColumn(m *sqlutil.Migrations) {
-	m.AddMigration(UpRemoveSendToDeviceSentColumn, DownRemoveSendToDeviceSentColumn)
-}
-
-func UpRemoveSendToDeviceSentColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpRemoveSendToDeviceSentColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_send_to_device
 		  DROP COLUMN IF EXISTS sent_by_token;
 	`)
@@ -36,8 +31,8 @@ func UpRemoveSendToDeviceSentColumn(tx *sql.Tx) error {
 	return nil
 }
 
-func DownRemoveSendToDeviceSentColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownRemoveSendToDeviceSentColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_send_to_device
 		  ADD COLUMN IF NOT EXISTS sent_by_token TEXT;
 	`)
diff --git a/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
index 2b4b65287..29008adef 100644
--- a/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
+++ b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
@@ -15,20 +15,24 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadAddHistoryVisibilityColumn(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddHistoryVisibilityColumn, DownAddHistoryVisibilityColumn)
-}
-
-func UpAddHistoryVisibilityColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpAddHistoryVisibilityColumnOutputRoomEvents(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_output_room_events ADD COLUMN IF NOT EXISTS history_visibility SMALLINT NOT NULL DEFAULT 2;
 		UPDATE syncapi_output_room_events SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
+	`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func UpAddHistoryVisibilityColumnCurrentRoomState(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS history_visibility SMALLINT NOT NULL DEFAULT 2;
 		UPDATE syncapi_current_room_state SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
 	`)
@@ -38,8 +42,8 @@ func UpAddHistoryVisibilityColumn(tx *sql.Tx) error {
 	return nil
 }
 
-func DownAddHistoryVisibilityColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownAddHistoryVisibilityColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_output_room_events DROP COLUMN IF EXISTS history_visibility;
 		ALTER TABLE syncapi_current_room_state DROP COLUMN IF EXISTS history_visibility;
 	`)
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index ddef27383..34ff6700f 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 
@@ -188,6 +189,17 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: add history visibility column (output_room_events)",
+		Up:      deltas.UpAddHistoryVisibilityColumnOutputRoomEvents,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	return s, sqlutil.StatementList{
 		{&s.insertEventStmt, insertEventSQL},
 		{&s.selectEventsStmt, selectEventsSQL},
diff --git a/syncapi/storage/postgres/receipt_table.go b/syncapi/storage/postgres/receipt_table.go
index 2a42ffd74..bbddaa939 100644
--- a/syncapi/storage/postgres/receipt_table.go
+++ b/syncapi/storage/postgres/receipt_table.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -73,6 +74,15 @@ func NewPostgresReceiptsTable(db *sql.DB) (tables.Receipts, error) {
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: fix sequences",
+		Up:      deltas.UpFixSequences,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	r := &receiptStatements{
 		db: db,
 	}
diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go
index 96d6844fd..2734fef3e 100644
--- a/syncapi/storage/postgres/send_to_device_table.go
+++ b/syncapi/storage/postgres/send_to_device_table.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/sirupsen/logrus"
@@ -76,6 +77,15 @@ func NewPostgresSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) {
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: drop sent_by_token",
+		Up:      deltas.UpRemoveSendToDeviceSentColumn,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go
index c6121e419..a044716ce 100644
--- a/syncapi/storage/postgres/syncserver.go
+++ b/syncapi/storage/postgres/syncserver.go
@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
 )
 
@@ -42,16 +41,18 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
 		return nil, err
 	}
-	if _, err = d.db.Exec(outputRoomEventsSchema); err != nil {
-		return nil, err
-	}
-	if _, err = d.db.Exec(currentRoomStateSchema); err != nil {
-		return nil, err
-	}
 	accountData, err := NewPostgresAccountDataTable(d.db)
 	if err != nil {
 		return nil, err
 	}
+	events, err := NewPostgresEventsTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	currState, err := NewPostgresCurrentRoomStateTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	invites, err := NewPostgresInvitesTable(d.db)
 	if err != nil {
 		return nil, err
@@ -96,22 +97,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if err != nil {
 		return nil, err
 	}
-	m := sqlutil.NewMigrations()
-	deltas.LoadFixSequences(m)
-	deltas.LoadRemoveSendToDeviceSentColumn(m)
-	deltas.LoadAddHistoryVisibilityColumn(m)
-	if err = m.RunDeltas(d.db, dbProperties); err != nil {
-		return nil, err
-	}
-	// prepare statements after the migrations have run
-	events, err := NewPostgresEventsTable(d.db)
-	if err != nil {
-		return nil, err
-	}
-	currState, err := NewPostgresCurrentRoomStateTable(d.db)
-	if err != nil {
-		return nil, err
-	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Writer:              d.writer,
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 501d0ee98..e19298aee 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -24,6 +24,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -120,6 +121,17 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (t
 	if err != nil {
 		return nil, err
 	}
+
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: add history visibility column (current_room_state)",
+		Up:      deltas.UpAddHistoryVisibilityColumnCurrentRoomState,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/sqlite3/deltas/20201211125500_sequences.go b/syncapi/storage/sqlite3/deltas/20201211125500_sequences.go
index 8e7ebff86..f476335d5 100644
--- a/syncapi/storage/sqlite3/deltas/20201211125500_sequences.go
+++ b/syncapi/storage/sqlite3/deltas/20201211125500_sequences.go
@@ -15,24 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpFixSequences, DownFixSequences)
-	goose.AddMigration(UpRemoveSendToDeviceSentColumn, DownRemoveSendToDeviceSentColumn)
-}
-
-func LoadFixSequences(m *sqlutil.Migrations) {
-	m.AddMigration(UpFixSequences, DownFixSequences)
-}
-
-func UpFixSequences(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpFixSequences(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		-- We need to delete all of the existing receipts because the indexes
 		-- will be wrong, and we'll get primary key violations if we try to
 		-- reuse existing stream IDs from a different sequence.
@@ -45,8 +34,8 @@ func UpFixSequences(tx *sql.Tx) error {
 	return nil
 }
 
-func DownFixSequences(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownFixSequences(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		-- We need to delete all of the existing receipts because the indexes
 		-- will be wrong, and we'll get primary key violations if we try to
 		-- reuse existing stream IDs from a different sequence.
diff --git a/syncapi/storage/sqlite3/deltas/20210112130000_sendtodevice_sentcolumn.go b/syncapi/storage/sqlite3/deltas/20210112130000_sendtodevice_sentcolumn.go
index e0c514102..34cae2241 100644
--- a/syncapi/storage/sqlite3/deltas/20210112130000_sendtodevice_sentcolumn.go
+++ b/syncapi/storage/sqlite3/deltas/20210112130000_sendtodevice_sentcolumn.go
@@ -15,18 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadRemoveSendToDeviceSentColumn(m *sqlutil.Migrations) {
-	m.AddMigration(UpRemoveSendToDeviceSentColumn, DownRemoveSendToDeviceSentColumn)
-}
-
-func UpRemoveSendToDeviceSentColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpRemoveSendToDeviceSentColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		CREATE TEMPORARY TABLE syncapi_send_to_device_backup(id, user_id, device_id, content);
 		INSERT INTO syncapi_send_to_device_backup SELECT id, user_id, device_id, content FROM syncapi_send_to_device;
 		DROP TABLE syncapi_send_to_device;
@@ -45,8 +40,8 @@ func UpRemoveSendToDeviceSentColumn(tx *sql.Tx) error {
 	return nil
 }
 
-func DownRemoveSendToDeviceSentColumn(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownRemoveSendToDeviceSentColumn(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 		CREATE TEMPORARY TABLE syncapi_send_to_device_backup(id, user_id, device_id, content);
 		INSERT INTO syncapi_send_to_device_backup SELECT id, user_id, device_id, content FROM syncapi_send_to_device;
 		DROP TABLE syncapi_send_to_device;
diff --git a/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
index f6bcaddfb..079177217 100644
--- a/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
+++ b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
@@ -15,36 +15,36 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadAddHistoryVisibilityColumn(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddHistoryVisibilityColumn, DownAddHistoryVisibilityColumn)
-}
-
-func UpAddHistoryVisibilityColumn(tx *sql.Tx) error {
+func UpAddHistoryVisibilityColumnOutputRoomEvents(ctx context.Context, tx *sql.Tx) error {
 	// SQLite doesn't have "if exists", so check if the column exists. If the query doesn't return an error, it already exists.
 	// Required for unit tests, as otherwise a duplicate column error will show up.
-	_, err := tx.Query("SELECT history_visibility FROM syncapi_output_room_events LIMIT 1")
+	_, err := tx.QueryContext(ctx, "SELECT history_visibility FROM syncapi_output_room_events LIMIT 1")
 	if err == nil {
 		return nil
 	}
-	_, err = tx.Exec(`
+	_, err = tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_output_room_events ADD COLUMN history_visibility SMALLINT NOT NULL DEFAULT 2;
 		UPDATE syncapi_output_room_events SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
 	`)
 	if err != nil {
 		return fmt.Errorf("failed to execute upgrade: %w", err)
 	}
+	return nil
+}
 
-	_, err = tx.Query("SELECT history_visibility FROM syncapi_current_room_state LIMIT 1")
+func UpAddHistoryVisibilityColumnCurrentRoomState(ctx context.Context, tx *sql.Tx) error {
+	// SQLite doesn't have "if exists", so check if the column exists. If the query doesn't return an error, it already exists.
+	// Required for unit tests, as otherwise a duplicate column error will show up.
+	_, err := tx.QueryContext(ctx, "SELECT history_visibility FROM syncapi_current_room_state LIMIT 1")
 	if err == nil {
 		return nil
 	}
-	_, err = tx.Exec(`
+	_, err = tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_current_room_state ADD COLUMN history_visibility SMALLINT NOT NULL DEFAULT 2;
 		UPDATE syncapi_current_room_state SET history_visibility = 4 WHERE type IN ('m.room.message', 'm.room.encrypted');
 	`)
@@ -54,25 +54,25 @@ func UpAddHistoryVisibilityColumn(tx *sql.Tx) error {
 	return nil
 }
 
-func DownAddHistoryVisibilityColumn(tx *sql.Tx) error {
+func DownAddHistoryVisibilityColumn(ctx context.Context, tx *sql.Tx) error {
 	// SQLite doesn't have "if exists", so check if the column exists.
-	_, err := tx.Query("SELECT history_visibility FROM syncapi_output_room_events LIMIT 1")
+	_, err := tx.QueryContext(ctx, "SELECT history_visibility FROM syncapi_output_room_events LIMIT 1")
 	if err != nil {
 		// The column probably doesn't exist
 		return nil
 	}
-	_, err = tx.Exec(`
+	_, err = tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_output_room_events DROP COLUMN history_visibility;
 	`)
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
-	_, err = tx.Query("SELECT history_visibility FROM syncapi_current_room_state LIMIT 1")
+	_, err = tx.QueryContext(ctx, "SELECT history_visibility FROM syncapi_current_room_state LIMIT 1")
 	if err != nil {
 		// The column probably doesn't exist
 		return nil
 	}
-	_, err = tx.Exec(`
+	_, err = tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_current_room_state DROP COLUMN history_visibility;
 	`)
 	if err != nil {
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index b3dcb44cb..de389fa9b 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 
@@ -136,6 +137,17 @@ func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Even
 	if err != nil {
 		return nil, err
 	}
+
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: add history visibility column (output_room_events)",
+		Up:      deltas.UpAddHistoryVisibilityColumnOutputRoomEvents,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	return s, sqlutil.StatementList{
 		{&s.insertEventStmt, insertEventSQL},
 		{&s.selectMaxEventIDStmt, selectMaxEventIDSQL},
diff --git a/syncapi/storage/sqlite3/receipt_table.go b/syncapi/storage/sqlite3/receipt_table.go
index bd778bf3c..31adb005b 100644
--- a/syncapi/storage/sqlite3/receipt_table.go
+++ b/syncapi/storage/sqlite3/receipt_table.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -70,6 +71,15 @@ func NewSqliteReceiptsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Re
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: fix sequences",
+		Up:      deltas.UpFixSequences,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	r := &receiptStatements{
 		db:                 db,
 		streamIDStatements: streamID,
diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go
index 5285acbe6..d05d3fe72 100644
--- a/syncapi/storage/sqlite3/send_to_device_table.go
+++ b/syncapi/storage/sqlite3/send_to_device_table.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/sirupsen/logrus"
@@ -76,6 +77,15 @@ func NewSqliteSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) {
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "syncapi: drop sent_by_token",
+		Up:      deltas.UpRemoveSendToDeviceSentColumn,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index 39ceec81d..65b2bb38a 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -22,7 +22,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
-	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas"
 )
 
 // SyncServerDatasource represents a sync server datasource which manages
@@ -42,26 +41,28 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
 		return nil, err
 	}
-	if err = d.prepare(dbProperties); err != nil {
+	if err = d.prepare(); err != nil {
 		return nil, err
 	}
 	return &d, nil
 }
 
-func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (err error) {
+func (d *SyncServerDatasource) prepare() (err error) {
 	if err = d.streamID.Prepare(d.db); err != nil {
 		return err
 	}
-	if _, err = d.db.Exec(outputRoomEventsSchema); err != nil {
-		return err
-	}
-	if _, err = d.db.Exec(currentRoomStateSchema); err != nil {
-		return err
-	}
 	accountData, err := NewSqliteAccountDataTable(d.db, &d.streamID)
 	if err != nil {
 		return err
 	}
+	events, err := NewSqliteEventsTable(d.db, &d.streamID)
+	if err != nil {
+		return err
+	}
+	roomState, err := NewSqliteCurrentRoomStateTable(d.db, &d.streamID)
+	if err != nil {
+		return err
+	}
 	invites, err := NewSqliteInvitesTable(d.db, &d.streamID)
 	if err != nil {
 		return err
@@ -106,22 +107,6 @@ func (d *SyncServerDatasource) prepare(dbProperties *config.DatabaseOptions) (er
 	if err != nil {
 		return err
 	}
-	m := sqlutil.NewMigrations()
-	deltas.LoadFixSequences(m)
-	deltas.LoadRemoveSendToDeviceSentColumn(m)
-	deltas.LoadAddHistoryVisibilityColumn(m)
-	if err = m.RunDeltas(d.db, dbProperties); err != nil {
-		return err
-	}
-	// prepare statements after the migrations have run
-	events, err := NewSqliteEventsTable(d.db, &d.streamID)
-	if err != nil {
-		return err
-	}
-	roomState, err := NewSqliteCurrentRoomStateTable(d.db, &d.streamID)
-	if err != nil {
-		return err
-	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Writer:              d.writer,
diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go
index e3cab56ee..33fb6dd42 100644
--- a/userapi/storage/postgres/accounts_table.go
+++ b/userapi/storage/postgres/accounts_table.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 
 	log "github.com/sirupsen/logrus"
@@ -85,6 +86,23 @@ func NewPostgresAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerNam
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations([]sqlutil.Migration{
+		{
+			Version: "userapi: add is active",
+			Up:      deltas.UpIsActive,
+			Down:    deltas.DownIsActive,
+		},
+		{
+			Version: "userapi: add account type",
+			Up:      deltas.UpAddAccountType,
+			Down:    deltas.DownAddAccountType,
+		},
+	}...)
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	return s, sqlutil.StatementList{
 		{&s.insertAccountStmt, insertAccountSQL},
 		{&s.updatePasswordStmt, updatePasswordSQL},
diff --git a/userapi/storage/postgres/deltas/20200929203058_is_active.go b/userapi/storage/postgres/deltas/20200929203058_is_active.go
index 32d3235be..24f87e073 100644
--- a/userapi/storage/postgres/deltas/20200929203058_is_active.go
+++ b/userapi/storage/postgres/deltas/20200929203058_is_active.go
@@ -1,33 +1,21 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/pressly/goose"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpIsActive, DownIsActive)
-	goose.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func LoadIsActive(m *sqlutil.Migrations) {
-	m.AddMigration(UpIsActive, DownIsActive)
-}
-
-func UpIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec("ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;")
+func UpIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;")
 	if err != nil {
 		return fmt.Errorf("failed to execute upgrade: %w", err)
 	}
 	return nil
 }
 
-func DownIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN is_deactivated;")
+func DownIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts DROP COLUMN is_deactivated;")
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
diff --git a/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go
index 1bbb0a9d3..edd3353f0 100644
--- a/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go
+++ b/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go
@@ -1,18 +1,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadLastSeenTSIP(m *sqlutil.Migrations) {
-	m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP)
-}
-
-func UpLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000;
 ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT;
 ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`)
@@ -22,8 +17,8 @@ ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`)
 	return nil
 }
 
-func DownLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	ALTER TABLE device_devices DROP COLUMN last_seen_ts;
 	ALTER TABLE device_devices DROP COLUMN ip;
 	ALTER TABLE device_devices DROP COLUMN user_agent;`)
diff --git a/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go b/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go
index 2fae00cb9..eb7c3a958 100644
--- a/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go
+++ b/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go
@@ -1,20 +1,15 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadAddAccountType(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func UpAddAccountType(tx *sql.Tx) error {
+func UpAddAccountType(ctx context.Context, tx *sql.Tx) error {
 	// initially set every account to useraccount, change appservice and guest accounts afterwards
 	// (user = 1, guest = 2, admin = 3, appservice = 4)
-	_, err := tx.Exec(`ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS account_type SMALLINT NOT NULL DEFAULT 1;
+	_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS account_type SMALLINT NOT NULL DEFAULT 1;
 UPDATE account_accounts SET account_type = 4 WHERE appservice_id <> '';
 UPDATE account_accounts SET account_type = 2 WHERE localpart ~ '^[0-9]+$';
 ALTER TABLE account_accounts ALTER COLUMN account_type DROP DEFAULT;`,
@@ -25,8 +20,8 @@ ALTER TABLE account_accounts ALTER COLUMN account_type DROP DEFAULT;`,
 	return nil
 }
 
-func DownAddAccountType(tx *sql.Tx) error {
-	_, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN account_type;")
+func DownAddAccountType(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts DROP COLUMN account_type;")
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go
index ccb776672..f65681aae 100644
--- a/userapi/storage/postgres/devices_table.go
+++ b/userapi/storage/postgres/devices_table.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -120,6 +121,15 @@ func NewPostgresDevicesTable(db *sql.DB, serverName gomatrixserverlib.ServerName
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "userapi: add last_seen_ts",
+		Up:      deltas.UpLastSeenTSIP,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	return s, sqlutil.StatementList{
 		{&s.insertDeviceStmt, insertDeviceSQL},
 		{&s.selectDeviceByTokenStmt, selectDeviceByTokenSQL},
diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go
index b9afb5a56..7d3b9b6a5 100644
--- a/userapi/storage/postgres/storage.go
+++ b/userapi/storage/postgres/storage.go
@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
 
 	// Import the postgres database driver.
@@ -37,19 +36,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		return nil, err
 	}
 
-	m := sqlutil.NewMigrations()
-	if _, err = db.Exec(accountsSchema); err != nil {
-		// do this so that the migration can and we don't fail on
-		// preparing statements for columns that don't exist yet
-		return nil, err
-	}
-	deltas.LoadIsActive(m)
-	//deltas.LoadLastSeenTSIP(m)
-	deltas.LoadAddAccountType(m)
-	if err = m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
-	}
-
 	accountDataTable, err := NewPostgresAccountDataTable(db)
 	if err != nil {
 		return nil, fmt.Errorf("NewPostgresAccountDataTable: %w", err)
diff --git a/userapi/storage/sqlite3/accounts_table.go b/userapi/storage/sqlite3/accounts_table.go
index 6c5fe3071..484e90056 100644
--- a/userapi/storage/sqlite3/accounts_table.go
+++ b/userapi/storage/sqlite3/accounts_table.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 
 	log "github.com/sirupsen/logrus"
@@ -87,6 +88,23 @@ func NewSQLiteAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations([]sqlutil.Migration{
+		{
+			Version: "userapi: add is active",
+			Up:      deltas.UpIsActive,
+			Down:    deltas.DownIsActive,
+		},
+		{
+			Version: "userapi: add account type",
+			Up:      deltas.UpAddAccountType,
+			Down:    deltas.DownAddAccountType,
+		},
+	}...)
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	return s, sqlutil.StatementList{
 		{&s.insertAccountStmt, insertAccountSQL},
 		{&s.updatePasswordStmt, updatePasswordSQL},
diff --git a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go
index c69614e83..e25efc695 100644
--- a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go
+++ b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go
@@ -1,25 +1,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/pressly/goose"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpIsActive, DownIsActive)
-	goose.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func LoadIsActive(m *sqlutil.Migrations) {
-	m.AddMigration(UpIsActive, DownIsActive)
-}
-
-func UpIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
@@ -42,8 +30,8 @@ DROP TABLE account_accounts_tmp;`)
 	return nil
 }
 
-func DownIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
diff --git a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go
index ebf908001..7f7e95d2d 100644
--- a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go
+++ b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go
@@ -1,18 +1,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadLastSeenTSIP(m *sqlutil.Migrations) {
-	m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP)
-}
-
-func UpLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
     ALTER TABLE device_devices RENAME TO device_devices_tmp;
     CREATE TABLE device_devices (
         access_token TEXT PRIMARY KEY,
@@ -39,8 +34,8 @@ func UpLastSeenTSIP(tx *sql.Tx) error {
 	return nil
 }
 
-func DownLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 ALTER TABLE device_devices RENAME TO device_devices_tmp;
 CREATE TABLE IF NOT EXISTS device_devices (
     access_token TEXT PRIMARY KEY,
diff --git a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go
index 9b058dedd..46532698c 100644
--- a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go
+++ b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go
@@ -1,26 +1,15 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/pressly/goose"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func init() {
-	goose.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func LoadAddAccountType(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func UpAddAccountType(tx *sql.Tx) error {
+func UpAddAccountType(ctx context.Context, tx *sql.Tx) error {
 	// initially set every account to useraccount, change appservice and guest accounts afterwards
 	// (user = 1, guest = 2, admin = 3, appservice = 4)
-	_, err := tx.Exec(`ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
+	_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
     created_ts BIGINT NOT NULL,
@@ -45,8 +34,8 @@ DROP TABLE account_accounts_tmp;`)
 	return nil
 }
 
-func DownAddAccountType(tx *sql.Tx) error {
-	_, err := tx.Exec(`ALTER TABLE account_accounts DROP COLUMN account_type;`)
+func DownAddAccountType(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts DROP COLUMN account_type;`)
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go
index 93291e6ad..27a7524d6 100644
--- a/userapi/storage/sqlite3/devices_table.go
+++ b/userapi/storage/sqlite3/devices_table.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 
 	"github.com/matrix-org/dendrite/clientapi/userutil"
@@ -107,6 +108,15 @@ func NewSQLiteDevicesTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "userapi: add last_seen_ts",
+		Up:      deltas.UpLastSeenTSIP,
+	})
+	if err = m.Up(context.Background()); err != nil {
+		return nil, err
+	}
+
 	return s, sqlutil.StatementList{
 		{&s.insertDeviceStmt, insertDeviceSQL},
 		{&s.selectDevicesCountStmt, selectDevicesCountSQL},
diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go
index a822f687d..78b7ce588 100644
--- a/userapi/storage/sqlite3/storage.go
+++ b/userapi/storage/sqlite3/storage.go
@@ -25,10 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
-	"github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
-
-	// Import the postgres database driver.
-	_ "github.com/lib/pq"
 )
 
 // NewDatabase creates a new accounts and profiles database
@@ -38,19 +34,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		return nil, err
 	}
 
-	m := sqlutil.NewMigrations()
-	if _, err = db.Exec(accountsSchema); err != nil {
-		// do this so that the migration can and we don't fail on
-		// preparing statements for columns that don't exist yet
-		return nil, err
-	}
-	deltas.LoadIsActive(m)
-	//deltas.LoadLastSeenTSIP(m)
-	deltas.LoadAddAccountType(m)
-	if err = m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
-	}
-
 	accountDataTable, err := NewSQLiteAccountDataTable(db)
 	if err != nil {
 		return nil, fmt.Errorf("NewSQLiteAccountDataTable: %w", err)

From b836243a24b7928d9986c97d0ae66be57f71ab42 Mon Sep 17 00:00:00 2001
From: Jonathan Bartlett <34320158+Jonnobrow@users.noreply.github.com>
Date: Mon, 25 Jul 2022 10:39:57 +0100
Subject: [PATCH 219/391] Add .well-known/matrix/client to clientapi (#2551)

Signed-off-by: Jonathan Bartlett <jonathan@jonnobrow.co.uk>

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 clientapi/clientapi.go        |  1 +
 clientapi/routing/routing.go  | 22 +++++++++++++++++++++-
 dendrite-sample.monolith.yaml |  4 ++++
 dendrite-sample.polylith.yaml |  4 ++++
 setup/config/config_global.go |  3 +++
 setup/config/config_test.go   |  1 +
 6 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index f550c29bb..557355604 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -59,6 +59,7 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		base.PublicClientAPIMux,
+		base.PublicWellKnownAPIMux,
 		base.SynapseAdminMux,
 		base.DendriteAdminMux,
 		cfg, rsAPI, asAPI,
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 0460850ef..ced4fdbcf 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -48,7 +48,7 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
+	publicAPIMux, wkMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
@@ -74,6 +74,26 @@ func Setup(
 		unstableFeatures["org.matrix."+msc] = true
 	}
 
+	if cfg.Matrix.WellKnownClientName != "" {
+		logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName)
+		wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: struct {
+					HomeserverName struct {
+						BaseUrl string `json:"base_url"`
+					} `json:"m.homeserver"`
+				}{
+					HomeserverName: struct {
+						BaseUrl string `json:"base_url"`
+					}{
+						BaseUrl: cfg.Matrix.WellKnownClientName,
+					},
+				},
+			}
+		})).Methods(http.MethodGet, http.MethodOptions)
+	}
+
 	publicAPIMux.Handle("/versions",
 		httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{
diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index c6050e407..cc6c173e8 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -64,6 +64,10 @@ global:
   # e.g. localhost:443
   well_known_server_name: ""
 
+  # The server name to delegate client-server communications to, with optional port
+  # e.g. localhost:443
+  well_known_client_name: ""
+
   # Lists of domains that the server will trust as identity servers to verify third
   # party identifiers such as phone numbers and email addresses.
   trusted_third_party_id_servers:
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 723547f55..92cab19b1 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -54,6 +54,10 @@ global:
   # e.g. localhost:443
   well_known_server_name: ""
 
+  # The server name to delegate client-server communications to, with optional port
+  # e.g. localhost:443
+  well_known_client_name: ""
+
   # Lists of domains that the server will trust as identity servers to verify third
   # party identifiers such as phone numbers and email addresses.
   trusted_third_party_id_servers:
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index 10827bb21..d4e54e203 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -46,6 +46,9 @@ type Global struct {
 	// The server name to delegate server-server communications to, with optional port
 	WellKnownServerName string `yaml:"well_known_server_name"`
 
+	// The server name to delegate client-server communications to, with optional port
+	WellKnownClientName string `yaml:"well_known_client_name"`
+
 	// Disables federation. Dendrite will not be able to make any outbound HTTP requests
 	// to other servers and the federation API will not be exposed.
 	DisableFederation bool `yaml:"disable_federation"`
diff --git a/setup/config/config_test.go b/setup/config/config_test.go
index b9b1e7bb5..ee7e7389c 100644
--- a/setup/config/config_test.go
+++ b/setup/config/config_test.go
@@ -42,6 +42,7 @@ global:
   key_id: ed25519:auto
   key_validity_period: 168h0m0s
   well_known_server_name: "localhost:443"
+  well_known_client_name: "localhost:443"
   trusted_third_party_id_servers:
   - matrix.org
   - vector.im

From ccea23cd4083aba534b96aead02c6cd10f95f833 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 25 Jul 2022 11:42:22 +0100
Subject: [PATCH 220/391] Remove `room_id` field from MSC2946 stripped events
 (closes #2588)

---
 go.mod                        | 2 +-
 go.sum                        | 4 ++--
 setup/mscs/msc2946/msc2946.go | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index 79216f0fa..377a156fe 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771
 	github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 7107d4ff6..10b1638a5 100644
--- a/go.sum
+++ b/go.sum
@@ -341,8 +341,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d h1:BWInUURXVOW+OiifMapoRIS7i122KWdEKj6fnDFXgBo=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220718085240-f08f98af7d2d/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771 h1:ZIPHFIPNDS9dmEbPEiJbNmyCGJtn9exfpLC7JOcn/bE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs=
 github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go
index 61520d50e..4cffa82ad 100644
--- a/setup/mscs/msc2946/msc2946.go
+++ b/setup/mscs/msc2946/msc2946.go
@@ -708,7 +708,6 @@ func stripped(ev *gomatrixserverlib.Event) *gomatrixserverlib.MSC2946StrippedEve
 		StateKey:       *ev.StateKey(),
 		Content:        ev.Content(),
 		Sender:         ev.Sender(),
-		RoomID:         ev.RoomID(),
 		OriginServerTS: ev.OriginServerTS(),
 	}
 }

From 497ab4e1b7cb4225f57b87932a3dccfbbec74463 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 25 Jul 2022 14:46:35 +0100
Subject: [PATCH 221/391] Remove `goose` from Dockerfiles

---
 build/docker/Dockerfile.monolith | 1 -
 build/docker/Dockerfile.polylith | 1 -
 2 files changed, 2 deletions(-)

diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
index 891a3a9e0..bb02934cd 100644
--- a/build/docker/Dockerfile.monolith
+++ b/build/docker/Dockerfile.monolith
@@ -8,7 +8,6 @@ COPY . /build
 
 RUN mkdir -p bin
 RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
-RUN go build -trimpath -o bin/ ./cmd/goose
 RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
index ffdc35586..166ea99cb 100644
--- a/build/docker/Dockerfile.polylith
+++ b/build/docker/Dockerfile.polylith
@@ -8,7 +8,6 @@ COPY . /build
 
 RUN mkdir -p bin
 RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
-RUN go build -trimpath -o bin/ ./cmd/goose
 RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 

From 962b76da445e8af408d7473abfa0cb9d00423264 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 25 Jul 2022 17:30:07 +0100
Subject: [PATCH 222/391] Make the User API responsible for sending account
 data output events (#2592)

* Make the User API responsible for sending account data output events

* Clean up producer

* Review comments
---
 clientapi/clientapi.go            |  1 -
 clientapi/producers/syncapi.go    | 32 -------------------------------
 clientapi/routing/account_data.go | 18 -----------------
 clientapi/routing/room_tagging.go | 11 -----------
 userapi/internal/api.go           | 31 +++++++++++++++++++++++++-----
 userapi/producers/syncapi.go      | 11 ++++-------
 6 files changed, 30 insertions(+), 74 deletions(-)

diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go
index 557355604..080d4d9fa 100644
--- a/clientapi/clientapi.go
+++ b/clientapi/clientapi.go
@@ -48,7 +48,6 @@ func AddPublicRoutes(
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
-		TopicClientData:        cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
 		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
 		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go
index 0ac637793..5933ce1a8 100644
--- a/clientapi/producers/syncapi.go
+++ b/clientapi/producers/syncapi.go
@@ -21,7 +21,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -32,7 +31,6 @@ import (
 
 // SyncAPIProducer produces events for the sync API server to consume
 type SyncAPIProducer struct {
-	TopicClientData        string
 	TopicReceiptEvent      string
 	TopicSendToDeviceEvent string
 	TopicTypingEvent       string
@@ -42,36 +40,6 @@ type SyncAPIProducer struct {
 	UserAPI                userapi.ClientUserAPI
 }
 
-// SendData sends account data to the sync API server
-func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON, ignoredUsers *types.IgnoredUsers) error {
-	m := &nats.Msg{
-		Subject: p.TopicClientData,
-		Header:  nats.Header{},
-	}
-	m.Header.Set(jetstream.UserID, userID)
-
-	data := eventutil.AccountData{
-		RoomID:       roomID,
-		Type:         dataType,
-		ReadMarker:   readMarker,
-		IgnoredUsers: ignoredUsers,
-	}
-	var err error
-	m.Data, err = json.Marshal(data)
-	if err != nil {
-		return err
-	}
-
-	log.WithFields(log.Fields{
-		"user_id":   userID,
-		"room_id":   roomID,
-		"data_type": dataType,
-	}).Tracef("Producing to topic '%s'", p.TopicClientData)
-
-	_, err = p.JetStream.PublishMsg(m)
-	return err
-}
-
 func (p *SyncAPIProducer) SendReceipt(
 	ctx context.Context,
 	userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp,
diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go
index a5a3014ab..0d3a49495 100644
--- a/clientapi/routing/account_data.go
+++ b/clientapi/routing/account_data.go
@@ -25,7 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/util"
@@ -127,18 +126,6 @@ func SaveAccountData(
 		return util.ErrorResponse(err)
 	}
 
-	var ignoredUsers *types.IgnoredUsers
-	if dataType == "m.ignored_user_list" {
-		ignoredUsers = &types.IgnoredUsers{}
-		_ = json.Unmarshal(body, ignoredUsers)
-	}
-
-	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
-		return jsonerror.InternalServerError()
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
@@ -191,11 +178,6 @@ func SaveReadMarker(
 		return util.ErrorResponse(err)
 	}
 
-	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
-		return jsonerror.InternalServerError()
-	}
-
 	// Handle the read receipt that may be included in the read marker
 	if r.Read != "" {
 		return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)
diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go
index 039289569..92b9e6655 100644
--- a/clientapi/routing/room_tagging.go
+++ b/clientapi/routing/room_tagging.go
@@ -18,8 +18,6 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/sirupsen/logrus"
-
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
@@ -98,10 +96,6 @@ func PutTag(
 		return jsonerror.InternalServerError()
 	}
 
-	if err = syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
-		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
@@ -150,11 +144,6 @@ func DeleteTag(
 		return jsonerror.InternalServerError()
 	}
 
-	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
-		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 27ed15a01..422eb076e 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -30,11 +30,13 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/pushrules"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	synctypes "github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/producers"
 	"github.com/matrix-org/dendrite/userapi/storage"
@@ -64,7 +66,24 @@ func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAc
 	if req.DataType == "" {
 		return fmt.Errorf("data type must not be empty")
 	}
-	return a.DB.SaveAccountData(ctx, local, req.RoomID, req.DataType, req.AccountData)
+	if err := a.DB.SaveAccountData(ctx, local, req.RoomID, req.DataType, req.AccountData); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("a.DB.SaveAccountData failed")
+		return fmt.Errorf("failed to save account data: %w", err)
+	}
+	var ignoredUsers *synctypes.IgnoredUsers
+	if req.DataType == "m.ignored_user_list" {
+		ignoredUsers = &synctypes.IgnoredUsers{}
+		_ = json.Unmarshal(req.AccountData, ignoredUsers)
+	}
+	if err := a.SyncProducer.SendAccountData(req.UserID, eventutil.AccountData{
+		RoomID:       req.RoomID,
+		Type:         req.DataType,
+		IgnoredUsers: ignoredUsers,
+	}); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("a.SyncProducer.SendAccountData failed")
+		return fmt.Errorf("failed to send account data to output: %w", err)
+	}
+	return nil
 }
 
 func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error {
@@ -93,7 +112,9 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P
 	}
 
 	// Inform the SyncAPI about the newly created push_rules
-	if err = a.SyncProducer.SendAccountData(acc.UserID, "", "m.push_rules"); err != nil {
+	if err = a.SyncProducer.SendAccountData(acc.UserID, eventutil.AccountData{
+		Type: "m.push_rules",
+	}); err != nil {
 		util.GetLogger(ctx).WithFields(logrus.Fields{
 			"user_id": acc.UserID,
 		}).WithError(err).Warn("failed to send account data to the SyncAPI")
@@ -732,11 +753,11 @@ func (a *UserInternalAPI) PerformPushRulesPut(
 	if err := a.InputAccountData(ctx, &userReq, &userRes); err != nil {
 		return err
 	}
-
-	if err := a.SyncProducer.SendAccountData(req.UserID, "" /* roomID */, pushRulesAccountDataType); err != nil {
+	if err := a.SyncProducer.SendAccountData(req.UserID, eventutil.AccountData{
+		Type: pushRulesAccountDataType,
+	}); err != nil {
 		util.GetLogger(ctx).WithError(err).Errorf("syncProducer.SendData failed")
 	}
-
 	return nil
 }
 
diff --git a/userapi/producers/syncapi.go b/userapi/producers/syncapi.go
index 4a206f333..27cfc2848 100644
--- a/userapi/producers/syncapi.go
+++ b/userapi/producers/syncapi.go
@@ -34,7 +34,7 @@ func NewSyncAPI(db storage.Database, js JetStreamPublisher, clientDataTopic stri
 }
 
 // SendAccountData sends account data to the Sync API server.
-func (p *SyncAPI) SendAccountData(userID string, roomID string, dataType string) error {
+func (p *SyncAPI) SendAccountData(userID string, data eventutil.AccountData) error {
 	m := &nats.Msg{
 		Subject: p.clientDataTopic,
 		Header:  nats.Header{},
@@ -42,18 +42,15 @@ func (p *SyncAPI) SendAccountData(userID string, roomID string, dataType string)
 	m.Header.Set(jetstream.UserID, userID)
 
 	var err error
-	m.Data, err = json.Marshal(eventutil.AccountData{
-		RoomID: roomID,
-		Type:   dataType,
-	})
+	m.Data, err = json.Marshal(data)
 	if err != nil {
 		return err
 	}
 
 	log.WithFields(log.Fields{
 		"user_id":   userID,
-		"room_id":   roomID,
-		"data_type": dataType,
+		"room_id":   data.RoomID,
+		"data_type": data.Type,
 	}).Tracef("Producing to topic '%s'", p.clientDataTopic)
 
 	_, err = p.producer.PublishMsg(m)

From 645f31ae24610110ee2c3100d228677fb3166b26 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 1 Aug 2022 09:51:13 +0100
Subject: [PATCH 223/391] Update NATS Server and nats.go to use upstream

---
 go.mod | 14 +++++---------
 go.sum | 20 ++++++++++----------
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/go.mod b/go.mod
index 377a156fe..5b2d8670d 100644
--- a/go.mod
+++ b/go.mod
@@ -1,9 +1,5 @@
 module github.com/matrix-org/dendrite
 
-replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f
-
-replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673
-
 require (
 	github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
@@ -29,8 +25,8 @@ require (
 	github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
-	github.com/nats-io/nats-server/v2 v2.7.4-0.20220309205833-773636c1c5bb
-	github.com/nats-io/nats.go v1.14.0
+	github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee
+	github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
@@ -76,7 +72,7 @@ require (
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
-	github.com/klauspost/compress v1.14.4 // indirect
+	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/lucas-clemente/quic-go v0.26.0 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
 	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
@@ -86,7 +82,7 @@ require (
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect
+	github.com/nats-io/jwt/v2 v2.3.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
@@ -104,7 +100,7 @@ require (
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
+	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
 	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
diff --git a/go.sum b/go.sum
index 10b1638a5..6321adada 100644
--- a/go.sum
+++ b/go.sum
@@ -302,8 +302,8 @@ github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloL
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4=
-github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -381,8 +381,12 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a h1:lem6QCvxR0Y28gth9P+wV2K/zYUUAkJ+55U8cpS0p5I=
-github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
+github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
+github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
+github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee h1:vAtoZ+LW6eIUjkCWWwO1DZ6o16UGrVOG+ot/AkwejO8=
+github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee/go.mod h1:3Yg3ApyQxPlAs1KKHKV5pobV5VtZk+TtOiUJx/iqkkg=
+github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b h1:CE9wSYLvwq8aC/0+6zH8lhhtZYvJ9p8PzwvZeYgdBc0=
+github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
 github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -390,10 +394,6 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo=
-github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4=
-github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673 h1:TcKfa3Tf0qwUotv63PQVu2d1bBoLi2iEA4RHVMGDh5M=
-github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
@@ -764,8 +764,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M=
-golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
+golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From c7f7aec4d07d59120d37d5b16a900f6d608a75c4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 1 Aug 2022 11:34:27 +0100
Subject: [PATCH 224/391] Set CORS headers for HTTP 404 and 405 errors (#2599)

* Set CORS headers for the 404s

* Use custom handlers, plus one for HTTP 405 too

* Tweak setup

* Add to muxes too

* Tidy up some more

* Use built-in HTTP 404 handler

* Don't bother setting it for federation-facing
---
 setup/base/base.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/setup/base/base.go b/setup/base/base.go
index 93ab87de1..b21eeba47 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -369,6 +369,25 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli
 	return client
 }
 
+func (b *BaseDendrite) configureHTTPErrors() {
+	notAllowedHandler := func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		_, _ = w.Write([]byte(fmt.Sprintf("405 %s not allowed on this endpoint", r.Method)))
+	}
+
+	notFoundCORSHandler := httputil.WrapHandlerInCORS(http.NotFoundHandler())
+	notAllowedCORSHandler := httputil.WrapHandlerInCORS(http.HandlerFunc(notAllowedHandler))
+
+	for _, router := range []*mux.Router{
+		b.PublicClientAPIMux, b.PublicMediaAPIMux,
+		b.DendriteAdminMux, b.SynapseAdminMux,
+		b.PublicWellKnownAPIMux,
+	} {
+		router.NotFoundHandler = notFoundCORSHandler
+		router.MethodNotAllowedHandler = notAllowedCORSHandler
+	}
+}
+
 // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on
 // ApiMux under /api/ and adds a prometheus handler under /metrics.
 func (b *BaseDendrite) SetupAndServeHTTP(
@@ -409,6 +428,8 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		}
 	}
 
+	b.configureHTTPErrors()
+
 	internalRouter.PathPrefix(httputil.InternalPathPrefix).Handler(b.InternalAPIMux)
 	if b.Cfg.Global.Metrics.Enabled {
 		internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth))

From 05c83923e3bf24fa7def55a14ac096cdff3a3882 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 1 Aug 2022 14:11:00 +0100
Subject: [PATCH 225/391] Optimise checking other servers allowed to see events
 (#2596)

* Try optimising checking if server is allowed to see event

* Fix error

* Handle case where snapshot NID is 0

* Fix query

* Update SQL

* Clean up `CheckServerAllowedToSeeEvent`

* Not supported on SQLite

* Maybe placate the unit tests

* Review comments
---
 roomserver/internal/helpers/helpers.go        | 36 +++++++++----
 roomserver/state/state.go                     | 23 +++++++++
 roomserver/storage/interface.go               |  2 +
 .../storage/postgres/state_snapshot_table.go  | 51 ++++++++++++++++++-
 roomserver/storage/shared/storage.go          | 32 ++++++++++++
 .../storage/sqlite3/state_snapshot_table.go   |  6 +++
 roomserver/storage/tables/interface.go        |  7 +++
 .../tables/state_snapshot_table_test.go       |  9 ++++
 8 files changed, 154 insertions(+), 12 deletions(-)

diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index 2653027e9..16a6f615d 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -236,13 +236,34 @@ func LoadStateEvents(
 func CheckServerAllowedToSeeEvent(
 	ctx context.Context, db storage.Database, info *types.RoomInfo, eventID string, serverName gomatrixserverlib.ServerName, isServerInRoom bool,
 ) (bool, error) {
+	stateAtEvent, err := db.GetHistoryVisibilityState(ctx, info, eventID, string(serverName))
+	switch err {
+	case nil:
+		// No error, so continue normally
+	case tables.OptimisationNotSupportedError:
+		// The database engine didn't support this optimisation, so fall back to using
+		// the old and slow method
+		stateAtEvent, err = slowGetHistoryVisibilityState(ctx, db, info, eventID, serverName)
+		if err != nil {
+			return false, err
+		}
+	default:
+		// Something else went wrong
+		return false, err
+	}
+	return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil
+}
+
+func slowGetHistoryVisibilityState(
+	ctx context.Context, db storage.Database, info *types.RoomInfo, eventID string, serverName gomatrixserverlib.ServerName,
+) ([]*gomatrixserverlib.Event, error) {
 	roomState := state.NewStateResolution(db, info)
 	stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID)
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) {
-			return false, nil
+			return nil, nil
 		}
-		return false, fmt.Errorf("roomState.LoadStateAtEvent: %w", err)
+		return nil, fmt.Errorf("roomState.LoadStateAtEvent: %w", err)
 	}
 
 	// Extract all of the event state key NIDs from the room state.
@@ -254,7 +275,7 @@ func CheckServerAllowedToSeeEvent(
 	// Then request those state key NIDs from the database.
 	stateKeys, err := db.EventStateKeys(ctx, stateKeyNIDs)
 	if err != nil {
-		return false, fmt.Errorf("db.EventStateKeys: %w", err)
+		return nil, fmt.Errorf("db.EventStateKeys: %w", err)
 	}
 
 	// If the event state key doesn't match the given servername
@@ -277,15 +298,10 @@ func CheckServerAllowedToSeeEvent(
 	}
 
 	if len(filteredEntries) == 0 {
-		return false, nil
+		return nil, nil
 	}
 
-	stateAtEvent, err := LoadStateEvents(ctx, db, filteredEntries)
-	if err != nil {
-		return false, err
-	}
-
-	return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil
+	return LoadStateEvents(ctx, db, filteredEntries)
 }
 
 // TODO: Remove this when we have tests to assert correctness of this function
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index d1d24b099..ca0c69f27 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -124,6 +124,29 @@ func (v *StateResolution) LoadStateAtEvent(
 	return stateEntries, nil
 }
 
+// LoadStateAtEvent loads the full state of a room before a particular event.
+func (v *StateResolution) LoadStateAtEventForHistoryVisibility(
+	ctx context.Context, eventID string,
+) ([]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAtEvent")
+	defer span.Finish()
+
+	snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID)
+	if err != nil {
+		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %w", eventID, err)
+	}
+	if snapshotNID == 0 {
+		return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID)
+	}
+
+	stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID)
+	if err != nil {
+		return nil, err
+	}
+
+	return stateEntries, nil
+}
+
 // LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events
 // and combines those snapshots together into a single list. At this point it is
 // possible to run into duplicate (type, state key) tuples.
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index a98fda073..b12025c41 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -166,4 +166,6 @@ type Database interface {
 	GetKnownRooms(ctx context.Context) ([]string, error)
 	// ForgetRoom sets a flag in the membership table, that the user wishes to forget a specific room
 	ForgetRoom(ctx context.Context, userID, roomID string, forget bool) error
+
+	GetHistoryVisibilityState(ctx context.Context, roomInfo *types.RoomInfo, eventID string, domain string) ([]*gomatrixserverlib.Event, error)
 }
diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go
index a24b7f3f0..99c76befe 100644
--- a/roomserver/storage/postgres/state_snapshot_table.go
+++ b/roomserver/storage/postgres/state_snapshot_table.go
@@ -72,9 +72,35 @@ const bulkSelectStateBlockNIDsSQL = "" +
 	"SELECT state_snapshot_nid, state_block_nids FROM roomserver_state_snapshots" +
 	" WHERE state_snapshot_nid = ANY($1) ORDER BY state_snapshot_nid ASC"
 
+// Looks up both the history visibility event and relevant membership events from
+// a given domain name from a given state snapshot. This is used to optimise the
+// helpers.CheckServerAllowedToSeeEvent function.
+// TODO: There's a sequence scan here because of the hash join strategy, which is
+// probably O(n) on state key entries, so there must be a way to avoid that somehow.
+// Event type NIDs are:
+// - 5: m.room.member as per https://github.com/matrix-org/dendrite/blob/c7f7aec4d07d59120d37d5b16a900f6d608a75c4/roomserver/storage/postgres/event_types_table.go#L40
+// - 7: m.room.history_visibility as per https://github.com/matrix-org/dendrite/blob/c7f7aec4d07d59120d37d5b16a900f6d608a75c4/roomserver/storage/postgres/event_types_table.go#L42
+const bulkSelectStateForHistoryVisibilitySQL = `
+	SELECT event_nid FROM (
+	  SELECT event_nid, event_type_nid, event_state_key_nid FROM roomserver_events
+	  WHERE (event_type_nid = 5 OR event_type_nid = 7)
+	  AND event_nid = ANY(
+	    SELECT UNNEST(event_nids) FROM roomserver_state_block
+	    WHERE state_block_nid = ANY(
+	      SELECT UNNEST(state_block_nids) FROM roomserver_state_snapshots
+	      WHERE state_snapshot_nid = $1
+	    )
+	  )
+	) AS roomserver_events
+	INNER JOIN roomserver_event_state_keys
+	  ON roomserver_events.event_state_key_nid = roomserver_event_state_keys.event_state_key_nid
+	  AND (event_type_nid = 7 OR event_state_key LIKE '%:' || $2);
+`
+
 type stateSnapshotStatements struct {
-	insertStateStmt              *sql.Stmt
-	bulkSelectStateBlockNIDsStmt *sql.Stmt
+	insertStateStmt                         *sql.Stmt
+	bulkSelectStateBlockNIDsStmt            *sql.Stmt
+	bulkSelectStateForHistoryVisibilityStmt *sql.Stmt
 }
 
 func CreateStateSnapshotTable(db *sql.DB) error {
@@ -88,6 +114,7 @@ func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) {
 	return s, sqlutil.StatementList{
 		{&s.insertStateStmt, insertStateSQL},
 		{&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL},
+		{&s.bulkSelectStateForHistoryVisibilityStmt, bulkSelectStateForHistoryVisibilitySQL},
 	}.Prepare(db)
 }
 
@@ -136,3 +163,23 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 	}
 	return results, nil
 }
+
+func (s *stateSnapshotStatements) BulkSelectStateForHistoryVisibility(
+	ctx context.Context, txn *sql.Tx, stateSnapshotNID types.StateSnapshotNID, domain string,
+) ([]types.EventNID, error) {
+	stmt := sqlutil.TxStmt(txn, s.bulkSelectStateForHistoryVisibilityStmt)
+	rows, err := stmt.QueryContext(ctx, stateSnapshotNID, domain)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+	results := make([]types.EventNID, 0, 16)
+	for rows.Next() {
+		var eventNID types.EventNID
+		if err = rows.Scan(&eventNID); err != nil {
+			return nil, err
+		}
+		results = append(results, eventNID)
+	}
+	return results, rows.Err()
+}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 2f9932ff8..daac4e821 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -988,6 +988,38 @@ func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event {
 	return &evs[0]
 }
 
+func (d *Database) GetHistoryVisibilityState(ctx context.Context, roomInfo *types.RoomInfo, eventID string, domain string) ([]*gomatrixserverlib.Event, error) {
+	eventStates, err := d.EventsTable.BulkSelectStateAtEventByID(ctx, nil, []string{eventID})
+	if err != nil {
+		return nil, err
+	}
+	stateSnapshotNID := eventStates[0].BeforeStateSnapshotNID
+	if stateSnapshotNID == 0 {
+		return nil, nil
+	}
+	eventNIDs, err := d.StateSnapshotTable.BulkSelectStateForHistoryVisibility(ctx, nil, stateSnapshotNID, domain)
+	if err != nil {
+		return nil, err
+	}
+	eventIDs, _ := d.EventsTable.BulkSelectEventID(ctx, nil, eventNIDs)
+	if err != nil {
+		eventIDs = map[types.EventNID]string{}
+	}
+	events := make([]*gomatrixserverlib.Event, 0, len(eventNIDs))
+	for _, eventNID := range eventNIDs {
+		data, err := d.EventJSONTable.BulkSelectEventJSON(ctx, nil, []types.EventNID{eventNID})
+		if err != nil {
+			return nil, err
+		}
+		ev, err := gomatrixserverlib.NewEventFromTrustedJSONWithEventID(eventIDs[eventNID], data[0].EventJSON, false, roomInfo.RoomVersion)
+		if err != nil {
+			return nil, err
+		}
+		events = append(events, ev)
+	}
+	return events, nil
+}
+
 // GetStateEvent returns the current state event of a given type for a given room with a given state key
 // If no event could be found, returns nil
 // If there was an issue during the retrieval, returns an error
diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go
index b8136b758..73827522c 100644
--- a/roomserver/storage/sqlite3/state_snapshot_table.go
+++ b/roomserver/storage/sqlite3/state_snapshot_table.go
@@ -140,3 +140,9 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs(
 	}
 	return results, nil
 }
+
+func (s *stateSnapshotStatements) BulkSelectStateForHistoryVisibility(
+	ctx context.Context, txn *sql.Tx, stateSnapshotNID types.StateSnapshotNID, domain string,
+) ([]types.EventNID, error) {
+	return nil, tables.OptimisationNotSupportedError
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index d228257b7..58c43ac45 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -3,12 +3,15 @@ package tables
 import (
 	"context"
 	"database/sql"
+	"errors"
 
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/tidwall/gjson"
 )
 
+var OptimisationNotSupportedError = errors.New("optimisation not supported")
+
 type EventJSONPair struct {
 	EventNID  types.EventNID
 	EventJSON []byte
@@ -80,6 +83,10 @@ type Rooms interface {
 type StateSnapshot interface {
 	InsertState(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs types.StateBlockNIDs) (stateNID types.StateSnapshotNID, err error)
 	BulkSelectStateBlockNIDs(ctx context.Context, txn *sql.Tx, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error)
+	// BulkSelectStateForHistoryVisibility is a PostgreSQL-only optimisation for finding
+	// which users are in a room faster than having to load the entire room state. In the
+	// case of SQLite, this will return tables.OptimisationNotSupportedError.
+	BulkSelectStateForHistoryVisibility(ctx context.Context, txn *sql.Tx, stateSnapshotNID types.StateSnapshotNID, domain string) ([]types.EventNID, error)
 }
 
 type StateBlock interface {
diff --git a/roomserver/storage/tables/state_snapshot_table_test.go b/roomserver/storage/tables/state_snapshot_table_test.go
index dcdb5d8f1..b2e59377d 100644
--- a/roomserver/storage/tables/state_snapshot_table_test.go
+++ b/roomserver/storage/tables/state_snapshot_table_test.go
@@ -23,6 +23,15 @@ func mustCreateStateSnapshotTable(t *testing.T, dbType test.DBType) (tab tables.
 	assert.NoError(t, err)
 	switch dbType {
 	case test.DBTypePostgres:
+		// for the PostgreSQL history visibility optimisation to work,
+		// we also need some other tables to exist
+		err = postgres.CreateEventStateKeysTable(db)
+		assert.NoError(t, err)
+		err = postgres.CreateEventsTable(db)
+		assert.NoError(t, err)
+		err = postgres.CreateStateBlockTable(db)
+		assert.NoError(t, err)
+		// ... and then the snapshot table itself
 		err = postgres.CreateStateSnapshotTable(db)
 		assert.NoError(t, err)
 		tab, err = postgres.PrepareStateSnapshotTable(db)

From 119cde37668badb04b9ca9445cf1d87a880bcbd5 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 1 Aug 2022 15:29:19 +0100
Subject: [PATCH 226/391] De-race `types.RoomInfo` (#2600)

---
 roomserver/internal/helpers/auth.go           |  4 +--
 roomserver/internal/helpers/helpers.go        |  2 +-
 roomserver/internal/perform/perform_admin.go  |  2 +-
 .../internal/perform/perform_backfill.go      |  6 ++--
 .../internal/perform/perform_inbound_peek.go  |  2 +-
 roomserver/internal/perform/perform_invite.go |  4 +--
 roomserver/internal/query/query.go            | 12 ++++----
 roomserver/storage/postgres/rooms_table.go    |  6 ++--
 roomserver/storage/shared/room_updater.go     |  4 +--
 roomserver/storage/shared/storage.go          | 12 ++++----
 roomserver/storage/sqlite3/rooms_table.go     |  6 ++--
 roomserver/storage/tables/rooms_table_test.go | 24 +++++++--------
 roomserver/types/types.go                     | 30 +++++++++++++++++--
 13 files changed, 72 insertions(+), 42 deletions(-)

diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go
index 0229f822f..648c50cf6 100644
--- a/roomserver/internal/helpers/auth.go
+++ b/roomserver/internal/helpers/auth.go
@@ -50,14 +50,14 @@ func CheckForSoftFail(
 		if err != nil {
 			return false, fmt.Errorf("db.RoomNID: %w", err)
 		}
-		if roomInfo == nil || roomInfo.IsStub {
+		if roomInfo == nil || roomInfo.IsStub() {
 			return false, nil
 		}
 
 		// Then get the state entries for the current state snapshot.
 		// We'll use this to check if the event is allowed right now.
 		roomState := state.NewStateResolution(db, roomInfo)
-		authStateEntries, err = roomState.LoadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID)
+		authStateEntries, err = roomState.LoadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID())
 		if err != nil {
 			return true, fmt.Errorf("roomState.LoadStateAtSnapshot: %w", err)
 		}
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index 16a6f615d..d61aa08cb 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -409,7 +409,7 @@ func QueryLatestEventsAndState(
 	if err != nil {
 		return err
 	}
-	if roomInfo == nil || roomInfo.IsStub {
+	if roomInfo == nil || roomInfo.IsStub() {
 		response.RoomExists = false
 		return nil
 	}
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index 1cb52966a..6c7c6c98b 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -52,7 +52,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 		}
 		return
 	}
-	if roomInfo == nil || roomInfo.IsStub {
+	if roomInfo == nil || roomInfo.IsStub() {
 		res.Error = &api.PerformError{
 			Code: api.PerformErrorNoRoom,
 			Msg:  fmt.Sprintf("Room %s not found", req.RoomID),
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 3f98fbc24..5b7ed22ee 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -73,7 +73,7 @@ func (r *Backfiller) PerformBackfill(
 	if err != nil {
 		return err
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		return fmt.Errorf("PerformBackfill: missing room info for room %s", request.RoomID)
 	}
 
@@ -106,7 +106,7 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform
 	if err != nil {
 		return err
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		return fmt.Errorf("backfillViaFederation: missing room info for room %s", req.RoomID)
 	}
 	requester := newBackfillRequester(r.DB, r.FSAPI, r.ServerName, req.BackwardsExtremities, r.PreferServers)
@@ -434,7 +434,7 @@ FindSuccessor:
 		logrus.WithError(err).WithField("room_id", roomID).Error("ServersAtEvent: failed to get RoomInfo for room")
 		return nil
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		logrus.WithField("room_id", roomID).Error("ServersAtEvent: failed to get RoomInfo for room, room is missing")
 		return nil
 	}
diff --git a/roomserver/internal/perform/perform_inbound_peek.go b/roomserver/internal/perform/perform_inbound_peek.go
index 32c81e849..29decd363 100644
--- a/roomserver/internal/perform/perform_inbound_peek.go
+++ b/roomserver/internal/perform/perform_inbound_peek.go
@@ -50,7 +50,7 @@ func (r *InboundPeeker) PerformInboundPeek(
 	if err != nil {
 		return err
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		return nil
 	}
 	response.RoomExists = true
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index fce529305..e1ff4eabb 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -125,7 +125,7 @@ func (r *Inviter) PerformInvite(
 		return outputUpdates, nil
 	}
 
-	if (info == nil || info.IsStub) && !isOriginLocal && isTargetLocal {
+	if (info == nil || info.IsStub()) && !isOriginLocal && isTargetLocal {
 		// The invite came in over federation for a room that we don't know about
 		// yet. We need to handle this a bit differently to most invites because
 		// we don't know the room state, therefore the roomserver can't process
@@ -276,7 +276,7 @@ func buildInviteStrippedState(
 	}
 	roomState := state.NewStateResolution(db, info)
 	stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples(
-		ctx, info.StateSnapshotNID, stateWanted,
+		ctx, info.StateSnapshotNID(), stateWanted,
 	)
 	if err != nil {
 		return nil, err
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index f652a8ea3..6d7c6e6bb 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -61,7 +61,7 @@ func (r *Queryer) QueryStateAfterEvents(
 	if err != nil {
 		return err
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		return nil
 	}
 
@@ -302,7 +302,7 @@ func (r *Queryer) QueryServerJoinedToRoom(
 	if err != nil {
 		return fmt.Errorf("r.DB.RoomInfo: %w", err)
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		return nil
 	}
 	response.RoomExists = true
@@ -390,7 +390,7 @@ func (r *Queryer) QueryMissingEvents(
 	if err != nil {
 		return err
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		return fmt.Errorf("missing RoomInfo for room %s", events[0].RoomID())
 	}
 
@@ -429,7 +429,7 @@ func (r *Queryer) QueryStateAndAuthChain(
 	if err != nil {
 		return err
 	}
-	if info == nil || info.IsStub {
+	if info == nil || info.IsStub() {
 		return nil
 	}
 	response.RoomExists = true
@@ -774,7 +774,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 	if err != nil {
 		return fmt.Errorf("r.DB.RoomInfo: %w", err)
 	}
-	if roomInfo == nil || roomInfo.IsStub {
+	if roomInfo == nil || roomInfo.IsStub() {
 		return nil // fmt.Errorf("room %q doesn't exist or is stub room", req.RoomID)
 	}
 	// If the room version doesn't allow restricted joins then don't
@@ -837,7 +837,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 		// See if the room exists. If it doesn't exist or if it's a stub
 		// room entry then we can't check memberships.
 		targetRoomInfo, err := r.DB.RoomInfo(ctx, rule.RoomID)
-		if err != nil || targetRoomInfo == nil || targetRoomInfo.IsStub {
+		if err != nil || targetRoomInfo == nil || targetRoomInfo.IsStub() {
 			res.Resident = false
 			continue
 		}
diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go
index 24362af74..994399532 100644
--- a/roomserver/storage/postgres/rooms_table.go
+++ b/roomserver/storage/postgres/rooms_table.go
@@ -147,14 +147,16 @@ func (s *roomStatements) InsertRoomNID(
 func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
 	var info types.RoomInfo
 	var latestNIDs pq.Int64Array
+	var stateSnapshotNID types.StateSnapshotNID
 	stmt := sqlutil.TxStmt(txn, s.selectRoomInfoStmt)
 	err := stmt.QueryRowContext(ctx, roomID).Scan(
-		&info.RoomVersion, &info.RoomNID, &info.StateSnapshotNID, &latestNIDs,
+		&info.RoomVersion, &info.RoomNID, &stateSnapshotNID, &latestNIDs,
 	)
 	if err == sql.ErrNoRows {
 		return nil, nil
 	}
-	info.IsStub = len(latestNIDs) == 0
+	info.SetStateSnapshotNID(stateSnapshotNID)
+	info.SetIsStub(len(latestNIDs) == 0)
 	return &info, err
 }
 
diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go
index c35ac653c..a7ae26d43 100644
--- a/roomserver/storage/shared/room_updater.go
+++ b/roomserver/storage/shared/room_updater.go
@@ -229,8 +229,8 @@ func (u *RoomUpdater) SetLatestEvents(
 		// Since it's entirely possible that this types.RoomInfo came from the
 		// cache, we should make sure to update that entry so that the next run
 		// works from live data.
-		u.roomInfo.StateSnapshotNID = currentStateSnapshotNID
-		u.roomInfo.IsStub = false
+		u.roomInfo.SetStateSnapshotNID(currentStateSnapshotNID)
+		u.roomInfo.SetIsStub(false)
 		return nil
 	})
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index daac4e821..05b897149 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1032,7 +1032,7 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s
 		return nil, fmt.Errorf("room %s doesn't exist", roomID)
 	}
 	// e.g invited rooms
-	if roomInfo.IsStub {
+	if roomInfo.IsStub() {
 		return nil, nil
 	}
 	eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, nil, evType)
@@ -1051,7 +1051,7 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s
 	if err != nil {
 		return nil, err
 	}
-	entries, err := d.loadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID)
+	entries, err := d.loadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID())
 	if err != nil {
 		return nil, err
 	}
@@ -1097,7 +1097,7 @@ func (d *Database) GetStateEventsWithEventType(ctx context.Context, roomID, evTy
 		return nil, fmt.Errorf("room %s doesn't exist", roomID)
 	}
 	// e.g invited rooms
-	if roomInfo.IsStub {
+	if roomInfo.IsStub() {
 		return nil, nil
 	}
 	eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, nil, evType)
@@ -1108,7 +1108,7 @@ func (d *Database) GetStateEventsWithEventType(ctx context.Context, roomID, evTy
 	if err != nil {
 		return nil, err
 	}
-	entries, err := d.loadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID)
+	entries, err := d.loadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID())
 	if err != nil {
 		return nil, err
 	}
@@ -1225,10 +1225,10 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu
 			return nil, fmt.Errorf("GetBulkStateContent: failed to load room info for room %s : %w", roomID, err2)
 		}
 		// for unknown rooms or rooms which we don't have the current state, skip them.
-		if roomInfo == nil || roomInfo.IsStub {
+		if roomInfo == nil || roomInfo.IsStub() {
 			continue
 		}
-		entries, err2 := d.loadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID)
+		entries, err2 := d.loadStateAtSnapshot(ctx, roomInfo.StateSnapshotNID())
 		if err2 != nil {
 			return nil, fmt.Errorf("GetBulkStateContent: failed to load state for room %s : %w", roomID, err2)
 		}
diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go
index 03ad4b3d0..25b611b3e 100644
--- a/roomserver/storage/sqlite3/rooms_table.go
+++ b/roomserver/storage/sqlite3/rooms_table.go
@@ -129,9 +129,10 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T
 func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
 	var info types.RoomInfo
 	var latestNIDsJSON string
+	var stateSnapshotNID types.StateSnapshotNID
 	stmt := sqlutil.TxStmt(txn, s.selectRoomInfoStmt)
 	err := stmt.QueryRowContext(ctx, roomID).Scan(
-		&info.RoomVersion, &info.RoomNID, &info.StateSnapshotNID, &latestNIDsJSON,
+		&info.RoomVersion, &info.RoomNID, &stateSnapshotNID, &latestNIDsJSON,
 	)
 	if err != nil {
 		if err == sql.ErrNoRows {
@@ -143,7 +144,8 @@ func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID
 	if err = json.Unmarshal([]byte(latestNIDsJSON), &latestNIDs); err != nil {
 		return nil, err
 	}
-	info.IsStub = len(latestNIDs) == 0
+	info.SetStateSnapshotNID(stateSnapshotNID)
+	info.SetIsStub(len(latestNIDs) == 0)
 	return &info, err
 }
 
diff --git a/roomserver/storage/tables/rooms_table_test.go b/roomserver/storage/tables/rooms_table_test.go
index 0a02369a1..eddd012c8 100644
--- a/roomserver/storage/tables/rooms_table_test.go
+++ b/roomserver/storage/tables/rooms_table_test.go
@@ -63,12 +63,12 @@ func TestRoomsTable(t *testing.T) {
 
 		roomInfo, err := tab.SelectRoomInfo(ctx, nil, room.ID)
 		assert.NoError(t, err)
-		assert.Equal(t, &types.RoomInfo{
-			RoomNID:          wantRoomNID,
-			RoomVersion:      room.Version,
-			StateSnapshotNID: 0,
-			IsStub:           true, // there are no latestEventNIDs
-		}, roomInfo)
+		expected := &types.RoomInfo{
+			RoomNID:     wantRoomNID,
+			RoomVersion: room.Version,
+		}
+		expected.SetIsStub(true) // there are no latestEventNIDs
+		assert.Equal(t, expected, roomInfo)
 
 		roomInfo, err = tab.SelectRoomInfo(ctx, nil, "!doesnotexist:localhost")
 		assert.NoError(t, err)
@@ -103,12 +103,12 @@ func TestRoomsTable(t *testing.T) {
 
 		roomInfo, err = tab.SelectRoomInfo(ctx, nil, room.ID)
 		assert.NoError(t, err)
-		assert.Equal(t, &types.RoomInfo{
-			RoomNID:          wantRoomNID,
-			RoomVersion:      room.Version,
-			StateSnapshotNID: 1,
-			IsStub:           false,
-		}, roomInfo)
+		expected = &types.RoomInfo{
+			RoomNID:     wantRoomNID,
+			RoomVersion: room.Version,
+		}
+		expected.SetStateSnapshotNID(1)
+		assert.Equal(t, expected, roomInfo)
 
 		eventNIDs, snapshotNID, err := tab.SelectLatestEventNIDs(ctx, nil, wantRoomNID)
 		assert.NoError(t, err)
diff --git a/roomserver/types/types.go b/roomserver/types/types.go
index bc01ca33c..726659ea0 100644
--- a/roomserver/types/types.go
+++ b/roomserver/types/types.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -279,8 +280,33 @@ func (e RejectedError) Error() string { return string(e) }
 
 // RoomInfo contains metadata about a room
 type RoomInfo struct {
+	mu               sync.RWMutex
 	RoomNID          RoomNID
 	RoomVersion      gomatrixserverlib.RoomVersion
-	StateSnapshotNID StateSnapshotNID
-	IsStub           bool
+	stateSnapshotNID StateSnapshotNID
+	isStub           bool
+}
+
+func (r *RoomInfo) StateSnapshotNID() StateSnapshotNID {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.stateSnapshotNID
+}
+
+func (r *RoomInfo) IsStub() bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.isStub
+}
+
+func (r *RoomInfo) SetStateSnapshotNID(nid StateSnapshotNID) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.stateSnapshotNID = nid
+}
+
+func (r *RoomInfo) SetIsStub(isStub bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.isStub = isStub
 }

From e94ef84aaba30e12baf7f524c4e7a36d2fdeb189 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 1 Aug 2022 15:55:56 +0100
Subject: [PATCH 227/391] De-race `CompleteSync` (#2601)

The `err` was coming from outside of the goroutine and being written to by concurrent goroutines.
---
 syncapi/streams/stream_pdu.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 00b3dfe3b..1832adbe5 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -109,12 +109,11 @@ func (p *PDUStreamProvider) CompleteSync(
 		p.queue(func() {
 			defer reqWaitGroup.Done()
 
-			var jr *types.JoinResponse
-			jr, err = p.getJoinResponseForCompleteSync(
+			jr, jerr := p.getJoinResponseForCompleteSync(
 				ctx, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device,
 			)
-			if err != nil {
-				req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed")
+			if jerr != nil {
+				req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
 				return
 			}
 

From 3d51624fefa7f78833652e21760a75527dafce9b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 1 Aug 2022 17:34:29 +0100
Subject: [PATCH 228/391] Version 0.9.0 (#2602)

---
 CHANGES.md          | 30 ++++++++++++++++++++++++++++++
 internal/version.go |  4 ++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 3df03b2f6..0ae927446 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,35 @@
 # Changelog
 
+## Dendrite 0.9.0 (2022-08-01)
+
+### Features
+
+* Dendrite now uses Ristretto for managing in-memory caches
+  * Should improve cache utilisation considerably over time by more intelligently selecting and managing cache entries compared to the previous LRU-based cache
+  * Defaults to a 1GB cache size if not configured otherwise
+  * The estimated cache size in memory and maximum age can now be configured with new [configuration options](https://github.com/matrix-org/dendrite/blob/e94ef84aaba30e12baf7f524c4e7a36d2fdeb189/dendrite-sample.monolith.yaml#L44-L61) to prevent unbounded cache growth
+* Added support for serving the `/.well-known/matrix/client` hint directly from Dendrite
+  * Configurable with the new [configuration option](https://github.com/matrix-org/dendrite/blob/e94ef84aaba30e12baf7f524c4e7a36d2fdeb189/dendrite-sample.monolith.yaml#L67-L69)
+* Refactored membership updater, which should eliminate some bugs caused by the membership table getting out of sync with the room state
+* The User API is now responsible for sending account data updates to other components, which may fix some races and duplicate account data events
+* Optimised database query for checking whether a remote server is allowed to request an event over federation without using anywhere near as much CPU time (PostgreSQL only)
+* Database migrations have been refactored to eliminate some problems that were present with `goose` and upgrading from older Dendrite versions
+* Media fetching will now use the `/v3` endpoints for downloading media from remote homeservers
+* HTTP 404 and HTTP 405 errors from the client-facing APIs should now be returned with CORS headers so that web-based clients do not produce incorrect access control warnings for unknown endpoints
+* Some preparation work for full history visibility support
+
+### Fixes
+
+* Fixes a crash that could occur during event redaction
+* The `/members` endpoint will no longer incorrectly return HTTP 500 as a result of some invite events
+* Send-to-device messages should now be ordered more reliably and the last position in the stream updated correctly
+* Parsing of appservice configuration files is now less strict (contributed by [Kab1r](https://github.com/Kab1r))
+* The sync API should now identify shared users correctly when waking up for E2EE key changes
+* The federation `/state` endpoint will now return a HTTP 403 when the state before an event isn't known instead of a HTTP 500
+* Presence timestamps should now be calculated with the correct precision
+* A race condition in the roomserver's room info has been fixed
+* A race condition in the sync API has been fixed
+
 ## Dendrite 0.8.9 (2022-07-01)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index 9568f08cb..6d29a68ee 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 8
-	VersionPatch = 9
+	VersionMinor = 9
+	VersionPatch = 0
 	VersionTag   = "" // example: "rc1"
 )
 

From 6b6b420b9f467d053fd1fe3d1872f94c0bcf8300 Mon Sep 17 00:00:00 2001
From: sergekh2 <104387024+sergekh2@users.noreply.github.com>
Date: Tue, 2 Aug 2022 01:43:48 -0700
Subject: [PATCH 229/391] Fix issue with sync API not advancing. (#2603)

Issue: During conversation, under some conditions, sync cookie is not advanced, and, as a result, client loops on the same sync API call creating high traffic and CPU load.
Fix: pdu component of cookie was updated incorrectly.
---
 syncapi/streams/stream_pdu.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 1832adbe5..1003208fd 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -261,9 +261,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		var pos types.StreamPosition
 		if _, pos, err = p.DB.PositionInTopology(ctx, mostRecentEventID); err == nil {
 			switch {
-			case r.Backwards && pos > latestPosition:
+			case r.Backwards && pos < latestPosition:
 				fallthrough
-			case !r.Backwards && pos < latestPosition:
+			case !r.Backwards && pos > latestPosition:
 				latestPosition = pos
 			}
 		}

From eab87ef07dd9461e38b716b7c4b4c95cedee0ca8 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 2 Aug 2022 10:22:17 +0100
Subject: [PATCH 230/391] Stronger checks for `/send_join` (#2604)

---
 federationapi/routing/join.go | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 41004cf51..30406a155 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -202,6 +202,14 @@ func SendJoin(
 		}
 	}
 
+	// Check that the event is from the server sending the request.
+	if event.Origin() != request.Origin() {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The join must be sent by the server it originated on"),
+		}
+	}
+
 	// Check that a state key is provided.
 	if event.StateKey() == nil || event.StateKeyEquals("") {
 		return util.JSONResponse{
@@ -216,6 +224,22 @@ func SendJoin(
 		}
 	}
 
+	// Check that the sender belongs to the server that is sending us
+	// the request. By this point we've already asserted that the sender
+	// and the state key are equal so we don't need to check both.
+	var domain gomatrixserverlib.ServerName
+	if _, domain, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender of the join is invalid"),
+		}
+	} else if domain != request.Origin() {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender of the join must belong to the origin server"),
+		}
+	}
+
 	// Check that the room ID is correct.
 	if event.RoomID() != roomID {
 		return util.JSONResponse{
@@ -242,14 +266,6 @@ func SendJoin(
 		}
 	}
 
-	// Check that the event is from the server sending the request.
-	if event.Origin() != request.Origin() {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The join must be sent by the server it originated on"),
-		}
-	}
-
 	// Check that this is in fact a join event
 	membership, err := event.Membership()
 	if err != nil {

From ca3fa58388aeb8ac84a05fe5e0fab6bfa85f3fc6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 2 Aug 2022 12:27:15 +0100
Subject: [PATCH 231/391] Various roominfo tweaks (#2607)

---
 roomserver/internal/input/input_missing.go |  6 +----
 roomserver/internal/query/query.go         |  4 +--
 roomserver/storage/shared/room_updater.go  | 14 +++++++++--
 roomserver/storage/shared/storage.go       | 29 ++++++++++++++++------
 roomserver/types/types.go                  | 13 ++++++++++
 5 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index edc153b7f..1fe25e38a 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -375,11 +375,7 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room
 	defer span.Finish()
 
 	var res parsedRespState
-	roomInfo, err := t.db.RoomInfo(ctx, roomID)
-	if err != nil {
-		return nil
-	}
-	roomState := state.NewStateResolution(t.db, roomInfo)
+	roomState := state.NewStateResolution(t.db, t.roomInfo)
 	stateAtEvents, err := t.db.StateAtEventIDs(ctx, []string{eventID})
 	if err != nil {
 		util.GetLogger(ctx).WithField("room_id", roomID).WithError(err).Warnf("failed to get state after %s locally", eventID)
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 6d7c6e6bb..5ba59b8f5 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -351,8 +351,8 @@ func (r *Queryer) QueryServerAllowedToSeeEvent(
 	if err != nil {
 		return err
 	}
-	if info == nil {
-		return fmt.Errorf("QueryServerAllowedToSeeEvent: no room info for room %s", roomID)
+	if info == nil || info.IsStub() {
+		return nil
 	}
 	response.AllowedToSeeEvent, err = helpers.CheckServerAllowedToSeeEvent(
 		ctx, r.DB, info, request.EventID, request.ServerName, inRoomRes.IsInRoom,
diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go
index a7ae26d43..42c0c8f2d 100644
--- a/roomserver/storage/shared/room_updater.go
+++ b/roomserver/storage/shared/room_updater.go
@@ -217,6 +217,14 @@ func (u *RoomUpdater) SetLatestEvents(
 	roomNID types.RoomNID, latest []types.StateAtEventAndReference, lastEventNIDSent types.EventNID,
 	currentStateSnapshotNID types.StateSnapshotNID,
 ) error {
+	switch {
+	case len(latest) == 0:
+		return fmt.Errorf("cannot set latest events with no latest event references")
+	case currentStateSnapshotNID == 0:
+		return fmt.Errorf("cannot set latest events with invalid state snapshot NID")
+	case lastEventNIDSent == 0:
+		return fmt.Errorf("cannot set latest events with invalid latest event NID")
+	}
 	eventNIDs := make([]types.EventNID, len(latest))
 	for i := range latest {
 		eventNIDs[i] = latest[i].EventNID
@@ -229,8 +237,10 @@ func (u *RoomUpdater) SetLatestEvents(
 		// Since it's entirely possible that this types.RoomInfo came from the
 		// cache, we should make sure to update that entry so that the next run
 		// works from live data.
-		u.roomInfo.SetStateSnapshotNID(currentStateSnapshotNID)
-		u.roomInfo.SetIsStub(false)
+		if u.roomInfo != nil {
+			u.roomInfo.SetStateSnapshotNID(currentStateSnapshotNID)
+			u.roomInfo.SetIsStub(false)
+		}
 		return nil
 	})
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 05b897149..9e6a4142c 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -156,15 +156,30 @@ func (d *Database) RoomInfo(ctx context.Context, roomID string) (*types.RoomInfo
 }
 
 func (d *Database) roomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
-	if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok && roomInfo != nil {
+	roomInfo, ok := d.Cache.GetRoomInfo(roomID)
+	if ok && roomInfo != nil && !roomInfo.IsStub() {
+		// The data that's in the cache is not stubby, so return it.
 		return roomInfo, nil
 	}
-	roomInfo, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID)
-	if err == nil && roomInfo != nil {
-		d.Cache.StoreRoomServerRoomID(roomInfo.RoomNID, roomID)
-		d.Cache.StoreRoomInfo(roomID, roomInfo)
+	// At this point we either don't have an entry in the cache, or
+	// it is stubby, so let's check the roomserver_rooms table again.
+	roomInfoFromDB, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID)
+	if err != nil {
+		return nil, err
 	}
-	return roomInfo, err
+	// If we have a stubby cache entry already, update it and return
+	// the reference to the cache entry.
+	if roomInfo != nil {
+		roomInfo.CopyFrom(roomInfoFromDB)
+		return roomInfo, nil
+	}
+	// Otherwise, try to admit the data into the cache and return the
+	// new reference from the database.
+	if roomInfoFromDB != nil {
+		d.Cache.StoreRoomServerRoomID(roomInfoFromDB.RoomNID, roomID)
+		d.Cache.StoreRoomInfo(roomID, roomInfoFromDB)
+	}
+	return roomInfoFromDB, err
 }
 
 func (d *Database) AddState(
@@ -676,7 +691,7 @@ func (d *Database) storeEvent(
 		succeeded := false
 		if updater == nil {
 			var roomInfo *types.RoomInfo
-			roomInfo, err = d.RoomInfo(ctx, event.RoomID())
+			roomInfo, err = d.roomInfo(ctx, txn, event.RoomID())
 			if err != nil {
 				return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.RoomInfo: %w", err)
 			}
diff --git a/roomserver/types/types.go b/roomserver/types/types.go
index 726659ea0..f40980994 100644
--- a/roomserver/types/types.go
+++ b/roomserver/types/types.go
@@ -310,3 +310,16 @@ func (r *RoomInfo) SetIsStub(isStub bool) {
 	defer r.mu.Unlock()
 	r.isStub = isStub
 }
+
+func (r *RoomInfo) CopyFrom(r2 *RoomInfo) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r2.mu.RLock()
+	defer r2.mu.RUnlock()
+
+	r.RoomNID = r2.RoomNID
+	r.RoomVersion = r2.RoomVersion
+	r.stateSnapshotNID = r2.stateSnapshotNID
+	r.isStub = r2.isStub
+}

From 7ec70272d28c6b37f8a412dc577ff27da398d1df Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 2 Aug 2022 13:58:08 +0200
Subject: [PATCH 232/391] Disable NATS Server logging, allow self-signed
 certificates (#2605)

* Disable NATS Server logs in CI

* Add option to disable TLS validation for NATS
---
 dendrite-sample.monolith.yaml    |  5 +++++
 dendrite-sample.polylith.yaml    |  5 +++++
 setup/config/config_jetstream.go |  6 ++++++
 setup/jetstream/nats.go          | 10 +++++++++-
 4 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index cc6c173e8..a34b8af55 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -113,6 +113,11 @@ global:
     addresses:
     # - localhost:4222
 
+    # Disable the validation of TLS certificates of NATS. This is
+    # not recommended in production since it may allow NATS traffic
+    # to be sent to an insecure endpoint.
+    disable_tls_validation: false
+
     # Persistent directory to store JetStream streams in. This directory should be
     # preserved across Dendrite restarts.
     storage_path: ./
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 92cab19b1..550611229 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -103,6 +103,11 @@ global:
     addresses:
       - hostname:4222
 
+    # Disable the validation of TLS certificates of NATS. This is
+    # not recommended in production since it may allow NATS traffic
+    # to be sent to an insecure endpoint.
+    disable_tls_validation: false
+
     # The prefix to use for stream names for this homeserver - really only useful
     # if you are running more than one Dendrite server on the same NATS deployment.
     topic_prefix: Dendrite
diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go
index e4cfd4d3b..a7827597e 100644
--- a/setup/config/config_jetstream.go
+++ b/setup/config/config_jetstream.go
@@ -17,6 +17,10 @@ type JetStream struct {
 	TopicPrefix string `yaml:"topic_prefix"`
 	// Keep all storage in memory. This is mostly useful for unit tests.
 	InMemory bool `yaml:"in_memory"`
+	// Disable logging. This is mostly useful for unit tests.
+	NoLog bool `yaml:"-"`
+	// Disables TLS validation. This should NOT be used in production
+	DisableTLSValidation bool `yaml:"disable_tls_validation"`
 }
 
 func (c *JetStream) Prefixed(name string) string {
@@ -32,6 +36,8 @@ func (c *JetStream) Defaults(generate bool) {
 	c.TopicPrefix = "Dendrite"
 	if generate {
 		c.StoragePath = Path("./")
+		c.NoLog = true
+		c.DisableTLSValidation = true
 	}
 }
 
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 248b0e656..be216a02a 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -1,6 +1,7 @@
 package jetstream
 
 import (
+	"crypto/tls"
 	"fmt"
 	"reflect"
 	"strings"
@@ -45,6 +46,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
 			NoSystemAccount: true,
 			MaxPayload:      16 * 1024 * 1024,
 			NoSigs:          true,
+			NoLog:           cfg.NoLog,
 		})
 		if err != nil {
 			panic(err)
@@ -75,7 +77,13 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
 func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
 	if nc == nil {
 		var err error
-		nc, err = natsclient.Connect(strings.Join(cfg.Addresses, ","))
+		opts := []nats.Option{}
+		if cfg.DisableTLSValidation {
+			opts = append(opts, nats.Secure(&tls.Config{
+				InsecureSkipVerify: true,
+			}))
+		}
+		nc, err = natsclient.Connect(strings.Join(cfg.Addresses, ","), opts...)
 		if err != nil {
 			logrus.WithError(err).Panic("Unable to connect to NATS")
 			return nil, nil

From f4345dafdecacb655ab3985871f28100ececb302 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 2 Aug 2022 13:01:03 +0100
Subject: [PATCH 233/391] Fix data race in `lookupMissingStateViaStateIDs`

---
 roomserver/internal/input/input_missing.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index 1fe25e38a..c78e5d79a 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -750,9 +750,8 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
 
 		// Define what we'll do in order to fetch the missing event ID.
 		fetch := func(missingEventID string) {
-			var h *gomatrixserverlib.Event
-			h, err = t.lookupEvent(ctx, roomVersion, roomID, missingEventID, false)
-			switch err.(type) {
+			h, herr := t.lookupEvent(ctx, roomVersion, roomID, missingEventID, false)
+			switch herr.(type) {
 			case verifySigError:
 				return
 			case nil:
@@ -761,7 +760,7 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
 				util.GetLogger(ctx).WithFields(logrus.Fields{
 					"event_id": missingEventID,
 					"room_id":  roomID,
-				}).Warn("Failed to fetch missing event")
+				}).WithError(herr).Warn("Failed to fetch missing event")
 				return
 			}
 			haveEventsMutex.Lock()

From e384eb683f158950956c8e003b33491ad0905b83 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 2 Aug 2022 15:33:28 +0100
Subject: [PATCH 234/391] Disable flakey test

---
 sytest-blacklist | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sytest-blacklist b/sytest-blacklist
index bcc345f6e..e0b2767b1 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -49,3 +49,7 @@ Notifications can be viewed with GET /notifications
 
 If remote user leaves room we no longer receive device updates
 Guest users can join guest_access rooms
+
+# You'll be shocked to discover this is flakey too
+
+Inbound /v1/send_join rejects joins from other servers

From df5d4dc7a36f7fe5ec17f9da81c535d5c01bd505 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 2 Aug 2022 17:00:16 +0200
Subject: [PATCH 235/391] Delete correct Send-to-Device messages (#2608)

* Add send-to-device tests

* Update tests, fix message deletion

* PR comments
---
 .../storage/postgres/send_to_device_table.go  |   2 +-
 .../storage/sqlite3/send_to_device_table.go   |   2 +-
 syncapi/storage/storage_test.go               |  23 +--
 syncapi/syncapi_test.go                       | 136 ++++++++++++++++++
 test/testrig/base.go                          |   3 +-
 5 files changed, 145 insertions(+), 21 deletions(-)

diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go
index 2734fef3e..fd0c1c56b 100644
--- a/syncapi/storage/postgres/send_to_device_table.go
+++ b/syncapi/storage/postgres/send_to_device_table.go
@@ -58,7 +58,7 @@ const selectSendToDeviceMessagesSQL = `
 
 const deleteSendToDeviceMessagesSQL = `
 	DELETE FROM syncapi_send_to_device
-	  WHERE user_id = $1 AND device_id = $2 AND id < $3
+	  WHERE user_id = $1 AND device_id = $2 AND id <= $3
 `
 
 const selectMaxSendToDeviceIDSQL = "" +
diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go
index d05d3fe72..e3aa1b7a1 100644
--- a/syncapi/storage/sqlite3/send_to_device_table.go
+++ b/syncapi/storage/sqlite3/send_to_device_table.go
@@ -55,7 +55,7 @@ const selectSendToDeviceMessagesSQL = `
 
 const deleteSendToDeviceMessagesSQL = `
 	DELETE FROM syncapi_send_to_device
-	  WHERE user_id = $1 AND device_id = $2 AND id < $3
+	  WHERE user_id = $1 AND device_id = $2 AND id <= $3
 `
 
 const selectMaxSendToDeviceIDSQL = "" +
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index df03a33c2..eda5ef3e6 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -416,11 +416,6 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 			t.Fatal("first call should have no updates")
 		}
 
-		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, 100)
-		if err != nil {
-			return
-		}
-
 		// Try sending a message.
 		streamPos, err := db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{
 			Sender:  bob.ID,
@@ -441,43 +436,35 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 		if count := len(events); count != 1 {
 			t.Fatalf("second call should have one update, got %d", count)
 		}
-		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos)
-		if err != nil {
-			return
-		}
 
 		// At this point we should still have one message because we haven't progressed the
 		// sync position yet. This is equivalent to the client failing to /sync and retrying
 		// with the same position.
-		streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100)
+		streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if len(events) != 1 {
 			t.Fatal("third call should have one update still")
 		}
-		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos+1)
+		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos)
 		if err != nil {
 			return
 		}
 
 		// At this point we should now have no updates, because we've progressed the sync
 		// position. Therefore the update from before will not be sent again.
-		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos+1, streamPos+2)
+		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if len(events) != 0 {
 			t.Fatal("fourth call should have no updates")
 		}
-		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos+1)
-		if err != nil {
-			return
-		}
 
 		// At this point we should still have no updates, because no new updates have been
 		// sent.
-		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+2)
+		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -491,7 +478,7 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 			streamPos, err = db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{
 				Sender:  bob.ID,
 				Type:    "m.type",
-				Content: json.RawMessage(fmt.Sprintf(`{ "count": %d }`, i)),
+				Content: json.RawMessage(fmt.Sprintf(`{"count":%d}`, i)),
 			})
 			if err != nil {
 				t.Fatal(err)
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 3ce7c64b7..b10864ff5 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -3,11 +3,14 @@ package syncapi
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi/producers"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
@@ -311,6 +314,139 @@ func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) {
 
 }
 
+func TestSendToDevice(t *testing.T) {
+	test.WithAllDatabases(t, testSendToDevice)
+}
+
+func testSendToDevice(t *testing.T, dbType test.DBType) {
+	user := test.NewUser(t)
+	alice := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      user.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "Alice",
+		AccountType: userapi.AccountTypeUser,
+	}
+
+	base, close := testrig.CreateBaseDendrite(t, dbType)
+	defer close()
+
+	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+
+	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, &syncKeyAPI{})
+
+	producer := producers.SyncAPIProducer{
+		TopicSendToDeviceEvent: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		JetStream:              jsctx,
+	}
+
+	msgCounter := 0
+
+	testCases := []struct {
+		name              string
+		since             string
+		want              []string
+		sendMessagesCount int
+	}{
+		{
+			name: "initial sync, no messages",
+			want: []string{},
+		},
+		{
+			name:              "initial sync, one new message",
+			sendMessagesCount: 1,
+			want: []string{
+				"message 1",
+			},
+		},
+		{
+			name:              "initial sync, two new messages", // we didn't advance the since token, so we'll receive two messages
+			sendMessagesCount: 1,
+			want: []string{
+				"message 1",
+				"message 2",
+			},
+		},
+		{
+			name:  "incremental sync, one message", // this deletes message 1, as we advanced the since token
+			since: types.StreamingToken{SendToDevicePosition: 1}.String(),
+			want: []string{
+				"message 2",
+			},
+		},
+		{
+			name:  "failed incremental sync, one message", // didn't advance since, so still the same message
+			since: types.StreamingToken{SendToDevicePosition: 1}.String(),
+			want: []string{
+				"message 2",
+			},
+		},
+		{
+			name:  "incremental sync, no message",                         // this should delete message 2
+			since: types.StreamingToken{SendToDevicePosition: 2}.String(), // next_batch from previous sync
+			want:  []string{},
+		},
+		{
+			name:              "incremental sync, three new messages",
+			since:             types.StreamingToken{SendToDevicePosition: 2}.String(),
+			sendMessagesCount: 3,
+			want: []string{
+				"message 3", // message 2 was deleted in the previous test
+				"message 4",
+				"message 5",
+			},
+		},
+		{
+			name: "initial sync, three messages", // we expect three messages, as we didn't go beyond "2"
+			want: []string{
+				"message 3",
+				"message 4",
+				"message 5",
+			},
+		},
+		{
+			name:  "incremental sync, no messages", // advance the sync token, no new messages
+			since: types.StreamingToken{SendToDevicePosition: 5}.String(),
+			want:  []string{},
+		},
+	}
+
+	ctx := context.Background()
+	for _, tc := range testCases {
+		// Send to-device messages of type "m.dendrite.test" with content `{"dummy":"message $counter"}`
+		for i := 0; i < tc.sendMessagesCount; i++ {
+			msgCounter++
+			msg := map[string]string{
+				"dummy": fmt.Sprintf("message %d", msgCounter),
+			}
+			if err := producer.SendToDevice(ctx, user.ID, user.ID, alice.ID, "m.dendrite.test", msg); err != nil {
+				t.Fatalf("unable to send to device message: %v", err)
+			}
+		}
+		time.Sleep((time.Millisecond * 15) * time.Duration(tc.sendMessagesCount)) // wait a bit, so the messages can be processed
+		// Execute a /sync request, recording the response
+		w := httptest.NewRecorder()
+		base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+			"access_token": alice.AccessToken,
+			"since":        tc.since,
+		})))
+
+		// Extract the to_device.events, # gets all values of an array, in this case a string slice with "message $counter" entries
+		events := gjson.Get(w.Body.String(), "to_device.events.#.content.dummy").Array()
+		got := make([]string, len(events))
+		for i := range events {
+			got[i] = events[i].String()
+		}
+
+		// Ensure the messages we received are as we expect them to be
+		if !reflect.DeepEqual(got, tc.want) {
+			t.Logf("[%s|since=%s]: Sync: %s", tc.name, tc.since, w.Body.String())
+			t.Fatalf("[%s|since=%s]: got: %+v, want: %+v", tc.name, tc.since, got, tc.want)
+		}
+	}
+}
+
 func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
 	result := make([]*nats.Msg, len(input))
 	for i, ev := range input {
diff --git a/test/testrig/base.go b/test/testrig/base.go
index facb49f3e..d13c43129 100644
--- a/test/testrig/base.go
+++ b/test/testrig/base.go
@@ -32,11 +32,11 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 	var cfg config.Dendrite
 	cfg.Defaults(false)
 	cfg.Global.JetStream.InMemory = true
-
 	switch dbType {
 	case test.DBTypePostgres:
 		cfg.Global.Defaults(true)   // autogen a signing key
 		cfg.MediaAPI.Defaults(true) // autogen a media path
+		cfg.Global.ServerName = "test"
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
 		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
@@ -50,6 +50,7 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 		return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), close
 	case test.DBTypeSQLite:
 		cfg.Defaults(true) // sets a sqlite db per component
+		cfg.Global.ServerName = "test"
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
 		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)

From ac2dbb3513b2e6ff24624f30a2a8b1bd51bdabc5 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 3 Aug 2022 10:55:21 +0200
Subject: [PATCH 236/391] Add Cache-Control header to media endpoints (#2612)

* Add Cache-Control header

* Raise rate_limiting threshold to 20
---
 dendrite-sample.monolith.yaml | 2 +-
 dendrite-sample.polylith.yaml | 2 +-
 mediaapi/routing/routing.go   | 3 +++
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index a34b8af55..816c4cae9 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -192,7 +192,7 @@ client_api:
   # and appservice users are exempt from rate limiting by default.
   rate_limiting:
     enabled: true
-    threshold: 5
+    threshold: 20
     cooloff_ms: 500
     exempt_user_ids:
     #  - "@user:domain.com"
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 550611229..4784dbaff 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -195,7 +195,7 @@ client_api:
   # and appservice users are exempt from rate limiting by default.
   rate_limiting:
     enabled: true
-    threshold: 5
+    threshold: 20
     cooloff_ms: 500
     exempt_user_ids:
     #  - "@user:domain.com"
diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go
index 196908184..9dcfa955f 100644
--- a/mediaapi/routing/routing.go
+++ b/mediaapi/routing/routing.go
@@ -149,6 +149,9 @@ func makeDownloadAPI(
 			}
 		}
 
+		// Cache media for at least one day.
+		w.Header().Set("Cache-Control", "public,max-age=86400,s-maxage=86400")
+
 		Download(
 			w,
 			req,

From f7f2453a859e9a2b2cccec5b0f47d558bc9ca507 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 3 Aug 2022 10:35:57 +0100
Subject: [PATCH 237/391] Test Go 1.19 in CI

---
 .github/workflows/dendrite.yml | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 4cbfb380f..0d1970efd 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -97,7 +97,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18"]
+        go: ["1.18", "1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
@@ -127,7 +127,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18"]
+        go: ["1.18", "1.19"]
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
@@ -160,7 +160,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.18"]
+        go: ["1.18", "1.19"]
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
@@ -384,7 +384,14 @@ jobs:
 
   integration-tests-done:
     name: Integration tests passed
-    needs: [initial-tests-done, upgrade_test, upgrade_test_direct, sytest, complement]
+    needs:
+      [
+        initial-tests-done,
+        upgrade_test,
+        upgrade_test_direct,
+        sytest,
+        complement,
+      ]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
     steps:

From 376391d1c7e309e4a09998c0717ec8adc70fe1a4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 3 Aug 2022 10:38:36 +0100
Subject: [PATCH 238/391] Update Pinecone

---
 go.mod | 11 ++++++-----
 go.sum | 25 ++++++++++++++-----------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/go.mod b/go.mod
index 5b2d8670d..3559c5bb1 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771
-	github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a
+	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
 	github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee
@@ -45,7 +45,7 @@ require (
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
 	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
 	golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
-	golang.org/x/net v0.0.0-20220524220425-1d687d428aca
+	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
@@ -73,10 +73,11 @@ require (
 	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/lucas-clemente/quic-go v0.26.0 // indirect
+	github.com/lucas-clemente/quic-go v0.28.1 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/miekg/dns v1.1.49 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
diff --git a/go.sum b/go.sum
index 6321adada..2c8bb4f18 100644
--- a/go.sum
+++ b/go.sum
@@ -321,8 +321,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
 github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lucas-clemente/quic-go v0.26.0 h1:ALBQXr9UJ8A1LyzvceX4jd9QFsHvlI0RR6BkV16o00A=
-github.com/lucas-clemente/quic-go v0.26.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
+github.com/lucas-clemente/quic-go v0.28.1 h1:Uo0lvVxWg5la9gflIF9lwa39ONq85Xq2D91YNEIslzU=
+github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
@@ -330,10 +330,12 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
 github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
-github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc=
-github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
-github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y=
-github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
+github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
+github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
+github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM=
+github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 h1:7m/WlWcSROrcK5NxuXaxYD32BZqe/LEEnBrWcH/cOqQ=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA=
@@ -343,8 +345,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771 h1:ZIPHFIPNDS9dmEbPEiJbNmyCGJtn9exfpLC7JOcn/bE=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
-github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a h1:DdG8vXMlZ65EAtc4V+3t7zHZ2Gqs24pSnyXS+4BRHUs=
-github.com/matrix-org/pinecone v0.0.0-20220708135211-1ce778fcde6a/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
+github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
+github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -549,7 +551,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
@@ -659,8 +660,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220524220425-1d687d428aca h1:xTaFYiPROfpPhqrfTIDXj0ri1SpfueYT951s4bAuDO8=
-golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -750,6 +751,7 @@ golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -758,6 +760,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0=
 golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

From bbff41b44bff2dbc53867cc0fd94ce8f31fd511a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 3 Aug 2022 10:50:45 +0100
Subject: [PATCH 239/391] Disable stack protector on Linux CI build pipelines
 for now (to avoid `relocation target __stack_chk_fail_local not defined`
 errors)

---
 .github/workflows/dendrite.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 0d1970efd..19ebd760c 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -151,6 +151,7 @@ jobs:
           GOOS: ${{ matrix.goos }}
           GOARCH: ${{ matrix.goarch }}
           CGO_ENABLED: 1
+          CGO_CFLAGS: -fno-stack-protector
         run: go build -trimpath -v -o "bin/" ./cmd/...
 
   # build for Windows 64-bit

From 2250768be16bd0e6b3a6a72b5e55eb3e2ad6e3c6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 3 Aug 2022 17:14:21 +0100
Subject: [PATCH 240/391] Remove roominfo cache (#2615)

* Remove roominfo cache

It's the source of a number of race conditions which are seemingly causing bugs and CI failures.

* Make the linter less sad
---
 internal/caching/cache_roominfo.go       | 33 ----------------------
 internal/caching/cache_roomservernids.go |  1 -
 internal/caching/caches.go               |  1 -
 internal/caching/impl_ristretto.go       |  7 -----
 roomserver/storage/shared/storage.go     | 35 ++++++------------------
 5 files changed, 9 insertions(+), 68 deletions(-)
 delete mode 100644 internal/caching/cache_roominfo.go

diff --git a/internal/caching/cache_roominfo.go b/internal/caching/cache_roominfo.go
deleted file mode 100644
index 5dfed3c85..000000000
--- a/internal/caching/cache_roominfo.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package caching
-
-import (
-	"github.com/matrix-org/dendrite/roomserver/types"
-)
-
-// WARNING: This cache is mutable because it's entirely possible that
-// the IsStub or StateSnaphotNID fields can change, even though the
-// room version and room NID fields will not. This is only safe because
-// the RoomInfoCache is used ONLY within the roomserver and because it
-// will be kept up-to-date by the latest events updater. It MUST NOT be
-// used from other components as we currently have no way to invalidate
-// the cache in downstream components.
-
-// RoomInfosCache contains the subset of functions needed for
-// a room Info cache. It must only be used from the roomserver only
-// It is not safe for use from other components.
-type RoomInfoCache interface {
-	GetRoomInfo(roomID string) (roomInfo *types.RoomInfo, ok bool)
-	StoreRoomInfo(roomID string, roomInfo *types.RoomInfo)
-}
-
-// GetRoomInfo must only be called from the roomserver only. It is not
-// safe for use from other components.
-func (c Caches) GetRoomInfo(roomID string) (*types.RoomInfo, bool) {
-	return c.RoomInfos.Get(roomID)
-}
-
-// StoreRoomInfo must only be called from the roomserver only. It is not
-// safe for use from other components.
-func (c Caches) StoreRoomInfo(roomID string, roomInfo *types.RoomInfo) {
-	c.RoomInfos.Set(roomID, roomInfo)
-}
diff --git a/internal/caching/cache_roomservernids.go b/internal/caching/cache_roomservernids.go
index f27154f19..88a5b28bc 100644
--- a/internal/caching/cache_roomservernids.go
+++ b/internal/caching/cache_roomservernids.go
@@ -7,7 +7,6 @@ import (
 type RoomServerCaches interface {
 	RoomServerNIDsCache
 	RoomVersionCache
-	RoomInfoCache
 	RoomServerEventsCache
 	EventStateKeyCache
 }
diff --git a/internal/caching/caches.go b/internal/caching/caches.go
index f13f743d3..78c9ab7ee 100644
--- a/internal/caching/caches.go
+++ b/internal/caching/caches.go
@@ -29,7 +29,6 @@ type Caches struct {
 	RoomServerRoomIDs   Cache[types.RoomNID, string]                           // room NID -> room ID
 	RoomServerEvents    Cache[int64, *gomatrixserverlib.Event]                 // event NID -> event
 	RoomServerStateKeys Cache[types.EventStateKeyNID, string]                  // event NID -> event state key
-	RoomInfos           Cache[string, *types.RoomInfo]                         // room ID -> room info
 	FederationPDUs      Cache[int64, *gomatrixserverlib.HeaderedEvent]         // queue NID -> PDU
 	FederationEDUs      Cache[int64, *gomatrixserverlib.EDU]                   // queue NID -> EDU
 	SpaceSummaryRooms   Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response
diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go
index fdbbb4d72..fc0c8cc0f 100644
--- a/internal/caching/impl_ristretto.go
+++ b/internal/caching/impl_ristretto.go
@@ -35,7 +35,6 @@ const (
 	roomNIDsCache
 	roomIDsCache
 	roomEventsCache
-	roomInfosCache
 	federationPDUsCache
 	federationEDUsCache
 	spaceSummaryRoomsCache
@@ -106,12 +105,6 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm
 			Prefix: eventStateKeyCache,
 			MaxAge: maxAge,
 		},
-		RoomInfos: &RistrettoCachePartition[string, *types.RoomInfo]{ // room ID -> room info
-			cache:   cache,
-			Prefix:  roomInfosCache,
-			Mutable: true,
-			MaxAge:  maxAge,
-		},
 		FederationPDUs: &RistrettoCostedCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{ // queue NID -> PDU
 			&RistrettoCachePartition[int64, *gomatrixserverlib.HeaderedEvent]{
 				cache:   cache,
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 9e6a4142c..cbf9c8b20 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -156,30 +156,15 @@ func (d *Database) RoomInfo(ctx context.Context, roomID string) (*types.RoomInfo
 }
 
 func (d *Database) roomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) {
-	roomInfo, ok := d.Cache.GetRoomInfo(roomID)
-	if ok && roomInfo != nil && !roomInfo.IsStub() {
-		// The data that's in the cache is not stubby, so return it.
-		return roomInfo, nil
-	}
-	// At this point we either don't have an entry in the cache, or
-	// it is stubby, so let's check the roomserver_rooms table again.
-	roomInfoFromDB, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID)
+	roomInfo, err := d.RoomsTable.SelectRoomInfo(ctx, txn, roomID)
 	if err != nil {
 		return nil, err
 	}
-	// If we have a stubby cache entry already, update it and return
-	// the reference to the cache entry.
 	if roomInfo != nil {
-		roomInfo.CopyFrom(roomInfoFromDB)
-		return roomInfo, nil
+		d.Cache.StoreRoomServerRoomID(roomInfo.RoomNID, roomID)
+		d.Cache.StoreRoomVersion(roomID, roomInfo.RoomVersion)
 	}
-	// Otherwise, try to admit the data into the cache and return the
-	// new reference from the database.
-	if roomInfoFromDB != nil {
-		d.Cache.StoreRoomServerRoomID(roomInfoFromDB.RoomNID, roomID)
-		d.Cache.StoreRoomInfo(roomID, roomInfoFromDB)
-	}
-	return roomInfoFromDB, err
+	return roomInfo, err
 }
 
 func (d *Database) AddState(
@@ -504,8 +489,8 @@ func (d *Database) events(
 	fetchNIDList := make([]types.RoomNID, 0, len(uniqueRoomNIDs))
 	for n := range uniqueRoomNIDs {
 		if roomID, ok := d.Cache.GetRoomServerRoomID(n); ok {
-			if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok {
-				roomVersions[n] = roomInfo.RoomVersion
+			if roomVersion, ok := d.Cache.GetRoomVersion(roomID); ok {
+				roomVersions[n] = roomVersion
 				continue
 			}
 		}
@@ -762,9 +747,6 @@ func (d *Database) MissingAuthPrevEvents(
 func (d *Database) assignRoomNID(
 	ctx context.Context, roomID string, roomVersion gomatrixserverlib.RoomVersion,
 ) (types.RoomNID, error) {
-	if roomInfo, ok := d.Cache.GetRoomInfo(roomID); ok {
-		return roomInfo.RoomNID, nil
-	}
 	// Check if we already have a numeric ID in the database.
 	roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID)
 	if err == sql.ErrNoRows {
@@ -837,8 +819,9 @@ func extractRoomVersionFromCreateEvent(event *gomatrixserverlib.Event) (
 // "servers should not apply or send redactions to clients until both the redaction event and original event have been seen, and are valid."
 // https://matrix.org/docs/spec/rooms/v3#authorization-rules-for-events
 // These cases are:
-//  - This is a redaction event, redact the event it references if we know about it.
-//  - This is a normal event which may have been previously redacted.
+//   - This is a redaction event, redact the event it references if we know about it.
+//   - This is a normal event which may have been previously redacted.
+//
 // In the first case, check if we have the referenced event then apply the redaction, else store it
 // in the redactions table with validated=FALSE. In the second case, check if there is a redaction for it:
 // if there is then apply the redactions and set validated=TRUE.

From 9fe509b18da997e294813fcc5f46a45b7f6e6784 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 3 Aug 2022 18:35:17 +0200
Subject: [PATCH 241/391] Fix syncapi shared users query & device lists (#2614)

* Fix query issue, only add "changed" users if we actually share a room

* Avoid log spam if context is done

* Undo changes to filterSharedUsers

* Add logging again..

* Fix SQLite shared users query

* Change query to include invited users
---
 keyserver/internal/internal.go                | 11 +++++
 syncapi/internal/keychange.go                 | 44 +++++++++---------
 syncapi/internal/keychange_test.go            |  1 +
 .../postgres/current_room_state_table.go      |  4 +-
 .../sqlite3/current_room_state_table.go       | 45 ++++++++++---------
 5 files changed, 62 insertions(+), 43 deletions(-)

diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index c146b2aa0..91f011517 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -314,6 +315,11 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 		for targetKeyID := range masterKey.Keys {
 			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID)
 			if err != nil {
+				// Stop executing the function if the context was canceled/the deadline was exceeded,
+				// as we can't continue without a valid context.
+				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+					return
+				}
 				logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
 				continue
 			}
@@ -335,6 +341,11 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 		for targetKeyID, key := range forUserID {
 			sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID))
 			if err != nil {
+				// Stop executing the function if the context was canceled/the deadline was exceeded,
+				// as we can't continue without a valid context.
+				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+					return
+				}
 				logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
 				continue
 			}
diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go
index 03df9285c..4bf54cae0 100644
--- a/syncapi/internal/keychange.go
+++ b/syncapi/internal/keychange.go
@@ -25,10 +25,9 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
-const DeviceListLogName = "dl"
-
 // DeviceOTKCounts adds one-time key counts to the /sync response
 func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, deviceID string, res *types.Response) error {
 	var queryRes keyapi.QueryOneTimeKeysResponse
@@ -93,18 +92,13 @@ func DeviceListCatchup(
 	queryRes.UserIDs = append(queryRes.UserIDs, joinUserIDs...)
 	queryRes.UserIDs = append(queryRes.UserIDs, leaveUserIDs...)
 	queryRes.UserIDs = util.UniqueStrings(queryRes.UserIDs)
-	var sharedUsersMap map[string]int
-	sharedUsersMap, queryRes.UserIDs = filterSharedUsers(ctx, db, userID, queryRes.UserIDs)
-	util.GetLogger(ctx).Debugf(
-		"QueryKeyChanges request off=%d,to=%d response off=%d uids=%v",
-		offset, toOffset, queryRes.Offset, queryRes.UserIDs,
-	)
+	sharedUsersMap := filterSharedUsers(ctx, db, userID, queryRes.UserIDs)
 	userSet := make(map[string]bool)
 	for _, userID := range res.DeviceLists.Changed {
 		userSet[userID] = true
 	}
-	for _, userID := range queryRes.UserIDs {
-		if !userSet[userID] {
+	for userID, count := range sharedUsersMap {
+		if !userSet[userID] && count > 0 {
 			res.DeviceLists.Changed = append(res.DeviceLists.Changed, userID)
 			hasNew = true
 			userSet[userID] = true
@@ -113,7 +107,7 @@ func DeviceListCatchup(
 	// Finally, add in users who have joined or left.
 	// TODO: This is sub-optimal because we will add users to `changed` even if we already shared a room with them.
 	for _, userID := range joinUserIDs {
-		if !userSet[userID] {
+		if !userSet[userID] && sharedUsersMap[userID] > 0 {
 			res.DeviceLists.Changed = append(res.DeviceLists.Changed, userID)
 			hasNew = true
 			userSet[userID] = true
@@ -126,6 +120,13 @@ func DeviceListCatchup(
 		}
 	}
 
+	util.GetLogger(ctx).WithFields(logrus.Fields{
+		"user_id":         userID,
+		"from":            offset,
+		"to":              toOffset,
+		"response_offset": queryRes.Offset,
+	}).Debugf("QueryKeyChanges request result: %+v", res.DeviceLists)
+
 	return types.StreamPosition(queryRes.Offset), hasNew, nil
 }
 
@@ -220,24 +221,27 @@ func TrackChangedUsers(
 // it down to include only users who the requesting user shares a room with.
 func filterSharedUsers(
 	ctx context.Context, db storage.SharedUsers, userID string, usersWithChangedKeys []string,
-) (map[string]int, []string) {
+) map[string]int {
 	sharedUsersMap := make(map[string]int, len(usersWithChangedKeys))
-	for _, userID := range usersWithChangedKeys {
-		sharedUsersMap[userID] = 0
+	for _, changedUserID := range usersWithChangedKeys {
+		sharedUsersMap[changedUserID] = 0
+		if changedUserID == userID {
+			// We forcibly put ourselves in this list because we should be notified about our own device updates
+			// and if we are in 0 rooms then we don't technically share any room with ourselves so we wouldn't
+			// be notified about key changes.
+			sharedUsersMap[userID] = 1
+		}
 	}
 	sharedUsers, err := db.SharedUsers(ctx, userID, usersWithChangedKeys)
 	if err != nil {
+		util.GetLogger(ctx).WithError(err).Errorf("db.SharedUsers failed: %s", err)
 		// default to all users so we do needless queries rather than miss some important device update
-		return nil, usersWithChangedKeys
+		return sharedUsersMap
 	}
 	for _, userID := range sharedUsers {
 		sharedUsersMap[userID]++
 	}
-	// We forcibly put ourselves in this list because we should be notified about our own device updates
-	// and if we are in 0 rooms then we don't technically share any room with ourselves so we wouldn't
-	// be notified about key changes.
-	sharedUsersMap[userID] = 1
-	return sharedUsersMap, sharedUsers
+	return sharedUsersMap
 }
 
 func joinedRooms(res *types.Response, userID string) []string {
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 79ed440e7..6bfc91edd 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -129,6 +129,7 @@ type wantCatchup struct {
 }
 
 func assertCatchup(t *testing.T, hasNew bool, syncResponse *types.Response, want wantCatchup) {
+	t.Helper()
 	if hasNew != want.hasNew {
 		t.Errorf("got hasNew=%v want %v", hasNew, want.hasNew)
 	}
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index d13b7be41..58f404511 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -112,7 +112,7 @@ const selectEventsWithEventIDsSQL = "" +
 const selectSharedUsersSQL = "" +
 	"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
 	"	SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
-	") AND state_key = ANY($2) AND membership='join';"
+	") AND state_key = ANY($2) AND membership IN ('join', 'invite');"
 
 type currentRoomStateStatements struct {
 	upsertRoomStateStmt                *sql.Stmt
@@ -407,7 +407,7 @@ func (s *currentRoomStateStatements) SelectSharedUsers(
 	ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string,
 ) ([]string, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectSharedUsersStmt)
-	rows, err := stmt.QueryContext(ctx, userID, otherUserIDs)
+	rows, err := stmt.QueryContext(ctx, userID, pq.Array(otherUserIDs))
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index e19298aee..3a10b2325 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -94,9 +94,9 @@ const selectEventsWithEventIDsSQL = "" +
 	" FROM syncapi_current_room_state WHERE event_id IN ($1)"
 
 const selectSharedUsersSQL = "" +
-	"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
+	"SELECT state_key FROM syncapi_current_room_state WHERE room_id IN(" +
 	"	SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
-	") AND state_key IN ($2) AND membership='join';"
+	") AND state_key IN ($2) AND membership IN ('join', 'invite');"
 
 type currentRoomStateStatements struct {
 	db                                 *sql.DB
@@ -420,25 +420,28 @@ func (s *currentRoomStateStatements) SelectStateEvent(
 func (s *currentRoomStateStatements) SelectSharedUsers(
 	ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string,
 ) ([]string, error) {
-	query := strings.Replace(selectSharedUsersSQL, "($2)", sqlutil.QueryVariadicOffset(len(otherUserIDs), 1), 1)
-	stmt, err := s.db.Prepare(query)
-	if err != nil {
-		return nil, fmt.Errorf("SelectSharedUsers s.db.Prepare: %w", err)
-	}
-	defer internal.CloseAndLogIfError(ctx, stmt, "SelectSharedUsers: stmt.close() failed")
-	rows, err := sqlutil.TxStmt(txn, stmt).QueryContext(ctx, userID, otherUserIDs)
-	if err != nil {
-		return nil, err
-	}
-	defer internal.CloseAndLogIfError(ctx, rows, "selectSharedUsersStmt: rows.close() failed")
 
-	var stateKey string
-	result := make([]string, 0, len(otherUserIDs))
-	for rows.Next() {
-		if err := rows.Scan(&stateKey); err != nil {
-			return nil, err
-		}
-		result = append(result, stateKey)
+	params := make([]interface{}, len(otherUserIDs)+1)
+	params[0] = userID
+	for k, v := range otherUserIDs {
+		params[k+1] = v
 	}
-	return result, rows.Err()
+
+	result := make([]string, 0, len(otherUserIDs))
+	query := strings.Replace(selectSharedUsersSQL, "($2)", sqlutil.QueryVariadicOffset(len(otherUserIDs), 1), 1)
+	err := sqlutil.RunLimitedVariablesQuery(
+		ctx, query, s.db, params, sqlutil.SQLite3MaxVariables,
+		func(rows *sql.Rows) error {
+			var stateKey string
+			for rows.Next() {
+				if err := rows.Scan(&stateKey); err != nil {
+					return err
+				}
+				result = append(result, stateKey)
+			}
+			return nil
+		},
+	)
+
+	return result, err
 }

From 3bf5ae5ffef0ebc140f55320658d9b07bc58e848 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 3 Aug 2022 17:37:27 +0100
Subject: [PATCH 242/391] Try more servers when calling `/state_ids` (#2610)

* Try more servers when calling `/state_ids`

* More logging

* Maybe fix concurrent map write

* Revert "Maybe fix concurrent map write"

This reverts commit da0dbb836207a911afe77e6f6d63c4809669693c.

* Enforce a limit of 20s per server, 5 mins total
---
 roomserver/internal/input/input_missing.go | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index c78e5d79a..0dd2b64c0 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -326,8 +326,10 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion
 		return respState, true, nil
 	}
 
+	logrus.WithContext(ctx).Warnf("State for event %s not available locally, falling back to federation (via %d servers)", eventID, len(t.servers))
 	respState, err := t.lookupStateBeforeEvent(ctx, roomVersion, roomID, eventID)
 	if err != nil {
+		logrus.WithContext(ctx).WithError(err).Errorf("Failed to look up state before event %s", eventID)
 		return nil, false, fmt.Errorf("t.lookupStateBeforeEvent: %w", err)
 	}
 
@@ -339,6 +341,7 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion
 	case nil:
 		// do nothing
 	default:
+		logrus.WithContext(ctx).WithError(err).Errorf("Failed to look up event %s", eventID)
 		return nil, false, fmt.Errorf("t.lookupEvent: %w", err)
 	}
 	h = t.cacheAndReturn(h)
@@ -662,9 +665,22 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
 
 	util.GetLogger(ctx).WithField("room_id", roomID).Infof("lookupMissingStateViaStateIDs %s", eventID)
 	// fetch the state event IDs at the time of the event
-	stateIDs, err := t.federation.LookupStateIDs(ctx, t.origin, roomID, eventID)
+	var stateIDs gomatrixserverlib.RespStateIDs
+	var err error
+	count := 0
+	totalctx, totalcancel := context.WithTimeout(ctx, time.Minute*5)
+	for _, serverName := range t.servers {
+		reqctx, reqcancel := context.WithTimeout(totalctx, time.Second*20)
+		stateIDs, err = t.federation.LookupStateIDs(reqctx, serverName, roomID, eventID)
+		reqcancel()
+		if err == nil {
+			break
+		}
+		count++
+	}
+	totalcancel()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("t.federation.LookupStateIDs tried %d server(s), last error: %w", count, err)
 	}
 	// work out which auth/state IDs are missing
 	wantIDs := append(stateIDs.StateEventIDs, stateIDs.AuthEventIDs...)

From a2bed259dd765bd0b9781cd65594343d22c07986 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 3 Aug 2022 17:42:13 +0100
Subject: [PATCH 243/391] Version 0.9.1 (#2616)

* Version 0.9.1

* Update CHANGES.md
---
 CHANGES.md          | 14 ++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 0ae927446..5dd8da362 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,19 @@
 # Changelog
 
+## Dendrite 0.9.1 (2022-08-03)
+
+### Fixes
+
+* Upgrades a dependency which caused issues building Dendrite with Go 1.19
+* The roomserver will no longer give up prematurely after failing to call `/state_ids`
+* Removes the faulty room info cache, which caused of a number of race conditions and occasional bugs (including when creating and joining rooms)
+* The media endpoint now sets the `Cache-Control` header correctly to prevent web-based clients from hitting media endpoints excessively
+* The sync API will now advance the PDU stream position correctly in all cases (contributed by [sergekh2](https://github.com/sergekh2))
+* The sync API will now delete the correct range of send-to-device messages when advancing the stream position
+* The device list `changed` key in the `/sync` response should now return the correct users 
+* A data race when looking up missing state has been fixed
+* The `/send_join` API is now applying stronger validation to the received membership event
+
 ## Dendrite 0.9.0 (2022-08-01)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index 6d29a68ee..38d0864e7 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 0
+	VersionPatch = 1
 	VersionTag   = "" // example: "rc1"
 )
 

From 9a655cb5e7f8b96a0b02203e69d866dfb1a184e2 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 5 Aug 2022 07:20:34 +0200
Subject: [PATCH 244/391] Only create a new destinationQueue if we don't have
 one (#2620)

---
 federationapi/queue/queue.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go
index 4c25c4ce6..88664fcf9 100644
--- a/federationapi/queue/queue.go
+++ b/federationapi/queue/queue.go
@@ -158,7 +158,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
 	oqs.queuesMutex.Lock()
 	defer oqs.queuesMutex.Unlock()
 	oq, ok := oqs.queues[destination]
-	if !ok || oq != nil {
+	if !ok || oq == nil {
 		destinationQueueTotal.Inc()
 		oq = &destinationQueue{
 			queues:           oqs,

From de78eab63a99653edf68f783e263688ad4b701d8 Mon Sep 17 00:00:00 2001
From: Brian Meek <brian@hntlabs.com>
Date: Fri, 5 Aug 2022 01:19:33 -0700
Subject: [PATCH 245/391] Add race testing to tests, and fix a few small race
 conditions in the tests (#2587)

* Add race testing to tests, and fix a few small race conditions in the tests

* Enable run-sytest on MacOS

* Remove deadlock detecting mutex, per code review feedback

* Remove autoformatting related changes and a closure that is not needed

* Adjust to importing nats client as 'natsclient'

Signed-off-by: Brian Meek <brian@hntlabs.com>

* Clarify the use of gooseMutex to proect goose internal state

Signed-off-by: Brian Meek <brian@hntlabs.com>

* Remove no longer needed mutex for guarding goose

Signed-off-by: Brian Meek <brian@hntlabs.com>
---
 build/scripts/build-test-lint.sh    |  2 +-
 docs/CONTRIBUTING.md                |  2 +-
 federationapi/federationapi_test.go | 10 ++++++++++
 keyserver/storage/storage_test.go   |  9 ++++++++-
 run-sytest.sh                       |  2 +-
 setup/jetstream/nats.go             | 14 +++++++-------
 syncapi/sync/requestpool_test.go    |  7 +++++++
 7 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/build/scripts/build-test-lint.sh b/build/scripts/build-test-lint.sh
index 8f0b775b1..32f89c076 100755
--- a/build/scripts/build-test-lint.sh
+++ b/build/scripts/build-test-lint.sh
@@ -13,4 +13,4 @@ go build ./cmd/...
 ./build/scripts/find-lint.sh
 
 echo "Testing..."
-go test -v ./...
+go test --race -v ./...
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 169224b9e..771af9ecf 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -64,7 +64,7 @@ comment. Please avoid doing this if you can.
 We also have unit tests which we run via:
 
 ```bash
-go test ./...
+go test --race ./...
 ```
 
 In general, we like submissions that come with tests. Anything that proves that the
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index ae244c566..8884e34c6 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -48,6 +49,7 @@ func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.Que
 
 // TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
 type fedClient struct {
+	fedClientMutex sync.Mutex
 	api.FederationClient
 	allowJoins []*test.Room
 	keys       map[gomatrixserverlib.ServerName]struct {
@@ -59,6 +61,8 @@ type fedClient struct {
 }
 
 func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) {
+	f.fedClientMutex.Lock()
+	defer f.fedClientMutex.Unlock()
 	fmt.Println("GetServerKeys:", matrixServer)
 	var keys gomatrixserverlib.ServerKeys
 	var keyID gomatrixserverlib.KeyID
@@ -122,6 +126,8 @@ func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName
 	return
 }
 func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
+	f.fedClientMutex.Lock()
+	defer f.fedClientMutex.Unlock()
 	for _, r := range f.allowJoins {
 		if r.ID == event.RoomID() {
 			r.InsertEvent(f.t, event.Headered(r.Version))
@@ -134,6 +140,8 @@ func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName
 }
 
 func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
+	f.fedClientMutex.Lock()
+	defer f.fedClientMutex.Unlock()
 	for _, edu := range t.EDUs {
 		if edu.Type == gomatrixserverlib.MDeviceListUpdate {
 			f.sentTxn = true
@@ -242,6 +250,8 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 
 	testrig.MustPublishMsgs(t, jsctx, msg)
 	time.Sleep(500 * time.Millisecond)
+	fc.fedClientMutex.Lock()
+	defer fc.fedClientMutex.Unlock()
 	if !fc.sentTxn {
 		t.Fatalf("did not send device list update")
 	}
diff --git a/keyserver/storage/storage_test.go b/keyserver/storage/storage_test.go
index 44cfb5f2a..e7a2af7c2 100644
--- a/keyserver/storage/storage_test.go
+++ b/keyserver/storage/storage_test.go
@@ -3,6 +3,7 @@ package storage_test
 import (
 	"context"
 	"reflect"
+	"sync"
 	"testing"
 
 	"github.com/matrix-org/dendrite/keyserver/api"
@@ -103,6 +104,9 @@ func TestKeyChangesUpperLimit(t *testing.T) {
 	})
 }
 
+var dbLock sync.Mutex
+var deviceArray = []string{"AAA", "another_device"}
+
 // The purpose of this test is to make sure that the storage layer is generating sequential stream IDs per user,
 // and that they are returned correctly when querying for device keys.
 func TestDeviceKeysStreamIDGeneration(t *testing.T) {
@@ -169,8 +173,11 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) {
 			t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=3 (new key same device) but got %d", msgs[0].StreamID)
 		}
 
+		dbLock.Lock()
+		defer dbLock.Unlock()
 		// Querying for device keys returns the latest stream IDs
-		msgs, err = db.DeviceKeysForUser(ctx, alice, []string{"AAA", "another_device"}, false)
+		msgs, err = db.DeviceKeysForUser(ctx, alice, deviceArray, false)
+
 		if err != nil {
 			t.Fatalf("DeviceKeysForUser returned error: %s", err)
 		}
diff --git a/run-sytest.sh b/run-sytest.sh
index 47635fd12..e23982397 100755
--- a/run-sytest.sh
+++ b/run-sytest.sh
@@ -17,7 +17,7 @@ main() {
 
     if [ -d ../sytest ]; then
         local tmpdir
-        tmpdir="$(mktemp -d --tmpdir run-systest.XXXXXXXXXX)"
+        tmpdir="$(mktemp -d -t run-systest.XXXXXXXXXX)"
         trap "rm -r '$tmpdir'" EXIT
 
         if [ -z "$DISABLE_BUILDING_SYTEST" ]; then
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index be216a02a..051d55a35 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -14,16 +14,16 @@ import (
 	"github.com/sirupsen/logrus"
 
 	natsserver "github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
 	natsclient "github.com/nats-io/nats.go"
 )
 
 type NATSInstance struct {
 	*natsserver.Server
-	sync.Mutex
 }
 
-func DeleteAllStreams(js nats.JetStreamContext, cfg *config.JetStream) {
+var natsLock sync.Mutex
+
+func DeleteAllStreams(js natsclient.JetStreamContext, cfg *config.JetStream) {
 	for _, stream := range streams { // streams are defined in streams.go
 		name := cfg.Prefixed(stream.Name)
 		_ = js.DeleteStream(name)
@@ -31,11 +31,12 @@ func DeleteAllStreams(js nats.JetStreamContext, cfg *config.JetStream) {
 }
 
 func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetStream) (natsclient.JetStreamContext, *natsclient.Conn) {
+	natsLock.Lock()
+	defer natsLock.Unlock()
 	// check if we need an in-process NATS Server
 	if len(cfg.Addresses) != 0 {
 		return setupNATS(process, cfg, nil)
 	}
-	s.Lock()
 	if s.Server == nil {
 		var err error
 		s.Server, err = natsserver.NewServer(&natsserver.Options{
@@ -63,7 +64,6 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
 			process.ComponentFinished()
 		}()
 	}
-	s.Unlock()
 	if !s.ReadyForConnections(time.Second * 10) {
 		logrus.Fatalln("NATS did not start in time")
 	}
@@ -77,9 +77,9 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
 func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
 	if nc == nil {
 		var err error
-		opts := []nats.Option{}
+		opts := []natsclient.Option{}
 		if cfg.DisableTLSValidation {
-			opts = append(opts, nats.Secure(&tls.Config{
+			opts = append(opts, natsclient.Secure(&tls.Config{
 				InsecureSkipVerify: true,
 			}))
 		}
diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go
index 48e6c6c7a..3e5769d8c 100644
--- a/syncapi/sync/requestpool_test.go
+++ b/syncapi/sync/requestpool_test.go
@@ -12,10 +12,13 @@ import (
 )
 
 type dummyPublisher struct {
+	lock  sync.Mutex
 	count int
 }
 
 func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, statusMsg *string) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
 	d.count++
 	return nil
 }
@@ -125,11 +128,15 @@ func TestRequestPool_updatePresence(t *testing.T) {
 	go rp.cleanPresence(db, time.Millisecond*50)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			publisher.lock.Lock()
 			beforeCount := publisher.count
+			publisher.lock.Unlock()
 			rp.updatePresence(db, tt.args.presence, tt.args.userID)
+			publisher.lock.Lock()
 			if tt.wantIncrease && publisher.count <= beforeCount {
 				t.Fatalf("expected count to increase: %d <= %d", publisher.count, beforeCount)
 			}
+			publisher.lock.Unlock()
 			time.Sleep(tt.args.sleep)
 		})
 	}

From 1b7f84250a46b401eccb89acafdef1b379f2dbc0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 5 Aug 2022 11:12:41 +0200
Subject: [PATCH 246/391] Fix linter issues (#2624)

* Try that again

* All hail the mighty linter?

* And once again

* goimport all the things
---
 clientapi/routing/directory_public.go         | 17 ++++++-----
 clientapi/routing/sendevent.go                |  7 +++--
 clientapi/routing/threepid.go                 |  5 ++--
 clientapi/routing/voip.go                     |  8 +++--
 cmd/dendrite-upgrade-tests/main.go            |  6 ++--
 cmd/dendrite-upgrade-tests/tests.go           |  6 ++--
 federationapi/consumers/roomserver.go         |  8 +++--
 federationapi/routing/join.go                 | 21 ++++++-------
 internal/log.go                               |  5 ++--
 keyserver/internal/device_list_update.go      |  7 +++--
 keyserver/internal/device_list_update_test.go |  3 +-
 roomserver/internal/input/input.go            | 15 +++++-----
 .../internal/input/input_latest_events.go     | 30 +++++++++----------
 .../internal/perform/perform_backfill.go      | 10 ++++---
 roomserver/storage/tables/interface.go        |  5 ++--
 setup/mscs/msc2836/msc2836_test.go            | 23 +++++++-------
 syncapi/internal/keychange_test.go            | 20 +++++++------
 syncapi/routing/filter.go                     | 11 ++++---
 syncapi/storage/tables/interface.go           | 17 ++++++-----
 userapi/storage/sqlite3/stats_table.go        | 17 ++++++-----
 20 files changed, 136 insertions(+), 105 deletions(-)

diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go
index c3e6141b2..8ddb3267a 100644
--- a/clientapi/routing/directory_public.go
+++ b/clientapi/routing/directory_public.go
@@ -23,13 +23,14 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 var (
@@ -196,14 +197,14 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 
 // sliceInto returns a subslice of `slice` which honours the since/limit values given.
 //
-//    0  1  2  3  4  5  6   index
-//   [A, B, C, D, E, F, G]  slice
+//	  0  1  2  3  4  5  6   index
+//	 [A, B, C, D, E, F, G]  slice
 //
-//   limit=3          => A,B,C (prev='', next='3')
-//   limit=3&since=3  => D,E,F (prev='0', next='6')
-//   limit=3&since=6  => G     (prev='3', next='')
+//	 limit=3          => A,B,C (prev='', next='3')
+//	 limit=3&since=3  => D,E,F (prev='0', next='6')
+//	 limit=3&since=6  => G     (prev='3', next='')
 //
-//  A value of '-1' for prev/next indicates no position.
+//	A value of '-1' for prev/next indicates no position.
 func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
 	prev = -1
 	next = -1
diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go
index 2e864adef..85f1053f3 100644
--- a/clientapi/routing/sendevent.go
+++ b/clientapi/routing/sendevent.go
@@ -63,9 +63,10 @@ var sendEventDuration = prometheus.NewHistogramVec(
 )
 
 // SendEvent implements:
-//   /rooms/{roomID}/send/{eventType}
-//   /rooms/{roomID}/send/{eventType}/{txnID}
-//   /rooms/{roomID}/state/{eventType}/{stateKey}
+//
+//	/rooms/{roomID}/send/{eventType}
+//	/rooms/{roomID}/send/{eventType}/{txnID}
+//	/rooms/{roomID}/state/{eventType}/{stateKey}
 func SendEvent(
 	req *http.Request,
 	device *userapi.Device,
diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go
index 94b658ee3..4b7989ecb 100644
--- a/clientapi/routing/threepid.go
+++ b/clientapi/routing/threepid.go
@@ -38,8 +38,9 @@ type threePIDsResponse struct {
 }
 
 // RequestEmailToken implements:
-//     POST /account/3pid/email/requestToken
-//     POST /register/email/requestToken
+//
+//	POST /account/3pid/email/requestToken
+//	POST /register/email/requestToken
 func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go
index c7ddaabcf..f0f69ce3c 100644
--- a/clientapi/routing/voip.go
+++ b/clientapi/routing/voip.go
@@ -22,15 +22,17 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/util"
 )
 
 // RequestTurnServer implements:
-//     GET /voip/turnServer
+//
+//	GET /voip/turnServer
 func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse {
 	turnConfig := cfg.TURN
 
diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index 39843dccb..ea5f5e2bc 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -47,7 +47,7 @@ const HEAD = "HEAD"
 // We cannot use the dockerfile associated with the repo with each version sadly due to changes in
 // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
 // due to the error:
-//   When using COPY with more than one source file, the destination must be a directory and end with a /
+// When using COPY with more than one source file, the destination must be a directory and end with a /
 // We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
 const Dockerfile = `FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
@@ -95,7 +95,9 @@ CMD /build/run_dendrite.sh `
 const dendriteUpgradeTestLabel = "dendrite_upgrade_test"
 
 // downloadArchive downloads an arbitrary github archive of the form:
-//   https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
+//
+//	https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
+//
 // and re-tarballs it without the top-level directory which contains branch information. It inserts
 // the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that
 // you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile.
diff --git a/cmd/dendrite-upgrade-tests/tests.go b/cmd/dendrite-upgrade-tests/tests.go
index e02af92a9..ff1e09dda 100644
--- a/cmd/dendrite-upgrade-tests/tests.go
+++ b/cmd/dendrite-upgrade-tests/tests.go
@@ -18,9 +18,9 @@ type user struct {
 }
 
 // runTests performs the following operations:
-//  - register alice and bob with branch name muxed into the localpart
-//  - create a DM room for the 2 users and exchange messages
-//  - create/join a public #global room and exchange messages
+// - register alice and bob with branch name muxed into the localpart
+// - create a DM room for the 2 users and exchange messages
+// - create/join a public #global room and exchange messages
 func runTests(baseURL, branchName string) error {
 	// register 2 users
 	users := []user{
diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index e50ec66ad..2622ecb3f 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -208,9 +208,11 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 // joinedHostsAtEvent works out a list of matrix servers that were joined to
 // the room at the event (including peeking ones)
 // It is important to use the state at the event for sending messages because:
-//   1) We shouldn't send messages to servers that weren't in the room.
-//   2) If a server is kicked from the rooms it should still be told about the
-//      kick event,
+//
+//  1. We shouldn't send messages to servers that weren't in the room.
+//  2. If a server is kicked from the rooms it should still be told about the
+//     kick event.
+//
 // Usually the list can be calculated locally, but sometimes it will need fetch
 // events from the room server.
 // Returns an error if there was a problem talking to the room server.
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 30406a155..9ad3bd8eb 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -21,13 +21,14 @@ import (
 	"sort"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 // MakeJoin implements the /make_join API
@@ -435,13 +436,13 @@ func SendJoin(
 // a restricted room join. If the room version does not support restricted
 // joins then this function returns with no side effects. This returns three
 // values:
-// * an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
-//   should always be sent back to the client if one is specified
-// * a user ID of an authorising user, typically a user that has power to
-//   issue invites in the room, if one has been found
-// * an error if there was a problem finding out if this was allowable,
-//   like if the room version isn't known or a problem happened talking to
-//   the roomserver
+//   - an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
+//     should always be sent back to the client if one is specified
+//   - a user ID of an authorising user, typically a user that has power to
+//     issue invites in the room, if one has been found
+//   - an error if there was a problem finding out if this was allowable,
+//     like if the room version isn't known or a problem happened talking to
+//     the roomserver
 func checkRestrictedJoin(
 	httpReq *http.Request,
 	rsAPI api.FederationRoomserverAPI,
diff --git a/internal/log.go b/internal/log.go
index bba0ac6e6..a171555ab 100644
--- a/internal/log.go
+++ b/internal/log.go
@@ -27,9 +27,10 @@ import (
 
 	"github.com/matrix-org/util"
 
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dugong"
 	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 type utcFormatter struct {
@@ -145,7 +146,7 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str
 	})
 }
 
-//CloseAndLogIfError Closes io.Closer and logs the error if any
+// CloseAndLogIfError Closes io.Closer and logs the error if any
 func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) {
 	if closer == nil {
 		return
diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index acbcd5b8f..e317755e4 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -22,12 +22,13 @@ import (
 	"sync"
 	"time"
 
-	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+
+	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/keyserver/api"
 )
 
 var (
@@ -66,12 +67,14 @@ func init() {
 //   - We don't have unbounded growth in proportion to the number of servers (this is more important in a P2P world where
 //     we have many many servers)
 //   - We can adjust concurrency (at the cost of memory usage) by tuning N, to accommodate mobile devices vs servers.
+//
 // The downsides are that:
 //   - Query requests can get queued behind other servers if they hash to the same worker, even if there are other free
 //     workers elsewhere. Whilst suboptimal, provided we cap how long a single request can last (e.g using context timeouts)
 //     we guarantee we will get around to it. Also, more users on a given server does not increase the number of requests
 //     (as /keys/query allows multiple users to be specified) so being stuck behind matrix.org won't materially be any worse
 //     than being stuck behind foo.bar
+//
 // In the event that the query fails, a lock is acquired and the server name along with the time to wait before retrying is
 // set in a map. A restarter goroutine periodically probes this map and injects servers which are ready to be retried.
 type DeviceListUpdater struct {
diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go
index 0033a5086..e65342e2a 100644
--- a/keyserver/internal/device_list_update_test.go
+++ b/keyserver/internal/device_list_update_test.go
@@ -27,8 +27,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/keyserver/api"
 )
 
 var (
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index ecd4ecbb5..339c9796c 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -25,6 +25,11 @@ import (
 
 	"github.com/Arceliar/phony"
 	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
 	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/roomserver/acls"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -35,10 +40,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 )
 
 // Inputer is responsible for consuming from the roomserver input
@@ -60,9 +61,9 @@ import (
 // per-room durable consumers will only progress through the stream
 // as events are processed.
 //
-//       A BC *  -> positions of each consumer (* = ephemeral)
-//       ⌄ ⌄⌄ ⌄
-// ABAABCAABCAA  -> newest (letter = subject for each message)
+//	      A BC *  -> positions of each consumer (* = ephemeral)
+//	      ⌄ ⌄⌄ ⌄
+//	ABAABCAABCAA  -> newest (letter = subject for each message)
 //
 // In this example, A is still processing an event but has two
 // pending events to process afterwards. Both B and C are caught
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index f7d15fdb5..d6efad79d 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -20,32 +20,32 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/opentracing/opentracing-go"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/opentracing/opentracing-go"
-	"github.com/sirupsen/logrus"
 )
 
 // updateLatestEvents updates the list of latest events for this room in the database and writes the
 // event to the output log.
 // The latest events are the events that aren't referenced by another event in the database:
 //
-//     Time goes down the page. 1 is the m.room.create event (root).
-//
-//        1                 After storing 1 the latest events are {1}
-//        |                 After storing 2 the latest events are {2}
-//        2                 After storing 3 the latest events are {3}
-//       / \                After storing 4 the latest events are {3,4}
-//      3   4               After storing 5 the latest events are {5,4}
-//      |   |               After storing 6 the latest events are {5,6}
-//      5   6 <--- latest   After storing 7 the latest events are {6,7}
-//      |
-//      7 <----- latest
+//	Time goes down the page. 1 is the m.room.create event (root).
+//	        1                 After storing 1 the latest events are {1}
+//	        |                 After storing 2 the latest events are {2}
+//	        2                 After storing 3 the latest events are {3}
+//	       / \                After storing 4 the latest events are {3,4}
+//	      3   4               After storing 5 the latest events are {5,4}
+//	      |   |               After storing 6 the latest events are {5,6}
+//	      5   6 <--- latest   After storing 7 the latest events are {6,7}
+//	      |
+//	      7 <----- latest
 //
 // Can only be called once at a time
 func (r *Inputer) updateLatestEvents(
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 5b7ed22ee..298ba04f6 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -19,6 +19,10 @@ import (
 	"fmt"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -26,9 +30,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 // the max number of servers to backfill from per request. If this is too low we may fail to backfill when
@@ -522,8 +523,9 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion,
 }
 
 // joinEventsFromHistoryVisibility returns all CURRENTLY joined members if our server can read the room history
+//
 // TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just
-//       pull all events and then filter by that table.
+// pull all events and then filter by that table.
 func joinEventsFromHistoryVisibility(
 	ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry,
 	thisServer gomatrixserverlib.ServerName) ([]types.Event, error) {
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 58c43ac45..0bc389b80 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -5,9 +5,10 @@ import (
 	"database/sql"
 	"errors"
 
-	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/roomserver/types"
 )
 
 var OptimisationNotSupportedError = errors.New("optimisation not supported")
@@ -178,7 +179,7 @@ type StrippedEvent struct {
 }
 
 // ExtractContentValue from the given state event. For example, given an m.room.name event with:
-//    content: { name: "Foo" }
+// content: { name: "Foo" }
 // this returns "Foo".
 func ExtractContentValue(ev *gomatrixserverlib.HeaderedEvent) string {
 	content := ev.Content()
diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go
index 9044823af..edb1e77dd 100644
--- a/setup/mscs/msc2836/msc2836_test.go
+++ b/setup/mscs/msc2836/msc2836_test.go
@@ -15,6 +15,8 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/internal/hooks"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserver "github.com/matrix-org/dendrite/roomserver/api"
@@ -22,7 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/mscs/msc2836"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 var (
@@ -32,15 +33,17 @@ var (
 )
 
 // Basic sanity check of MSC2836 logic. Injects a thread that looks like:
-//   A
-//   |
-//   B
-//  / \
-// C   D
-//    /|\
-//   E F G
-//   |
-//   H
+//
+//	  A
+//	  |
+//	  B
+//	 / \
+//	C   D
+//	   /|\
+//	  E F G
+//	  |
+//	  H
+//
 // And makes sure POST /event_relationships works with various parameters
 func TestMSC2836(t *testing.T) {
 	alice := "@alice:localhost"
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 6bfc91edd..c7d8df740 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -6,12 +6,13 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 var (
@@ -364,13 +365,14 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) {
 
 // tests that joining/leaving the SAME room puts users in `left` if the final state is leave.
 // NB: Consider the case:
-// - Alice and Bob are in a room.
-// - Alice goes offline, Charlie joins, sends encrypted messages then leaves the room.
-// - Alice comes back online. Technically nothing has changed in the set of users between those two points in time,
-//   it's still just (Alice,Bob) but then we won't be tracking Charlie -- is this okay though? It's device keys
-//   which are only relevant when actively sending events I think? And if Alice does need the keys she knows
-//   charlie's (user_id, device_id) so can just hit /keys/query - no need to keep updated about it because she
-//   doesn't share any rooms with him.
+//   - Alice and Bob are in a room.
+//   - Alice goes offline, Charlie joins, sends encrypted messages then leaves the room.
+//   - Alice comes back online. Technically nothing has changed in the set of users between those two points in time,
+//     it's still just (Alice,Bob) but then we won't be tracking Charlie -- is this okay though? It's device keys
+//     which are only relevant when actively sending events I think? And if Alice does need the keys she knows
+//     charlie's (user_id, device_id) so can just hit /keys/query - no need to keep updated about it because she
+//     doesn't share any rooms with him.
+//
 // Ergo, we put them in `left` as it is simpler.
 func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) {
 	newShareUser := "@berta:localhost"
diff --git a/syncapi/routing/filter.go b/syncapi/routing/filter.go
index 1a10bd649..b41714dfc 100644
--- a/syncapi/routing/filter.go
+++ b/syncapi/routing/filter.go
@@ -19,13 +19,14 @@ import (
 	"io/ioutil"
 	"net/http"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/sync"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/tidwall/gjson"
 )
 
 // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId}
@@ -65,7 +66,9 @@ type filterResponse struct {
 	FilterID string `json:"filter_id"`
 }
 
-//PutFilter implements POST /_matrix/client/r0/user/{userId}/filter
+// PutFilter implements
+//
+//	POST /_matrix/client/r0/user/{userId}/filter
 func PutFilter(
 	req *http.Request, device *api.Device, syncDB storage.Database, userID string,
 ) util.JSONResponse {
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index d8fc8f508..d68351d4c 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -18,10 +18,11 @@ import (
 	"context"
 	"database/sql"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type AccountData interface {
@@ -122,12 +123,14 @@ type CurrentRoomState interface {
 //
 // We persist the previous event IDs as well, one per row, so when we do fetch even
 // earlier events we can simply delete rows which referenced it. Consider the graph:
-//        A
-//        |   Event C has 1 prev_event ID: A.
-//    B   C
-//    |___|   Event D has 2 prev_event IDs: B and C.
-//      |
-//      D
+//
+//	    A
+//	    |   Event C has 1 prev_event ID: A.
+//	B   C
+//	|___|   Event D has 2 prev_event IDs: B and C.
+//	  |
+//	  D
+//
 // The earliest known event we have is D, so this table has 2 rows.
 // A backfill request gives us C but not B. We delete rows where prev_event=C. This
 // still means that D is a backwards extremity as we do not have event B. However, event
diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go
index e00ed417b..8aa1746c5 100644
--- a/userapi/storage/sqlite3/stats_table.go
+++ b/userapi/storage/sqlite3/stats_table.go
@@ -20,13 +20,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 	"github.com/matrix-org/dendrite/userapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 )
 
 const userDailyVisitsSchema = `
@@ -297,11 +298,10 @@ func (s *statsStatements) monthlyUsers(ctx context.Context, txn *sql.Tx) (result
 	return
 }
 
-/* R30Users counts the number of 30 day retained users, defined as:
-- Users who have created their accounts more than 30 days ago
-- Where last seen at most 30 days ago
-- Where account creation and last_seen are > 30 days apart
-*/
+// R30Users counts the number of 30 day retained users, defined as:
+//   - Users who have created their accounts more than 30 days ago
+//   - Where last seen at most 30 days ago
+//   - Where account creation and last_seen are > 30 days apart
 func (s *statsStatements) r30Users(ctx context.Context, txn *sql.Tx) (map[string]int64, error) {
 	stmt := sqlutil.TxStmt(txn, s.countR30UsersStmt)
 	lastSeenAfter := time.Now().AddDate(0, 0, -30)
@@ -334,7 +334,8 @@ func (s *statsStatements) r30Users(ctx context.Context, txn *sql.Tx) (map[string
 	return result, rows.Err()
 }
 
-/* R30UsersV2 counts the number of 30 day retained users, defined as users that:
+/*
+R30UsersV2 counts the number of 30 day retained users, defined as users that:
 - Appear more than once in the past 60 days
 - Have more than 30 days between the most and least recent appearances that occurred in the past 60 days.
 */

From c8935fb53f122b367eda61ec7811c406193d29ba Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 5 Aug 2022 10:26:59 +0100
Subject: [PATCH 247/391] Do not use `ioutil` as it is deprecated (#2625)

---
 clientapi/auth/login.go                       |  3 +--
 clientapi/httputil/httputil.go                |  6 +++---
 clientapi/routing/account_data.go             |  6 +++---
 clientapi/routing/deactivate.go               |  4 ++--
 clientapi/routing/device.go                   |  4 ++--
 clientapi/routing/register.go                 |  6 +++---
 clientapi/routing/register_secret_test.go     |  4 ++--
 cmd/create-account/main.go                    |  5 ++---
 cmd/dendrite-demo-pinecone/main.go            |  5 ++---
 cmd/dendrite-demo-yggdrasil/yggconn/node.go   |  5 ++---
 cmd/dendrite-upgrade-tests/main.go            |  8 ++++----
 cmd/furl/main.go                              |  3 +--
 federationapi/federationapi_keys_test.go      |  6 +++---
 internal/log_unix.go                          |  4 ++--
 keyserver/internal/device_list_update_test.go |  6 +++---
 mediaapi/fileutils/fileutils.go               |  3 +--
 mediaapi/routing/download.go                  |  5 ++---
 setup/config/config.go                        | 10 +++++-----
 setup/config/config_appservice.go             |  4 ++--
 setup/mscs/msc2836/msc2836_test.go            |  6 +++---
 syncapi/routing/filter.go                     |  4 ++--
 test/keys.go                                  |  5 ++---
 22 files changed, 52 insertions(+), 60 deletions(-)

diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go
index 5f51c662a..5467e814d 100644
--- a/clientapi/auth/login.go
+++ b/clientapi/auth/login.go
@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -34,7 +33,7 @@ import (
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
 func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
-	reqBytes, err := ioutil.ReadAll(r)
+	reqBytes, err := io.ReadAll(r)
 	if err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,
diff --git a/clientapi/httputil/httputil.go b/clientapi/httputil/httputil.go
index b47701368..74f84f1e7 100644
--- a/clientapi/httputil/httputil.go
+++ b/clientapi/httputil/httputil.go
@@ -16,7 +16,7 @@ package httputil
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"unicode/utf8"
 
@@ -29,9 +29,9 @@ import (
 func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse {
 	// encoding/json allows invalid utf-8, matrix does not
 	// https://matrix.org/docs/spec/client_server/r0.6.1#api-standards
-	body, err := ioutil.ReadAll(req.Body)
+	body, err := io.ReadAll(req.Body)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
+		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
 		resp := jsonerror.InternalServerError()
 		return &resp
 	}
diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go
index 0d3a49495..b28f0bb1f 100644
--- a/clientapi/routing/account_data.go
+++ b/clientapi/routing/account_data.go
@@ -17,7 +17,7 @@ package routing
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -101,9 +101,9 @@ func SaveAccountData(
 		}
 	}
 
-	body, err := ioutil.ReadAll(req.Body)
+	body, err := io.ReadAll(req.Body)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
+		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
 		return jsonerror.InternalServerError()
 	}
 
diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go
index c8aa6a3bc..f213db7f3 100644
--- a/clientapi/routing/deactivate.go
+++ b/clientapi/routing/deactivate.go
@@ -1,7 +1,7 @@
 package routing
 
 import (
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
@@ -20,7 +20,7 @@ func Deactivate(
 ) util.JSONResponse {
 	ctx := req.Context()
 	defer req.Body.Close() // nolint:errcheck
-	bodyBytes, err := ioutil.ReadAll(req.Body)
+	bodyBytes, err := io.ReadAll(req.Body)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go
index bb1cf47bd..e3a02661c 100644
--- a/clientapi/routing/device.go
+++ b/clientapi/routing/device.go
@@ -15,7 +15,7 @@
 package routing
 
 import (
-	"io/ioutil"
+	"io"
 	"net"
 	"net/http"
 
@@ -175,7 +175,7 @@ func DeleteDeviceById(
 	}()
 	ctx := req.Context()
 	defer req.Body.Close() // nolint:errcheck
-	bodyBytes, err := ioutil.ReadAll(req.Body)
+	bodyBytes, err := io.ReadAll(req.Body)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go
index c4ac0f2e7..af0329a48 100644
--- a/clientapi/routing/register.go
+++ b/clientapi/routing/register.go
@@ -19,7 +19,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -371,7 +371,7 @@ func validateRecaptcha(
 
 	// Grab the body of the response from the captcha server
 	var r recaptchaResponse
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return &util.JSONResponse{
 			Code: http.StatusGatewayTimeout,
@@ -539,7 +539,7 @@ func Register(
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	defer req.Body.Close() // nolint: errcheck
-	reqBody, err := ioutil.ReadAll(req.Body)
+	reqBody, err := io.ReadAll(req.Body)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
diff --git a/clientapi/routing/register_secret_test.go b/clientapi/routing/register_secret_test.go
index e702b2152..a2ed35853 100644
--- a/clientapi/routing/register_secret_test.go
+++ b/clientapi/routing/register_secret_test.go
@@ -2,7 +2,7 @@ package routing
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"testing"
 
 	"github.com/patrickmn/go-cache"
@@ -13,7 +13,7 @@ func TestSharedSecretRegister(t *testing.T) {
 	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
 	sharedSecret := "dendritetest"
 
-	req, err := NewSharedSecretRegistrationRequest(ioutil.NopCloser(bytes.NewBuffer(jsonStr)))
+	req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))
 	if err != nil {
 		t.Fatalf("failed to read request: %s", err)
 	}
diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 7f6d5105e..92179a049 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -19,7 +19,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -157,7 +156,7 @@ func main() {
 func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string, error) {
 	// read password from file
 	if pwdFile != "" {
-		pw, err := ioutil.ReadFile(pwdFile)
+		pw, err := os.ReadFile(pwdFile)
 		if err != nil {
 			return "", fmt.Errorf("Unable to read password from file: %v", err)
 		}
@@ -166,7 +165,7 @@ func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string,
 
 	// read password from stdin
 	if pwdStdin {
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		if err != nil {
 			return "", fmt.Errorf("Unable to read password from stdin: %v", err)
 		}
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 8fa935ddf..75f29fe27 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -21,7 +21,6 @@ import (
 	"encoding/hex"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -76,11 +75,11 @@ func main() {
 		if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
 			panic(err)
 		}
-		if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil {
+		if err = os.WriteFile(keyfile, sk, 0644); err != nil {
 			panic(err)
 		}
 	} else if err == nil {
-		if sk, err = ioutil.ReadFile(keyfile); err != nil {
+		if sk, err = os.ReadFile(keyfile); err != nil {
 			panic(err)
 		}
 		if len(sk) != ed25519.PrivateKeySize {
diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
index d93272e2e..ff3c73ec8 100644
--- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
@@ -20,7 +20,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -69,7 +68,7 @@ func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
 
 	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
 	if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
-		yggconf, e := ioutil.ReadFile(yggfile)
+		yggconf, e := os.ReadFile(yggfile)
 		if e != nil {
 			panic(err)
 		}
@@ -88,7 +87,7 @@ func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
 	if err != nil {
 		panic(err)
 	}
-	if e := ioutil.WriteFile(yggfile, j, 0600); e != nil {
+	if e := os.WriteFile(yggfile, j, 0600); e != nil {
 		n.log.Printf("Couldn't write private key to file '%s': %s\n", yggfile, e)
 	}
 
diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go
index ea5f5e2bc..dce22472d 100644
--- a/cmd/dendrite-upgrade-tests/main.go
+++ b/cmd/dendrite-upgrade-tests/main.go
@@ -6,7 +6,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -128,7 +128,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 		return nil, err
 	}
 	// add top level Dockerfile
-	err = ioutil.WriteFile(path.Join(tmpDir, "Dockerfile"), dockerfile, os.ModePerm)
+	err = os.WriteFile(path.Join(tmpDir, "Dockerfile"), dockerfile, os.ModePerm)
 	if err != nil {
 		return nil, fmt.Errorf("failed to inject /Dockerfile: %w", err)
 	}
@@ -150,7 +150,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
 	if branchOrTagName == HEAD && *flagHead != "" {
 		log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
 		// add top level Dockerfile
-		err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
+		err = os.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
 		if err != nil {
 			return "", fmt.Errorf("custom HEAD: failed to inject /Dockerfile: %w", err)
 		}
@@ -388,7 +388,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
 		})
 		// ignore errors when cannot get logs, it's just for debugging anyways
 		if err == nil {
-			logbody, err := ioutil.ReadAll(logs)
+			logbody, err := io.ReadAll(logs)
 			if err == nil {
 				log.Printf("Container logs:\n\n%s\n\n", string(logbody))
 			}
diff --git a/cmd/furl/main.go b/cmd/furl/main.go
index 75e223388..f59f9c8ce 100644
--- a/cmd/furl/main.go
+++ b/cmd/furl/main.go
@@ -9,7 +9,6 @@ import (
 	"encoding/pem"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 
@@ -30,7 +29,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	data, err := ioutil.ReadFile(*requestKey)
+	data, err := os.ReadFile(*requestKey)
 	if err != nil {
 		panic(err)
 	}
diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go
index d1bfe1847..9c3446222 100644
--- a/federationapi/federationapi_keys_test.go
+++ b/federationapi/federationapi_keys_test.go
@@ -6,7 +6,7 @@ import (
 	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"os"
 	"testing"
@@ -66,7 +66,7 @@ func TestMain(m *testing.M) {
 			s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false)
 
 			// Create a temporary directory for JetStream.
-			d, err := ioutil.TempDir("./", "jetstream*")
+			d, err := os.MkdirTemp("./", "jetstream*")
 			if err != nil {
 				panic(err)
 			}
@@ -136,7 +136,7 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err
 	// And respond.
 	res = &http.Response{
 		StatusCode: 200,
-		Body:       ioutil.NopCloser(bytes.NewReader(body)),
+		Body:       io.NopCloser(bytes.NewReader(body)),
 	}
 	return
 }
diff --git a/internal/log_unix.go b/internal/log_unix.go
index 1e1094f23..75332af73 100644
--- a/internal/log_unix.go
+++ b/internal/log_unix.go
@@ -18,7 +18,7 @@
 package internal
 
 import (
-	"io/ioutil"
+	"io"
 	"log/syslog"
 
 	"github.com/MFAshby/stdemuxerhook"
@@ -63,7 +63,7 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
 		setupStdLogHook(logrus.InfoLevel)
 	}
 	// Hooks are now configured for stdout/err, so throw away the default logger output
-	logrus.SetOutput(ioutil.Discard)
+	logrus.SetOutput(io.Discard)
 }
 
 func checkSyslogHookParams(params map[string]interface{}) {
diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go
index e65342e2a..0c2405f34 100644
--- a/keyserver/internal/device_list_update_test.go
+++ b/keyserver/internal/device_list_update_test.go
@@ -18,7 +18,7 @@ import (
 	"context"
 	"crypto/ed25519"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -203,7 +203,7 @@ func TestUpdateNoPrevID(t *testing.T) {
 		}
 		return &http.Response{
 			StatusCode: 200,
-			Body: ioutil.NopCloser(strings.NewReader(`
+			Body: io.NopCloser(strings.NewReader(`
 			{
 				"user_id": "` + remoteUserID + `",
 				"stream_id": 5,
@@ -318,7 +318,7 @@ func TestDebounce(t *testing.T) {
 	// now send the response over federation
 	fedCh <- &http.Response{
 		StatusCode: 200,
-		Body: ioutil.NopCloser(strings.NewReader(`
+		Body: io.NopCloser(strings.NewReader(`
 		{
 			"user_id": "` + userID + `",
 			"stream_id": 5,
diff --git a/mediaapi/fileutils/fileutils.go b/mediaapi/fileutils/fileutils.go
index 754e4644b..2e719dc82 100644
--- a/mediaapi/fileutils/fileutils.go
+++ b/mediaapi/fileutils/fileutils.go
@@ -21,7 +21,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -180,7 +179,7 @@ func createTempDir(baseDirectory config.Path) (types.Path, error) {
 	if err := os.MkdirAll(baseTmpDir, 0770); err != nil {
 		return "", fmt.Errorf("failed to create base temp dir: %w", err)
 	}
-	tmpDir, err := ioutil.TempDir(baseTmpDir, "")
+	tmpDir, err := os.MkdirTemp(baseTmpDir, "")
 	if err != nil {
 		return "", fmt.Errorf("failed to create temp dir: %w", err)
 	}
diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go
index 10b25a5cd..c9299b1fc 100644
--- a/mediaapi/routing/download.go
+++ b/mediaapi/routing/download.go
@@ -19,7 +19,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
 	"net/url"
@@ -695,7 +694,7 @@ func (r *downloadRequest) GetContentLengthAndReader(contentLengthHeader string,
 
 		// We successfully parsed the Content-Length, so we'll return a limited
 		// reader that restricts us to reading only up to this size.
-		reader = ioutil.NopCloser(io.LimitReader(*body, parsedLength))
+		reader = io.NopCloser(io.LimitReader(*body, parsedLength))
 		contentLength = parsedLength
 	} else {
 		// Content-Length header is missing. If we have a maximum file size
@@ -704,7 +703,7 @@ func (r *downloadRequest) GetContentLengthAndReader(contentLengthHeader string,
 		// ultimately it will get rewritten later when the temp file is written
 		// to disk.
 		if maxFileSizeBytes > 0 {
-			reader = ioutil.NopCloser(io.LimitReader(*body, int64(maxFileSizeBytes)))
+			reader = io.NopCloser(io.LimitReader(*body, int64(maxFileSizeBytes)))
 		}
 		contentLength = 0
 	}
diff --git a/setup/config/config.go b/setup/config/config.go
index 9b9000a62..924b51f22 100644
--- a/setup/config/config.go
+++ b/setup/config/config.go
@@ -19,8 +19,8 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -191,7 +191,7 @@ type ConfigErrors []string
 // Load a yaml config file for a server run as multiple processes or as a monolith.
 // Checks the config to ensure that it is valid.
 func Load(configPath string, monolith bool) (*Dendrite, error) {
-	configData, err := ioutil.ReadFile(configPath)
+	configData, err := os.ReadFile(configPath)
 	if err != nil {
 		return nil, err
 	}
@@ -199,9 +199,9 @@ func Load(configPath string, monolith bool) (*Dendrite, error) {
 	if err != nil {
 		return nil, err
 	}
-	// Pass the current working directory and ioutil.ReadFile so that they can
+	// Pass the current working directory and os.ReadFile so that they can
 	// be mocked in the tests
-	return loadConfig(basePath, configData, ioutil.ReadFile, monolith)
+	return loadConfig(basePath, configData, os.ReadFile, monolith)
 }
 
 func loadConfig(
@@ -530,7 +530,7 @@ func (config *Dendrite) KeyServerURL() string {
 // SetupTracing configures the opentracing using the supplied configuration.
 func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) {
 	if !config.Tracing.Enabled {
-		return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		return io.NopCloser(bytes.NewReader([]byte{})), nil
 	}
 	return config.Tracing.Jaeger.InitGlobalTracer(
 		serviceName,
diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go
index 9b89fc9af..b8f99a612 100644
--- a/setup/config/config_appservice.go
+++ b/setup/config/config_appservice.go
@@ -16,7 +16,7 @@ package config
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -181,7 +181,7 @@ func loadAppServices(config *AppServiceAPI, derived *Derived) error {
 		}
 
 		// Read the application service's config file
-		configData, err := ioutil.ReadFile(absPath)
+		configData, err := os.ReadFile(absPath)
 		if err != nil {
 			return err
 		}
diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go
index edb1e77dd..eeded4275 100644
--- a/setup/mscs/msc2836/msc2836_test.go
+++ b/setup/mscs/msc2836/msc2836_test.go
@@ -7,7 +7,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"sort"
 	"strings"
@@ -428,12 +428,12 @@ func postRelationships(t *testing.T, expectCode int, accessToken string, req *ms
 		t.Fatalf("failed to do request: %s", err)
 	}
 	if res.StatusCode != expectCode {
-		body, _ := ioutil.ReadAll(res.Body)
+		body, _ := io.ReadAll(res.Body)
 		t.Fatalf("wrong response code, got %d want %d - body: %s", res.StatusCode, expectCode, string(body))
 	}
 	if res.StatusCode == 200 {
 		var result msc2836.EventRelationshipResponse
-		body, err := ioutil.ReadAll(res.Body)
+		body, err := io.ReadAll(res.Body)
 		if err != nil {
 			t.Fatalf("response 200 OK but failed to read response body: %s", err)
 		}
diff --git a/syncapi/routing/filter.go b/syncapi/routing/filter.go
index b41714dfc..f5acdbde3 100644
--- a/syncapi/routing/filter.go
+++ b/syncapi/routing/filter.go
@@ -16,7 +16,7 @@ package routing
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -88,7 +88,7 @@ func PutFilter(
 	var filter gomatrixserverlib.Filter
 
 	defer req.Body.Close() // nolint:errcheck
-	body, err := ioutil.ReadAll(req.Body)
+	body, err := io.ReadAll(req.Body)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
diff --git a/test/keys.go b/test/keys.go
index 75e3800e0..327c6ed7b 100644
--- a/test/keys.go
+++ b/test/keys.go
@@ -22,7 +22,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"os"
 	"strings"
@@ -144,7 +143,7 @@ func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPat
 	}
 
 	// load the authority key
-	dat, err := ioutil.ReadFile(authorityKeyPath)
+	dat, err := os.ReadFile(authorityKeyPath)
 	if err != nil {
 		return err
 	}
@@ -158,7 +157,7 @@ func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPat
 	}
 
 	// load the authority certificate
-	dat, err = ioutil.ReadFile(authorityCertPath)
+	dat, err = os.ReadFile(authorityCertPath)
 	if err != nil {
 		return err
 	}

From 2a1df0129e43e09421b32b0961225dab61e31703 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 5 Aug 2022 10:44:08 +0100
Subject: [PATCH 248/391] Update readme

---
 README.md | 59 ++++++++-----------------------------------------------
 1 file changed, 8 insertions(+), 51 deletions(-)

diff --git a/README.md b/README.md
index 8f54db7b7..10e2b1b86 100644
--- a/README.md
+++ b/README.md
@@ -21,8 +21,7 @@ As of October 2020 (current [progress below](#progress)), Dendrite has now enter
 This does not mean:
 
 - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
-- All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular,
-  presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
+- Dendrite is feature-complete. There may be client or federation APIs that are not implemented.
 - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
@@ -36,6 +35,9 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 
 ## Requirements
 
+See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
+more information on requirements.
+
 To build Dendrite, you will need Go 1.18 or later.
 
 For a usable federating Dendrite deployment, you will also need:
@@ -83,11 +85,11 @@ $ ./bin/create-account --config dendrite.yaml -username alice
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
 
-## <a id="progress"></a> Progress
+## Progress
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of April 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of August 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well, although there are still some missing features (like Search).
 
@@ -119,53 +121,8 @@ We would be grateful for any help on issues marked as
 all have related Sytests which need to pass in order for the issue to be closed. Once you've written your
 code, you can quickly run Sytest to ensure that the test names are now passing.
 
-For example, if the test `Local device key changes get to remote servers` was marked as failing, find the
-test file (e.g via `grep` or via the
-[CI log output](https://buildkite.com/matrix-dot-org/dendrite/builds/2826#39cff5de-e032-4ad0-ad26-f819e6919c42)
-it's `tests/50federation/40devicelists.pl` ) then to run Sytest:
-
-```
-docker run --rm --name sytest
--v "/Users/kegan/github/sytest:/sytest"
--v "/Users/kegan/github/dendrite:/src"
--v "/Users/kegan/logs:/logs"
--v "/Users/kegan/go/:/gopath"
--e "POSTGRES=1" -e "DENDRITE_TRACE_HTTP=1"
-matrixdotorg/sytest-dendrite:latest tests/50federation/40devicelists.pl
-```
-
-See [sytest.md](docs/sytest.md) for the full description of these flags.
-
-You can try running sytest outside of docker for faster runs, but the dependencies can be temperamental
-and we recommend using docker where possible.
-
-```
-cd sytest
-export PERL5LIB=$HOME/lib/perl5
-export PERL_MB_OPT=--install_base=$HOME
-export PERL_MM_OPT=INSTALL_BASE=$HOME
-./install-deps.pl
-
-./run-tests.pl -I Dendrite::Monolith -d $PATH_TO_DENDRITE_BINARIES
-```
-
-Sometimes Sytest is testing the wrong thing or is flakey, so it will need to be patched.
-Ask on `#dendrite-dev:matrix.org` if you think this is the case for you and we'll be happy to help.
-
-If you're new to the project, see [CONTRIBUTING.md](docs/CONTRIBUTING.md) to get up to speed then
+If you're new to the project, see our
+[Contributing page](https://matrix-org.github.io/dendrite/development/contributing) to get up to speed, then
 look for [Good First Issues](https://github.com/matrix-org/dendrite/labels/good%20first%20issue). If you're
 familiar with the project, look for [Help Wanted](https://github.com/matrix-org/dendrite/labels/help-wanted)
 issues.
-
-## Hardware requirements
-
-Dendrite in Monolith + SQLite works in a range of environments including iOS and in-browser via WASM.
-
-For small homeserver installations joined on ~10s rooms on matrix.org with ~100s of users in those rooms, including some
-encrypted rooms:
-
-- Memory: uses around 100MB of RAM, with peaks at around 200MB.
-- Disk space: After a few months of usage, the database grew to around 2GB (in Monolith mode).
-- CPU: Brief spikes when processing events, typically idles at 1% CPU.
-
-This means Dendrite should comfortably work on things like Raspberry Pis.

From cecd11be9a44ff93bd846fdd4b40a4dcd41f9f70 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 5 Aug 2022 13:44:20 +0200
Subject: [PATCH 249/391] Partly fix notification counts (#2621)

* Fix notification query

* Also for SQLite

* Move tests to whitelist

* Revert "Move tests to whitelist"

This reverts commit a7d0120019a111ce45a447ba40233d9c101e6e9b.
---
 .../storage/postgres/notification_data_table.go  |  2 +-
 .../storage/sqlite3/notification_data_table.go   | 16 +++++++++++-----
 syncapi/storage/sqlite3/stream_id_table.go       |  8 ++++++++
 syncapi/storage/sqlite3/syncserver.go            |  2 +-
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/syncapi/storage/postgres/notification_data_table.go b/syncapi/storage/postgres/notification_data_table.go
index f3fc4451f..9cd8b7362 100644
--- a/syncapi/storage/postgres/notification_data_table.go
+++ b/syncapi/storage/postgres/notification_data_table.go
@@ -58,7 +58,7 @@ const upsertRoomUnreadNotificationCountsSQL = `INSERT INTO syncapi_notification_
   (user_id, room_id, notification_count, highlight_count)
   VALUES ($1, $2, $3, $4)
   ON CONFLICT (user_id, room_id)
-  DO UPDATE SET notification_count = $3, highlight_count = $4
+  DO UPDATE SET id = nextval('syncapi_notification_data_id_seq'), notification_count = $3, highlight_count = $4
   RETURNING id`
 
 const selectUserUnreadNotificationCountsSQL = `SELECT
diff --git a/syncapi/storage/sqlite3/notification_data_table.go b/syncapi/storage/sqlite3/notification_data_table.go
index 4b3f074db..eaa11a8c0 100644
--- a/syncapi/storage/sqlite3/notification_data_table.go
+++ b/syncapi/storage/sqlite3/notification_data_table.go
@@ -25,12 +25,14 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
-func NewSqliteNotificationDataTable(db *sql.DB) (tables.NotificationData, error) {
+func NewSqliteNotificationDataTable(db *sql.DB, streamID *StreamIDStatements) (tables.NotificationData, error) {
 	_, err := db.Exec(notificationDataSchema)
 	if err != nil {
 		return nil, err
 	}
-	r := &notificationDataStatements{}
+	r := &notificationDataStatements{
+		streamIDStatements: streamID,
+	}
 	return r, sqlutil.StatementList{
 		{&r.upsertRoomUnreadCounts, upsertRoomUnreadNotificationCountsSQL},
 		{&r.selectUserUnreadCounts, selectUserUnreadNotificationCountsSQL},
@@ -39,6 +41,7 @@ func NewSqliteNotificationDataTable(db *sql.DB) (tables.NotificationData, error)
 }
 
 type notificationDataStatements struct {
+	streamIDStatements     *StreamIDStatements
 	upsertRoomUnreadCounts *sql.Stmt
 	selectUserUnreadCounts *sql.Stmt
 	selectMaxID            *sql.Stmt
@@ -58,8 +61,7 @@ const upsertRoomUnreadNotificationCountsSQL = `INSERT INTO syncapi_notification_
   (user_id, room_id, notification_count, highlight_count)
   VALUES ($1, $2, $3, $4)
   ON CONFLICT (user_id, room_id)
-  DO UPDATE SET notification_count = $3, highlight_count = $4
-  RETURNING id`
+  DO UPDATE SET id = $5, notification_count = $6, highlight_count = $7`
 
 const selectUserUnreadNotificationCountsSQL = `SELECT
   id, room_id, notification_count, highlight_count
@@ -71,7 +73,11 @@ const selectUserUnreadNotificationCountsSQL = `SELECT
 const selectMaxNotificationIDSQL = `SELECT CASE COUNT(*) WHEN 0 THEN 0 ELSE MAX(id) END FROM syncapi_notification_data`
 
 func (r *notificationDataStatements) UpsertRoomUnreadCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
-	err = r.upsertRoomUnreadCounts.QueryRowContext(ctx, userID, roomID, notificationCount, highlightCount).Scan(&pos)
+	pos, err = r.streamIDStatements.nextNotificationID(ctx, nil)
+	if err != nil {
+		return
+	}
+	_, err = r.upsertRoomUnreadCounts.ExecContext(ctx, userID, roomID, notificationCount, highlightCount, pos, notificationCount, highlightCount)
 	return
 }
 
diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go
index 71980b806..1160a437e 100644
--- a/syncapi/storage/sqlite3/stream_id_table.go
+++ b/syncapi/storage/sqlite3/stream_id_table.go
@@ -26,6 +26,8 @@ INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("invite", 0)
   ON CONFLICT DO NOTHING;
 INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("presence", 0)
   ON CONFLICT DO NOTHING;
+INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("notification", 0)
+  ON CONFLICT DO NOTHING;
 `
 
 const increaseStreamIDStmt = "" +
@@ -78,3 +80,9 @@ func (s *StreamIDStatements) nextPresenceID(ctx context.Context, txn *sql.Tx) (p
 	err = increaseStmt.QueryRowContext(ctx, "presence").Scan(&pos)
 	return
 }
+
+func (s *StreamIDStatements) nextNotificationID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) {
+	increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt)
+	err = increaseStmt.QueryRowContext(ctx, "notification").Scan(&pos)
+	return
+}
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index 65b2bb38a..5c5eb0f55 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -95,7 +95,7 @@ func (d *SyncServerDatasource) prepare() (err error) {
 	if err != nil {
 		return err
 	}
-	notificationData, err := NewSqliteNotificationDataTable(d.db)
+	notificationData, err := NewSqliteNotificationDataTable(d.db, &d.streamID)
 	if err != nil {
 		return err
 	}

From 3efc646f2567ced52a3e9980490c463866691414 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 5 Aug 2022 13:16:54 +0100
Subject: [PATCH 250/391] Update TURN config in sample config files

---
 dendrite-sample.monolith.yaml | 9 ++++++---
 dendrite-sample.polylith.yaml | 9 ++++++---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index 816c4cae9..f753c3d9b 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -178,13 +178,16 @@ client_api:
 
   # TURN server information that this homeserver should send to clients.
   turn:
-    turn_user_lifetime: ""
+    turn_user_lifetime: "5m"
     turn_uris:
     #  - turn:turn.server.org?transport=udp
     #  - turn:turn.server.org?transport=tcp
     turn_shared_secret: ""
-    turn_username: ""
-    turn_password: ""
+    # If your TURN server requires static credentials, then you will need to enter
+    # them here instead of supplying a shared secret. Note that these credentials
+    # will be visible to clients!
+    # turn_username: ""
+    # turn_password: ""
 
   # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
   # number of "slots" have been taken by requests from a specific host. Each "slot"
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 4784dbaff..856b4ab22 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -181,13 +181,16 @@ client_api:
 
   # TURN server information that this homeserver should send to clients.
   turn:
-    turn_user_lifetime: ""
+    turn_user_lifetime: "5m"
     turn_uris:
     #  - turn:turn.server.org?transport=udp
     #  - turn:turn.server.org?transport=tcp
     turn_shared_secret: ""
-    turn_username: ""
-    turn_password: ""
+    # If your TURN server requires static credentials, then you will need to enter
+    # them here instead of supplying a shared secret. Note that these credentials
+    # will be visible to clients!
+    # turn_username: ""
+    # turn_password: ""
 
   # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
   # number of "slots" have been taken by requests from a specific host. Each "slot"

From 3a156a434ac193781774c7acfd6bf1c594028f74 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 5 Aug 2022 14:27:27 +0200
Subject: [PATCH 251/391] Invalidate lazyLoadCache if we're doing an initial
 sync (#2623)

* Bypass lazyLoadCache if we're doing an initial sync

* Make the linter happy again?

* Revert "Make the linter happy again?"

This reverts commit 52a5691ba3c17c05698bcc6a13092090f27ace63.

* Try that again

* Invalidate LazyLoadCache on initial syncs

* Remove unneeded check

* Add TODO

* Rename Invalite -> InvalidateLazyLoadedUser

* Thanks IDE
---
 internal/caching/cache_lazy_load_members.go | 10 ++++++++++
 syncapi/routing/messages.go                 |  9 +++++----
 syncapi/streams/stream_pdu.go               | 13 +++++++++++++
 syncapi/streams/streams.go                  |  1 +
 4 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/internal/caching/cache_lazy_load_members.go b/internal/caching/cache_lazy_load_members.go
index 0d7009c94..390334da7 100644
--- a/internal/caching/cache_lazy_load_members.go
+++ b/internal/caching/cache_lazy_load_members.go
@@ -14,6 +14,7 @@ type lazyLoadingCacheKey struct {
 type LazyLoadCache interface {
 	StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string)
 	IsLazyLoadedUserCached(device *userapi.Device, roomID, userID string) (string, bool)
+	InvalidateLazyLoadedUser(device *userapi.Device, roomID, userID string)
 }
 
 func (c Caches) StoreLazyLoadedUser(device *userapi.Device, roomID, userID, eventID string) {
@@ -33,3 +34,12 @@ func (c Caches) IsLazyLoadedUserCached(device *userapi.Device, roomID, userID st
 		TargetUserID: userID,
 	})
 }
+
+func (c Caches) InvalidateLazyLoadedUser(device *userapi.Device, roomID, userID string) {
+	c.LazyLoading.Unset(lazyLoadingCacheKey{
+		UserID:       device.UserID,
+		DeviceID:     device.ID,
+		RoomID:       roomID,
+		TargetUserID: userID,
+	})
+}
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 990ca55b1..b4c9a5423 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -20,6 +20,10 @@ import (
 	"net/http"
 	"sort"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -28,9 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/sync"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 type messagesReq struct {
@@ -262,7 +263,7 @@ func (m *messagesResp) applyLazyLoadMembers(
 		}
 	}
 	for _, evt := range membershipToUser {
-		m.State = append(m.State, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatSync))
+		m.State = append(m.State, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatAll))
 	}
 }
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 1003208fd..1ad3adc4c 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -12,9 +12,12 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/tidwall/gjson"
 	"go.uber.org/atomic"
+
+	"github.com/matrix-org/dendrite/syncapi/notifier"
 )
 
 // The max number of per-room goroutines to have running.
@@ -34,6 +37,7 @@ type PDUStreamProvider struct {
 	// userID+deviceID -> lazy loading cache
 	lazyLoadCache caching.LazyLoadCache
 	rsAPI         roomserverAPI.SyncRoomserverAPI
+	notifier      *notifier.Notifier
 }
 
 func (p *PDUStreamProvider) worker() {
@@ -100,6 +104,15 @@ func (p *PDUStreamProvider) CompleteSync(
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
 	}
 
+	// Invalidate the lazyLoadCache, otherwise we end up with missing displaynames/avatars
+	// TODO: This might be inefficient, when joined to many and/or large rooms.
+	for _, roomID := range joinedRoomIDs {
+		joinedUsers := p.notifier.JoinedUsers(roomID)
+		for _, sharedUser := range joinedUsers {
+			p.lazyLoadCache.InvalidateLazyLoadedUser(req.Device, roomID, sharedUser)
+		}
+	}
+
 	// Build up a /sync response. Add joined rooms.
 	var reqMutex sync.Mutex
 	var reqWaitGroup sync.WaitGroup
diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go
index 1ca4ee8c3..dbc053bd8 100644
--- a/syncapi/streams/streams.go
+++ b/syncapi/streams/streams.go
@@ -34,6 +34,7 @@ func NewSyncStreamProviders(
 			StreamProvider: StreamProvider{DB: d},
 			lazyLoadCache:  lazyLoadCache,
 			rsAPI:          rsAPI,
+			notifier:       notifier,
 		},
 		TypingStreamProvider: &TypingStreamProvider{
 			StreamProvider: StreamProvider{DB: d},

From 10a151cb55ba925c3ece2026c65cb8e57207bf46 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 5 Aug 2022 15:37:13 +0200
Subject: [PATCH 252/391] Don't panic if we fail to upsert account data

---
 syncapi/consumers/clientapi.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go
index eec369c1a..02633b567 100644
--- a/syncapi/consumers/clientapi.go
+++ b/syncapi/consumers/clientapi.go
@@ -21,6 +21,11 @@ import (
 	"fmt"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -29,10 +34,6 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/producers"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/sirupsen/logrus"
-	log "github.com/sirupsen/logrus"
 )
 
 // OutputClientDataConsumer consumes events that originated in the client API server.
@@ -107,7 +108,8 @@ func (s *OutputClientDataConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 			"type":       output.Type,
 			"room_id":    output.RoomID,
 			log.ErrorKey: err,
-		}).Panicf("could not save account data")
+		}).Errorf("could not save account data")
+		return false
 	}
 
 	if err = s.sendReadUpdate(ctx, userID, output); err != nil {

From 03ddd98f5e6582f5aacaee643ddd53506ccfec50 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 8 Aug 2022 10:18:57 +0200
Subject: [PATCH 253/391] Fix issues with migrations not getting executed
 (#2628)

* Fix issues with migrations not getting executed

* Check actual postgres error

* Return error if it's not "column does not exist"
---
 .../storage/postgres/key_changes_table.go     | 15 ++++++++++++++-
 .../storage/sqlite3/key_changes_table.go      |  3 ++-
 roomserver/storage/postgres/storage.go        | 19 ++++++++++++++++---
 roomserver/storage/sqlite3/storage.go         | 10 ++++++----
 4 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/keyserver/storage/postgres/key_changes_table.go b/keyserver/storage/postgres/key_changes_table.go
index 6894d7b7c..004f15d82 100644
--- a/keyserver/storage/postgres/key_changes_table.go
+++ b/keyserver/storage/postgres/key_changes_table.go
@@ -18,6 +18,8 @@ import (
 	"context"
 	"database/sql"
 
+	"github.com/lib/pq"
+
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/storage/postgres/deltas"
@@ -64,7 +66,8 @@ func NewPostgresKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) {
 	// TODO: Remove when we are sure we are not having goose artefacts in the db
 	// This forces an error, which indicates the migration is already applied, since the
 	// column partition was removed from the table
-	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan()
+	var count int
+	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan(&count)
 	if err == nil {
 		m := sqlutil.NewMigrator(db)
 		m.AddMigrations(sqlutil.Migration{
@@ -72,6 +75,16 @@ func NewPostgresKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) {
 			Up:      deltas.UpRefactorKeyChanges,
 		})
 		return s, m.Up(context.Background())
+	} else {
+		switch e := err.(type) {
+		case *pq.Error:
+			// ignore undefined_column (42703) errors, as this is expected at this point
+			if e.Code != "42703" {
+				return nil, err
+			}
+		default:
+			return nil, err
+		}
 	}
 	return s, nil
 }
diff --git a/keyserver/storage/sqlite3/key_changes_table.go b/keyserver/storage/sqlite3/key_changes_table.go
index 1b27c3d05..217fa7a5d 100644
--- a/keyserver/storage/sqlite3/key_changes_table.go
+++ b/keyserver/storage/sqlite3/key_changes_table.go
@@ -61,7 +61,8 @@ func NewSqliteKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) {
 	// TODO: Remove when we are sure we are not having goose artefacts in the db
 	// This forces an error, which indicates the migration is already applied, since the
 	// column partition was removed from the table
-	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan()
+	var count int
+	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan(&count)
 	if err == nil {
 		m := sqlutil.NewMigrator(db)
 		m.AddMigrations(sqlutil.Migration{
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index 4c271ea9b..f47a64c80 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -19,8 +19,10 @@ import (
 	"database/sql"
 	"fmt"
 
+	"github.com/lib/pq"
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
+
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas"
@@ -53,21 +55,32 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 	// TODO: Remove when we are sure we are not having goose artefacts in the db
 	// This forces an error, which indicates the migration is already applied, since the
 	// column event_nid was removed from the table
-	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan()
+	var eventNID int
+	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan(&eventNID)
 	if err == nil {
 		m := sqlutil.NewMigrator(db)
 		m.AddMigrations(sqlutil.Migration{
 			Version: "roomserver: state blocks refactor",
 			Up:      deltas.UpStateBlocksRefactor,
 		})
-		if err := m.Up(base.Context()); err != nil {
+		if err = m.Up(base.Context()); err != nil {
+			return nil, err
+		}
+	} else {
+		switch e := err.(type) {
+		case *pq.Error:
+			// ignore undefined_column (42703) errors, as this is expected at this point
+			if e.Code != "42703" {
+				return nil, err
+			}
+		default:
 			return nil, err
 		}
 	}
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
 	// to in the database code should now exist.
-	if err := d.prepare(db, writer, cache); err != nil {
+	if err = d.prepare(db, writer, cache); err != nil {
 		return nil, err
 	}
 
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index bb9c15b5a..9f8a1b118 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -20,6 +20,8 @@ import (
 	"database/sql"
 	"fmt"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
@@ -27,7 +29,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // A Database is used to store room events and stream offsets.
@@ -63,21 +64,22 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 	// TODO: Remove when we are sure we are not having goose artefacts in the db
 	// This forces an error, which indicates the migration is already applied, since the
 	// column event_nid was removed from the table
-	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan()
+	var eventNID int
+	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan(&eventNID)
 	if err == nil {
 		m := sqlutil.NewMigrator(db)
 		m.AddMigrations(sqlutil.Migration{
 			Version: "roomserver: state blocks refactor",
 			Up:      deltas.UpStateBlocksRefactor,
 		})
-		if err := m.Up(base.Context()); err != nil {
+		if err = m.Up(base.Context()); err != nil {
 			return nil, err
 		}
 	}
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
 	// to in the database code should now exist.
-	if err := d.prepare(db, writer, cache); err != nil {
+	if err = d.prepare(db, writer, cache); err != nil {
 		return nil, err
 	}
 

From e930959e4911453ec51bba0b1469f4b15c594f32 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 9 Aug 2022 10:40:46 +0200
Subject: [PATCH 254/391] Send-to-device/sync tweaks (#2630)

* Always delete send to device messages

* Omit empty to_device

* Tweak /sync response to omit empty values
---
 syncapi/streams/stream_sendtodevice.go | 20 ++++++--------------
 syncapi/sync/requestpool.go            | 15 +++++++++++----
 syncapi/types/types.go                 | 23 ++++++++++++-----------
 3 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go
index 6a18df506..31c6187cb 100644
--- a/syncapi/streams/stream_sendtodevice.go
+++ b/syncapi/streams/stream_sendtodevice.go
@@ -39,21 +39,13 @@ func (p *SendToDeviceStreamProvider) IncrementalSync(
 		return from
 	}
 
-	if len(events) > 0 {
-		// Clean up old send-to-device messages from before this stream position.
-		if err := p.DB.CleanSendToDeviceUpdates(req.Context, req.Device.UserID, req.Device.ID, from); err != nil {
-			req.Log.WithError(err).Error("p.DB.CleanSendToDeviceUpdates failed")
-			return from
-		}
-
-		// Add the updates into the sync response.
-		for _, event := range events {
-			// skip ignored user events
-			if _, ok := req.IgnoredUsers.List[event.Sender]; ok {
-				continue
-			}
-			req.Response.ToDevice.Events = append(req.Response.ToDevice.Events, event.SendToDeviceEvent)
+	// Add the updates into the sync response.
+	for _, event := range events {
+		// skip ignored user events
+		if _, ok := req.IgnoredUsers.List[event.Sender]; ok {
+			continue
 		}
+		req.Response.ToDevice.Events = append(req.Response.ToDevice.Events, event.SendToDeviceEvent)
 	}
 
 	return lastPos
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index b6b4779a5..d908a9629 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -25,6 +25,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@@ -35,10 +40,6 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 )
 
 // RequestPool manages HTTP long-poll connections for /sync
@@ -251,6 +252,12 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 	waitingSyncRequests.Inc()
 	defer waitingSyncRequests.Dec()
 
+	// Clean up old send-to-device messages from before this stream position.
+	// This is needed to avoid sending the same message multiple times
+	if err = rp.db.CleanSendToDeviceUpdates(syncReq.Context, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Since.SendToDevicePosition); err != nil {
+		syncReq.Log.WithError(err).Error("p.DB.CleanSendToDeviceUpdates failed")
+	}
+
 	// loop until we get some data
 	for {
 		startTime := time.Now()
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index 159fa08b6..39b085d9c 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -21,9 +21,10 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
 )
 
 var (
@@ -330,23 +331,23 @@ type Response struct {
 	NextBatch   StreamingToken `json:"next_batch"`
 	AccountData struct {
 		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"account_data"`
+	} `json:"account_data,omitempty"`
 	Presence struct {
 		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"presence"`
+	} `json:"presence,omitempty"`
 	Rooms struct {
-		Join   map[string]JoinResponse   `json:"join"`
-		Peek   map[string]JoinResponse   `json:"peek"`
-		Invite map[string]InviteResponse `json:"invite"`
-		Leave  map[string]LeaveResponse  `json:"leave"`
-	} `json:"rooms"`
+		Join   map[string]JoinResponse   `json:"join,omitempty"`
+		Peek   map[string]JoinResponse   `json:"peek,omitempty"`
+		Invite map[string]InviteResponse `json:"invite,omitempty"`
+		Leave  map[string]LeaveResponse  `json:"leave,omitempty"`
+	} `json:"rooms,omitempty"`
 	ToDevice struct {
-		Events []gomatrixserverlib.SendToDeviceEvent `json:"events"`
-	} `json:"to_device"`
+		Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"`
+	} `json:"to_device,omitempty"`
 	DeviceLists struct {
 		Changed []string `json:"changed,omitempty"`
 		Left    []string `json:"left,omitempty"`
-	} `json:"device_lists"`
+	} `json:"device_lists,omitempty"`
 	DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
 }
 

From 240ae257deb74b7be8a17500b77d5e1bca56e8f5 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 9 Aug 2022 11:15:58 +0200
Subject: [PATCH 255/391] Add housekeeping function to delete old/expired EDUs
 (#2399)

* Add housekeeping function to delete old/expired EDUs

* Add migrations

* Evict EDUs from cache

* Fix queries

* Fix upgrade

* Use map[string]time.Duration to specify different expiry times

* Fix copy & paste mistake

* Set expires_at to tomorrow

* Don't allow NULL

* Add comment

* Add tests

* Use new testrig package

* Fix migrations

* Never expire m.direct_to_device

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
Co-authored-by: kegsay <kegan@matrix.org>
---
 federationapi/federationapi.go                |  13 +++
 federationapi/queue/destinationqueue.go       |   1 +
 federationapi/storage/interface.go            |   5 +-
 .../deltas/2022042812473400_addexpiresat.go   |  44 ++++++++
 .../storage/postgres/queue_edus_table.go      |  99 ++++++++++++-----
 federationapi/storage/postgres/storage.go     |   3 +
 federationapi/storage/shared/storage_edus.go  |  49 +++++++++
 .../deltas/2022042812473400_addexpiresat.go   |  68 ++++++++++++
 .../storage/sqlite3/queue_edus_table.go       | 100 ++++++++++++++----
 federationapi/storage/sqlite3/storage.go      |   3 +
 federationapi/storage/storage_test.go         |  81 ++++++++++++++
 federationapi/storage/tables/interface.go     |   5 +-
 12 files changed, 422 insertions(+), 49 deletions(-)
 create mode 100644 federationapi/storage/postgres/deltas/2022042812473400_addexpiresat.go
 create mode 100644 federationapi/storage/sqlite3/deltas/2022042812473400_addexpiresat.go
 create mode 100644 federationapi/storage/storage_test.go

diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index 97bcc12a5..ff01b1952 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -15,6 +15,8 @@
 package federationapi
 
 import (
+	"time"
+
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
@@ -167,5 +169,16 @@ func NewInternalAPI(
 	if err = presenceConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start presence consumer")
 	}
+
+	var cleanExpiredEDUs func()
+	cleanExpiredEDUs = func() {
+		logrus.Infof("Cleaning expired EDUs")
+		if err := federationDB.DeleteExpiredEDUs(base.Context()); err != nil {
+			logrus.WithError(err).Error("Failed to clean expired EDUs")
+		}
+		time.AfterFunc(time.Hour, cleanExpiredEDUs)
+	}
+	time.AfterFunc(time.Minute, cleanExpiredEDUs)
+
 	return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, stats, caches, queues, keyRing)
 }
diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go
index b6edec5da..0d937ffaf 100644
--- a/federationapi/queue/destinationqueue.go
+++ b/federationapi/queue/destinationqueue.go
@@ -127,6 +127,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 		oq.destination, // the destination server name
 		receipt,        // NIDs from federationapi_queue_json table
 		event.Type,
+		nil, // this will use the default expireEDUTypes map
 	); err != nil {
 		logrus.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination)
 		return
diff --git a/federationapi/storage/interface.go b/federationapi/storage/interface.go
index 29254948b..b8109b432 100644
--- a/federationapi/storage/interface.go
+++ b/federationapi/storage/interface.go
@@ -16,6 +16,7 @@ package storage
 
 import (
 	"context"
+	"time"
 
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	"github.com/matrix-org/dendrite/federationapi/types"
@@ -38,7 +39,7 @@ type Database interface {
 	GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error)
 
 	AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error
-	AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string) error
+	AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error
 
 	CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
 	CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
@@ -70,4 +71,6 @@ type Database interface {
 	// Query the notary for the server keys for the given server. If `optKeyIDs` is not empty, multiple server keys may be returned (between 1 - len(optKeyIDs))
 	// such that the combination of all server keys will include all the `optKeyIDs`.
 	GetNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, optKeyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error)
+	// DeleteExpiredEDUs cleans up expired EDUs
+	DeleteExpiredEDUs(ctx context.Context) error
 }
diff --git a/federationapi/storage/postgres/deltas/2022042812473400_addexpiresat.go b/federationapi/storage/postgres/deltas/2022042812473400_addexpiresat.go
new file mode 100644
index 000000000..53a7a025e
--- /dev/null
+++ b/federationapi/storage/postgres/deltas/2022042812473400_addexpiresat.go
@@ -0,0 +1,44 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func UpAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus ADD COLUMN IF NOT EXISTS expires_at BIGINT NOT NULL DEFAULT 0;")
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, "UPDATE federationsender_queue_edus SET expires_at = $1 WHERE edu_type != 'm.direct_to_device'", gomatrixserverlib.AsTimestamp(time.Now().Add(time.Hour*24)))
+	if err != nil {
+		return fmt.Errorf("failed to update queue_edus: %w", err)
+	}
+	return nil
+}
+
+func DownAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus DROP COLUMN expires_at;")
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}
diff --git a/federationapi/storage/postgres/queue_edus_table.go b/federationapi/storage/postgres/queue_edus_table.go
index 1fedf0ef1..d6507e13b 100644
--- a/federationapi/storage/postgres/queue_edus_table.go
+++ b/federationapi/storage/postgres/queue_edus_table.go
@@ -19,9 +19,11 @@ import (
 	"database/sql"
 
 	"github.com/lib/pq"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const queueEDUsSchema = `
@@ -31,7 +33,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
     -- The domain part of the user ID the EDU event is for.
 	server_name TEXT NOT NULL,
 	-- The JSON NID from the federationsender_queue_edus_json table.
-	json_nid BIGINT NOT NULL
+	json_nid BIGINT NOT NULL,
+	-- The expiry time of this edu, if any.
+	expires_at BIGINT NOT NULL DEFAULT 0
 );
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
@@ -43,8 +47,8 @@ CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
 `
 
 const insertQueueEDUSQL = "" +
-	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid)" +
-	" VALUES ($1, $2, $3)"
+	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid, expires_at)" +
+	" VALUES ($1, $2, $3, $4)"
 
 const deleteQueueEDUSQL = "" +
 	"DELETE FROM federationsender_queue_edus WHERE server_name = $1 AND json_nid = ANY($2)"
@@ -65,6 +69,12 @@ const selectQueueEDUCountSQL = "" +
 const selectQueueServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_edus"
 
+const selectExpiredEDUsSQL = "" +
+	"SELECT DISTINCT json_nid FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
+const deleteExpiredEDUsSQL = "" +
+	"DELETE FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
 type queueEDUsStatements struct {
 	db                                   *sql.DB
 	insertQueueEDUStmt                   *sql.Stmt
@@ -73,6 +83,8 @@ type queueEDUsStatements struct {
 	selectQueueEDUReferenceJSONCountStmt *sql.Stmt
 	selectQueueEDUCountStmt              *sql.Stmt
 	selectQueueEDUServerNamesStmt        *sql.Stmt
+	selectExpiredEDUsStmt                *sql.Stmt
+	deleteExpiredEDUsStmt                *sql.Stmt
 }
 
 func NewPostgresQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
@@ -81,27 +93,34 @@ func NewPostgresQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
 	}
 	_, err = s.db.Exec(queueEDUsSchema)
 	if err != nil {
-		return
+		return s, err
 	}
-	if s.insertQueueEDUStmt, err = s.db.Prepare(insertQueueEDUSQL); err != nil {
-		return
+
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "federationapi: add expiresat column",
+			Up:      deltas.UpAddexpiresat,
+		},
+	)
+	if err := m.Up(context.Background()); err != nil {
+		return s, err
 	}
-	if s.deleteQueueEDUStmt, err = s.db.Prepare(deleteQueueEDUSQL); err != nil {
-		return
-	}
-	if s.selectQueueEDUStmt, err = s.db.Prepare(selectQueueEDUSQL); err != nil {
-		return
-	}
-	if s.selectQueueEDUReferenceJSONCountStmt, err = s.db.Prepare(selectQueueEDUReferenceJSONCountSQL); err != nil {
-		return
-	}
-	if s.selectQueueEDUCountStmt, err = s.db.Prepare(selectQueueEDUCountSQL); err != nil {
-		return
-	}
-	if s.selectQueueEDUServerNamesStmt, err = s.db.Prepare(selectQueueServerNamesSQL); err != nil {
-		return
-	}
-	return
+
+	return s, nil
+}
+
+func (s *queueEDUsStatements) Prepare() error {
+	return sqlutil.StatementList{
+		{&s.insertQueueEDUStmt, insertQueueEDUSQL},
+		{&s.deleteQueueEDUStmt, deleteQueueEDUSQL},
+		{&s.selectQueueEDUStmt, selectQueueEDUSQL},
+		{&s.selectQueueEDUReferenceJSONCountStmt, selectQueueEDUReferenceJSONCountSQL},
+		{&s.selectQueueEDUCountStmt, selectQueueEDUCountSQL},
+		{&s.selectQueueEDUServerNamesStmt, selectQueueServerNamesSQL},
+		{&s.selectExpiredEDUsStmt, selectExpiredEDUsSQL},
+		{&s.deleteExpiredEDUsStmt, deleteExpiredEDUsSQL},
+	}.Prepare(s.db)
 }
 
 func (s *queueEDUsStatements) InsertQueueEDU(
@@ -110,6 +129,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 	eduType string,
 	serverName gomatrixserverlib.ServerName,
 	nid int64,
+	expiresAt gomatrixserverlib.Timestamp,
 ) error {
 	stmt := sqlutil.TxStmt(txn, s.insertQueueEDUStmt)
 	_, err := stmt.ExecContext(
@@ -117,6 +137,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 		eduType,    // the EDU type
 		serverName, // destination server name
 		nid,        // JSON blob NID
+		expiresAt,  // timestamp of expiry
 	)
 	return err
 }
@@ -150,7 +171,7 @@ func (s *queueEDUsStatements) SelectQueueEDUs(
 		}
 		result = append(result, nid)
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *queueEDUsStatements) SelectQueueEDUReferenceJSONCount(
@@ -200,3 +221,33 @@ func (s *queueEDUsStatements) SelectQueueEDUServerNames(
 
 	return result, rows.Err()
 }
+
+func (s *queueEDUsStatements) SelectExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectExpiredEDUsStmt)
+	rows, err := stmt.QueryContext(ctx, expiredBefore)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectExpiredEDUs: rows.close() failed")
+	var result []int64
+	var nid int64
+	for rows.Next() {
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+	return result, rows.Err()
+}
+
+func (s *queueEDUsStatements) DeleteExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteExpiredEDUsStmt)
+	_, err := stmt.ExecContext(ctx, expiredBefore)
+	return err
+}
diff --git a/federationapi/storage/postgres/storage.go b/federationapi/storage/postgres/storage.go
index 7c2883c1b..6e208d096 100644
--- a/federationapi/storage/postgres/storage.go
+++ b/federationapi/storage/postgres/storage.go
@@ -91,6 +91,9 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, err
 	}
+	if err = queueEDUs.Prepare(); err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                       d.db,
 		ServerName:               serverName,
diff --git a/federationapi/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go
index 02a23338f..b62e5d9c5 100644
--- a/federationapi/storage/shared/storage_edus.go
+++ b/federationapi/storage/shared/storage_edus.go
@@ -20,10 +20,21 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+// defaultExpiry for EDUs if not listed below
+var defaultExpiry = time.Hour * 24
+
+// defaultExpireEDUTypes contains EDUs which can/should be expired after a given time
+// if the target server isn't reachable for some reason.
+var defaultExpireEDUTypes = map[string]time.Duration{
+	gomatrixserverlib.MTyping:   time.Minute,
+	gomatrixserverlib.MPresence: time.Minute * 10,
+}
+
 // AssociateEDUWithDestination creates an association that the
 // destination queues will use to determine which JSON blobs to send
 // to which servers.
@@ -32,7 +43,21 @@ func (d *Database) AssociateEDUWithDestination(
 	serverName gomatrixserverlib.ServerName,
 	receipt *Receipt,
 	eduType string,
+	expireEDUTypes map[string]time.Duration,
 ) error {
+	if expireEDUTypes == nil {
+		expireEDUTypes = defaultExpireEDUTypes
+	}
+	expiresAt := gomatrixserverlib.AsTimestamp(time.Now().Add(defaultExpiry))
+	if duration, ok := expireEDUTypes[eduType]; ok {
+		// Keep EDUs for at least x minutes before deleting them
+		expiresAt = gomatrixserverlib.AsTimestamp(time.Now().Add(duration))
+	}
+	// We forcibly set m.direct_to_device events to 0, as we always want them
+	// to be delivered. (required for E2EE)
+	if eduType == gomatrixserverlib.MDirectToDevice {
+		expiresAt = 0
+	}
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		if err := d.FederationQueueEDUs.InsertQueueEDU(
 			ctx,         // context
@@ -40,6 +65,7 @@ func (d *Database) AssociateEDUWithDestination(
 			eduType,     // EDU type for coalescing
 			serverName,  // destination server name
 			receipt.nid, // NID from the federationapi_queue_json table
+			expiresAt,   // The timestamp this EDU will expire
 		); err != nil {
 			return fmt.Errorf("InsertQueueEDU: %w", err)
 		}
@@ -150,3 +176,26 @@ func (d *Database) GetPendingEDUServerNames(
 ) ([]gomatrixserverlib.ServerName, error) {
 	return d.FederationQueueEDUs.SelectQueueEDUServerNames(ctx, nil)
 }
+
+// DeleteExpiredEDUs deletes expired EDUs
+func (d *Database) DeleteExpiredEDUs(ctx context.Context) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		expiredBefore := gomatrixserverlib.AsTimestamp(time.Now())
+		jsonNIDs, err := d.FederationQueueEDUs.SelectExpiredEDUs(ctx, txn, expiredBefore)
+		if err != nil {
+			return err
+		}
+		if len(jsonNIDs) == 0 {
+			return nil
+		}
+		for i := range jsonNIDs {
+			d.Cache.EvictFederationQueuedEDU(jsonNIDs[i])
+		}
+
+		if err = d.FederationQueueJSON.DeleteQueueJSON(ctx, txn, jsonNIDs); err != nil {
+			return err
+		}
+
+		return d.FederationQueueEDUs.DeleteExpiredEDUs(ctx, txn, expiredBefore)
+	})
+}
diff --git a/federationapi/storage/sqlite3/deltas/2022042812473400_addexpiresat.go b/federationapi/storage/sqlite3/deltas/2022042812473400_addexpiresat.go
new file mode 100644
index 000000000..c5030163b
--- /dev/null
+++ b/federationapi/storage/sqlite3/deltas/2022042812473400_addexpiresat.go
@@ -0,0 +1,68 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func UpAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus RENAME TO federationsender_queue_edus_old;")
+	if err != nil {
+		return fmt.Errorf("failed to rename table: %w", err)
+	}
+
+	_, err = tx.ExecContext(ctx, `
+CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
+	edu_type TEXT NOT NULL,
+	server_name TEXT NOT NULL,
+	json_nid BIGINT NOT NULL,
+	expires_at BIGINT NOT NULL DEFAULT 0
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
+    ON federationsender_queue_edus (json_nid, server_name);
+`)
+	if err != nil {
+		return fmt.Errorf("failed to create new table: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, `
+INSERT
+    INTO federationsender_queue_edus (
+        edu_type, server_name, json_nid, expires_at
+    )  SELECT edu_type, server_name, json_nid, 0 FROM federationsender_queue_edus_old;
+`)
+	if err != nil {
+		return fmt.Errorf("failed to update queue_edus: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, "UPDATE federationsender_queue_edus SET expires_at = $1 WHERE edu_type != 'm.direct_to_device'", gomatrixserverlib.AsTimestamp(time.Now().Add(time.Hour*24)))
+	if err != nil {
+		return fmt.Errorf("failed to update queue_edus: %w", err)
+	}
+	return nil
+}
+
+func DownAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus DROP COLUMN expires_at;")
+	if err != nil {
+		return fmt.Errorf("failed to rename table: %w", err)
+	}
+	return nil
+}
diff --git a/federationapi/storage/sqlite3/queue_edus_table.go b/federationapi/storage/sqlite3/queue_edus_table.go
index f4c84f094..8e7e7901f 100644
--- a/federationapi/storage/sqlite3/queue_edus_table.go
+++ b/federationapi/storage/sqlite3/queue_edus_table.go
@@ -20,9 +20,11 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const queueEDUsSchema = `
@@ -32,7 +34,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
     -- The domain part of the user ID the EDU event is for.
 	server_name TEXT NOT NULL,
 	-- The JSON NID from the federationsender_queue_edus_json table.
-	json_nid BIGINT NOT NULL
+	json_nid BIGINT NOT NULL,
+	-- The expiry time of this edu, if any.
+	expires_at BIGINT NOT NULL DEFAULT 0
 );
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
@@ -44,8 +48,8 @@ CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
 `
 
 const insertQueueEDUSQL = "" +
-	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid)" +
-	" VALUES ($1, $2, $3)"
+	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid, expires_at)" +
+	" VALUES ($1, $2, $3, $4)"
 
 const deleteQueueEDUsSQL = "" +
 	"DELETE FROM federationsender_queue_edus WHERE server_name = $1 AND json_nid IN ($2)"
@@ -66,13 +70,22 @@ const selectQueueEDUCountSQL = "" +
 const selectQueueServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_edus"
 
+const selectExpiredEDUsSQL = "" +
+	"SELECT DISTINCT json_nid FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
+const deleteExpiredEDUsSQL = "" +
+	"DELETE FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
 type queueEDUsStatements struct {
-	db                                   *sql.DB
-	insertQueueEDUStmt                   *sql.Stmt
+	db                 *sql.DB
+	insertQueueEDUStmt *sql.Stmt
+	// deleteQueueEDUStmt                *sql.Stmt - prepared at runtime due to variadic
 	selectQueueEDUStmt                   *sql.Stmt
 	selectQueueEDUReferenceJSONCountStmt *sql.Stmt
 	selectQueueEDUCountStmt              *sql.Stmt
 	selectQueueEDUServerNamesStmt        *sql.Stmt
+	selectExpiredEDUsStmt                *sql.Stmt
+	deleteExpiredEDUsStmt                *sql.Stmt
 }
 
 func NewSQLiteQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
@@ -81,24 +94,33 @@ func NewSQLiteQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
 	}
 	_, err = db.Exec(queueEDUsSchema)
 	if err != nil {
-		return
+		return s, err
 	}
-	if s.insertQueueEDUStmt, err = db.Prepare(insertQueueEDUSQL); err != nil {
-		return
+
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "federationapi: add expiresat column",
+			Up:      deltas.UpAddexpiresat,
+		},
+	)
+	if err := m.Up(context.Background()); err != nil {
+		return s, err
 	}
-	if s.selectQueueEDUStmt, err = db.Prepare(selectQueueEDUSQL); err != nil {
-		return
-	}
-	if s.selectQueueEDUReferenceJSONCountStmt, err = db.Prepare(selectQueueEDUReferenceJSONCountSQL); err != nil {
-		return
-	}
-	if s.selectQueueEDUCountStmt, err = db.Prepare(selectQueueEDUCountSQL); err != nil {
-		return
-	}
-	if s.selectQueueEDUServerNamesStmt, err = db.Prepare(selectQueueServerNamesSQL); err != nil {
-		return
-	}
-	return
+
+	return s, nil
+}
+
+func (s *queueEDUsStatements) Prepare() error {
+	return sqlutil.StatementList{
+		{&s.insertQueueEDUStmt, insertQueueEDUSQL},
+		{&s.selectQueueEDUStmt, selectQueueEDUSQL},
+		{&s.selectQueueEDUReferenceJSONCountStmt, selectQueueEDUReferenceJSONCountSQL},
+		{&s.selectQueueEDUCountStmt, selectQueueEDUCountSQL},
+		{&s.selectQueueEDUServerNamesStmt, selectQueueServerNamesSQL},
+		{&s.selectExpiredEDUsStmt, selectExpiredEDUsSQL},
+		{&s.deleteExpiredEDUsStmt, deleteExpiredEDUsSQL},
+	}.Prepare(s.db)
 }
 
 func (s *queueEDUsStatements) InsertQueueEDU(
@@ -107,6 +129,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 	eduType string,
 	serverName gomatrixserverlib.ServerName,
 	nid int64,
+	expiresAt gomatrixserverlib.Timestamp,
 ) error {
 	stmt := sqlutil.TxStmt(txn, s.insertQueueEDUStmt)
 	_, err := stmt.ExecContext(
@@ -114,6 +137,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 		eduType,    // the EDU type
 		serverName, // destination server name
 		nid,        // JSON blob NID
+		expiresAt,  // timestamp of expiry
 	)
 	return err
 }
@@ -159,7 +183,7 @@ func (s *queueEDUsStatements) SelectQueueEDUs(
 		}
 		result = append(result, nid)
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *queueEDUsStatements) SelectQueueEDUReferenceJSONCount(
@@ -209,3 +233,33 @@ func (s *queueEDUsStatements) SelectQueueEDUServerNames(
 
 	return result, rows.Err()
 }
+
+func (s *queueEDUsStatements) SelectExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectExpiredEDUsStmt)
+	rows, err := stmt.QueryContext(ctx, expiredBefore)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectExpiredEDUs: rows.close() failed")
+	var result []int64
+	var nid int64
+	for rows.Next() {
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+	return result, rows.Err()
+}
+
+func (s *queueEDUsStatements) DeleteExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteExpiredEDUsStmt)
+	_, err := stmt.ExecContext(ctx, expiredBefore)
+	return err
+}
diff --git a/federationapi/storage/sqlite3/storage.go b/federationapi/storage/sqlite3/storage.go
index 9594aaec6..c89cb6bea 100644
--- a/federationapi/storage/sqlite3/storage.go
+++ b/federationapi/storage/sqlite3/storage.go
@@ -90,6 +90,9 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, err
 	}
+	if err = queueEDUs.Prepare(); err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                       d.db,
 		ServerName:               serverName,
diff --git a/federationapi/storage/storage_test.go b/federationapi/storage/storage_test.go
new file mode 100644
index 000000000..7eba2cbee
--- /dev/null
+++ b/federationapi/storage/storage_test.go
@@ -0,0 +1,81 @@
+package storage_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/matrix-org/dendrite/federationapi/storage"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+)
+
+func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	b, baseClose := testrig.CreateBaseDendrite(t, dbType)
+	connStr, dbClose := test.PrepareDBConnectionString(t, dbType)
+	db, err := storage.NewDatabase(b, &config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, b.Caches, b.Cfg.Global.ServerName)
+	if err != nil {
+		t.Fatalf("NewDatabase returned %s", err)
+	}
+	return db, func() {
+		dbClose()
+		baseClose()
+	}
+}
+
+func TestExpireEDUs(t *testing.T) {
+	var expireEDUTypes = map[string]time.Duration{
+		gomatrixserverlib.MReceipt: time.Millisecond,
+	}
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateFederationDatabase(t, dbType)
+		defer close()
+		// insert some data
+		for i := 0; i < 100; i++ {
+			receipt, err := db.StoreJSON(ctx, "{}")
+			assert.NoError(t, err)
+
+			err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MReceipt, expireEDUTypes)
+			assert.NoError(t, err)
+		}
+		// add data without expiry
+		receipt, err := db.StoreJSON(ctx, "{}")
+		assert.NoError(t, err)
+
+		// m.read_marker gets the default expiry of 24h, so won't be deleted further down in this test
+		err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, "m.read_marker", expireEDUTypes)
+		assert.NoError(t, err)
+
+		// Delete expired EDUs
+		err = db.DeleteExpiredEDUs(ctx)
+		assert.NoError(t, err)
+
+		// verify the data is gone
+		data, err := db.GetPendingEDUs(ctx, "localhost", 100)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(data))
+
+		// check that m.direct_to_device is never expired
+		receipt, err = db.StoreJSON(ctx, "{}")
+		assert.NoError(t, err)
+
+		err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MDirectToDevice, expireEDUTypes)
+		assert.NoError(t, err)
+
+		err = db.DeleteExpiredEDUs(ctx)
+		assert.NoError(t, err)
+
+		// We should get two EDUs, the m.read_marker and the m.direct_to_device
+		data, err = db.GetPendingEDUs(ctx, "localhost", 100)
+		assert.NoError(t, err)
+		assert.Equal(t, 2, len(data))
+	})
+}
diff --git a/federationapi/storage/tables/interface.go b/federationapi/storage/tables/interface.go
index 19357393d..3c116a1d0 100644
--- a/federationapi/storage/tables/interface.go
+++ b/federationapi/storage/tables/interface.go
@@ -34,12 +34,15 @@ type FederationQueuePDUs interface {
 }
 
 type FederationQueueEDUs interface {
-	InsertQueueEDU(ctx context.Context, txn *sql.Tx, eduType string, serverName gomatrixserverlib.ServerName, nid int64) error
+	InsertQueueEDU(ctx context.Context, txn *sql.Tx, eduType string, serverName gomatrixserverlib.ServerName, nid int64, expiresAt gomatrixserverlib.Timestamp) error
 	DeleteQueueEDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, jsonNIDs []int64) error
 	SelectQueueEDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, limit int) ([]int64, error)
 	SelectQueueEDUReferenceJSONCount(ctx context.Context, txn *sql.Tx, jsonNID int64) (int64, error)
 	SelectQueueEDUCount(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (int64, error)
 	SelectQueueEDUServerNames(ctx context.Context, txn *sql.Tx) ([]gomatrixserverlib.ServerName, error)
+	SelectExpiredEDUs(ctx context.Context, txn *sql.Tx, expiredBefore gomatrixserverlib.Timestamp) ([]int64, error)
+	DeleteExpiredEDUs(ctx context.Context, txn *sql.Tx, expiredBefore gomatrixserverlib.Timestamp) error
+	Prepare() error
 }
 
 type FederationQueueJSON interface {

From c45d0936b59b0eb65f12fe22a3da3690ae0b5494 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 11 Aug 2022 15:29:33 +0100
Subject: [PATCH 256/391] Generic-based internal HTTP API (#2626)

* Generic-based internal HTTP API (tested out on a few endpoints in the federation API)

* Add `PerformInvite`

* More tweaks

* Fix metric name

* Fix LookupStateIDs

* Lots of changes to clients

* Some serverside stuff

* Some error handling

* Use paths as metric names

* Revert "Use paths as metric names"

This reverts commit a9323a6a343f5ce6461a2e5bd570fe06465f1b15.

* Namespace metric names

* Remove duplicate entry

* Remove another duplicate entry

* Tweak error handling

* Some more tweaks

* Update error behaviour

* Some more error tweaking

* Fix API path for `PerformDeleteKeys`

* Fix another path

* Tweak federation client proxying

* Fix another path

* Don't return typed nils

* Some more tweaks, not that it makes any difference

* Tweak federation client proxying

* Maybe fix the key backup test
---
 appservice/inthttp/client.go                  |  19 +-
 appservice/inthttp/server.go                  |  29 +-
 clientapi/jsonerror/jsonerror.go              |  14 +
 clientapi/routing/admin.go                    |  12 +-
 clientapi/routing/createroom.go               |   6 +-
 clientapi/routing/directory.go                |   6 +-
 clientapi/routing/joinroom.go                 |   5 +-
 clientapi/routing/key_backup.go               |  12 +-
 clientapi/routing/key_crosssigning.go         |   8 +-
 clientapi/routing/keys.go                     |  16 +-
 clientapi/routing/peekroom.go                 |   9 +-
 clientapi/routing/upgrade_room.go             |   4 +-
 federationapi/api/api.go                      |   2 +-
 federationapi/federationapi_test.go           |   5 +-
 federationapi/inthttp/client.go               | 437 +++++---------
 federationapi/inthttp/server.go               | 461 +++++----------
 federationapi/routing/devices.go              |  10 +-
 federationapi/routing/join.go                 |   6 +-
 federationapi/routing/keys.go                 |  16 +-
 federationapi/routing/leave.go                |   6 +-
 federationapi/routing/send.go                 |   4 +-
 federationapi/routing/send_test.go            |   3 +-
 internal/httputil/http.go                     |  40 +-
 internal/httputil/internalapi.go              |  93 +++
 keyserver/api/api.go                          |  28 +-
 keyserver/internal/cross_signing.go           |  41 +-
 keyserver/internal/device_list_update.go      |   4 +-
 keyserver/internal/device_list_update_test.go |   4 +-
 keyserver/internal/internal.go                |  32 +-
 keyserver/inthttp/client.go                   | 161 ++---
 keyserver/inthttp/server.go                   | 143 ++---
 roomserver/api/api.go                         |  18 +-
 roomserver/api/api_trace.go                   |  63 +-
 roomserver/api/wrapper.go                     |   4 +-
 roomserver/internal/input/input.go            |  10 +-
 roomserver/internal/perform/perform_admin.go  |  37 +-
 roomserver/internal/perform/perform_invite.go |   4 +-
 roomserver/internal/perform/perform_join.go   |  12 +-
 roomserver/internal/perform/perform_leave.go  |   4 +-
 roomserver/internal/perform/perform_peek.go   |   3 +-
 .../internal/perform/perform_publish.go       |   3 +-
 roomserver/internal/perform/perform_unpeek.go |   3 +-
 .../internal/perform/perform_upgrade.go       |  17 +-
 roomserver/inthttp/client.go                  | 431 ++++++--------
 roomserver/inthttp/server.go                  | 525 ++++-------------
 setup/mscs/msc2836/msc2836_test.go            |   6 +-
 syncapi/internal/keychange.go                 |   4 +-
 syncapi/internal/keychange_test.go            |  30 +-
 syncapi/syncapi_test.go                       |   7 +-
 userapi/api/api.go                            |   2 +-
 userapi/api/api_trace.go                      |   5 +-
 userapi/internal/api.go                       |  46 +-
 userapi/inthttp/client.go                     | 440 +++++++-------
 userapi/inthttp/client_logintoken.go          |  28 +-
 userapi/inthttp/server.go                     | 550 +++++-------------
 userapi/inthttp/server_logintoken.go          |  51 +-
 userapi/userapi_test.go                       |  14 +-
 57 files changed, 1535 insertions(+), 2418 deletions(-)
 create mode 100644 internal/httputil/internalapi.go

diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go
index 0a8baea99..3ae2c9278 100644
--- a/appservice/inthttp/client.go
+++ b/appservice/inthttp/client.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/opentracing/opentracing-go"
 )
 
 // HTTP paths for the internal HTTP APIs
@@ -42,11 +41,10 @@ func (h *httpAppServiceQueryAPI) RoomAliasExists(
 	request *api.RoomAliasExistsRequest,
 	response *api.RoomAliasExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists")
-	defer span.Finish()
-
-	apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"RoomAliasExists", h.appserviceURL+AppServiceRoomAliasExistsPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // UserIDExists implements AppServiceQueryAPI
@@ -55,9 +53,8 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
 	request *api.UserIDExistsRequest,
 	response *api.UserIDExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists")
-	defer span.Finish()
-
-	apiURL := h.appserviceURL + AppServiceUserIDExistsPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"UserIDExists", h.appserviceURL+AppServiceUserIDExistsPath,
+		h.httpClient, ctx, request, response,
+	)
 }
diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go
index 645b43871..01d9f9895 100644
--- a/appservice/inthttp/server.go
+++ b/appservice/inthttp/server.go
@@ -1,43 +1,20 @@
 package inthttp
 
 import (
-	"encoding/json"
-	"net/http"
-
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/util"
 )
 
 // AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
 func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		AppServiceRoomAliasExistsPath,
-		httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
-			var request api.RoomAliasExistsRequest
-			var response api.RoomAliasExistsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
 	)
+
 	internalAPIMux.Handle(
 		AppServiceUserIDExistsPath,
-		httputil.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
-			var request api.UserIDExistsRequest
-			var response api.UserIDExistsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := a.UserIDExists(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
 	)
 }
diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go
index 70bac61dc..be7d13a96 100644
--- a/clientapi/jsonerror/jsonerror.go
+++ b/clientapi/jsonerror/jsonerror.go
@@ -15,11 +15,13 @@
 package jsonerror
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // MatrixError represents the "standard error response" in Matrix.
@@ -213,3 +215,15 @@ func NotTrusted(serverName string) *MatrixError {
 		Err:     fmt.Sprintf("Untrusted server '%s'", serverName),
 	}
 }
+
+// InternalAPIError is returned when Dendrite failed to reach an internal API.
+func InternalAPIError(ctx context.Context, err error) util.JSONResponse {
+	logrus.WithContext(ctx).WithError(err).Error("Error reaching an internal API")
+	return util.JSONResponse{
+		Code: http.StatusInternalServerError,
+		JSON: &MatrixError{
+			ErrCode: "M_INTERNAL_SERVER_ERROR",
+			Err:     "Dendrite encountered an error reaching an internal API.",
+		},
+	}
+}
diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index 523b88c99..a8dd0e64f 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -30,13 +30,15 @@ func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserv
 		}
 	}
 	res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
-	rsAPI.PerformAdminEvacuateRoom(
+	if err := rsAPI.PerformAdminEvacuateRoom(
 		req.Context(),
 		&roomserverAPI.PerformAdminEvacuateRoomRequest{
 			RoomID: roomID,
 		},
 		res,
-	)
+	); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if err := res.Error; err != nil {
 		return err.JSONResponse()
 	}
@@ -67,13 +69,15 @@ func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserv
 		}
 	}
 	res := &roomserverAPI.PerformAdminEvacuateUserResponse{}
-	rsAPI.PerformAdminEvacuateUser(
+	if err := rsAPI.PerformAdminEvacuateUser(
 		req.Context(),
 		&roomserverAPI.PerformAdminEvacuateUserRequest{
 			UserID: userID,
 		},
 		res,
-	)
+	); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if err := res.Error; err != nil {
 		return err.JSONResponse()
 	}
diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 3f92b7ba6..874908639 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -556,10 +556,12 @@ func createRoom(
 	if r.Visibility == "public" {
 		// expose this room in the published room list
 		var pubRes roomserverAPI.PerformPublishResponse
-		rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
+		if err := rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
 			RoomID:     roomID,
 			Visibility: "public",
-		}, &pubRes)
+		}, &pubRes); err != nil {
+			return jsonerror.InternalAPIError(ctx, err)
+		}
 		if pubRes.Error != nil {
 			// treat as non-fatal since the room is already made by this point
 			util.GetLogger(ctx).WithError(pubRes.Error).Error("failed to visibility:public")
diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go
index 53ba3f190..836d9e152 100644
--- a/clientapi/routing/directory.go
+++ b/clientapi/routing/directory.go
@@ -302,10 +302,12 @@ func SetVisibility(
 	}
 
 	var publishRes roomserverAPI.PerformPublishResponse
-	rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
+	if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
 		RoomID:     roomID,
 		Visibility: v.Visibility,
-	}, &publishRes)
+	}, &publishRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if publishRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
 		return publishRes.Error.JSONResponse()
diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go
index 4e6acebc3..c50e552bd 100644
--- a/clientapi/routing/joinroom.go
+++ b/clientapi/routing/joinroom.go
@@ -81,8 +81,9 @@ func JoinRoomByIDOrAlias(
 	done := make(chan util.JSONResponse, 1)
 	go func() {
 		defer close(done)
-		rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes)
-		if joinRes.Error != nil {
+		if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
+			done <- jsonerror.InternalAPIError(req.Context(), err)
+		} else if joinRes.Error != nil {
 			done <- joinRes.Error.JSONResponse()
 		} else {
 			done <- util.JSONResponse{
diff --git a/clientapi/routing/key_backup.go b/clientapi/routing/key_backup.go
index 28c80415b..b6f8fe1b9 100644
--- a/clientapi/routing/key_backup.go
+++ b/clientapi/routing/key_backup.go
@@ -91,10 +91,12 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, de
 // Implements GET  /_matrix/client/r0/room_keys/version and GET /_matrix/client/r0/room_keys/version/{version}
 func KeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
 	var queryResp userapi.QueryKeyBackupResponse
-	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
+	if err := userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
 		UserID:  device.UserID,
 		Version: version,
-	}, &queryResp)
+	}, &queryResp); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if queryResp.Error != "" {
 		return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
 	}
@@ -233,13 +235,15 @@ func GetBackupKeys(
 	req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version, roomID, sessionID string,
 ) util.JSONResponse {
 	var queryResp userapi.QueryKeyBackupResponse
-	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
+	if err := userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
 		UserID:           device.UserID,
 		Version:          version,
 		ReturnKeys:       true,
 		KeysForRoomID:    roomID,
 		KeysForSessionID: sessionID,
-	}, &queryResp)
+	}, &queryResp); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if queryResp.Error != "" {
 		return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
 	}
diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go
index 8fbb86f7a..2570db09c 100644
--- a/clientapi/routing/key_crosssigning.go
+++ b/clientapi/routing/key_crosssigning.go
@@ -72,7 +72,9 @@ func UploadCrossSigningDeviceKeys(
 	sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
 
 	uploadReq.UserID = device.UserID
-	keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes)
+	if err := keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	if err := uploadRes.Error; err != nil {
 		switch {
@@ -114,7 +116,9 @@ func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.Clie
 	}
 
 	uploadReq.UserID = device.UserID
-	keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes)
+	if err := keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	if err := uploadRes.Error; err != nil {
 		switch {
diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go
index fdda34a53..b7a76b47e 100644
--- a/clientapi/routing/keys.go
+++ b/clientapi/routing/keys.go
@@ -62,7 +62,9 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi
 	}
 
 	var uploadRes api.PerformUploadKeysResponse
-	keyAPI.PerformUploadKeys(req.Context(), uploadReq, &uploadRes)
+	if err := keyAPI.PerformUploadKeys(req.Context(), uploadReq, &uploadRes); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if uploadRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(uploadRes.Error).Error("Failed to PerformUploadKeys")
 		return jsonerror.InternalServerError()
@@ -107,12 +109,14 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devic
 		return *resErr
 	}
 	queryRes := api.QueryKeysResponse{}
-	keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
+	if err := keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
 		UserID:        device.UserID,
 		UserToDevices: r.DeviceKeys,
 		Timeout:       r.GetTimeout(),
 		// TODO: Token?
-	}, &queryRes)
+	}, &queryRes); err != nil {
+		return util.ErrorResponse(err)
+	}
 	return util.JSONResponse{
 		Code: 200,
 		JSON: map[string]interface{}{
@@ -145,10 +149,12 @@ func ClaimKeys(req *http.Request, keyAPI api.ClientKeyAPI) util.JSONResponse {
 		return *resErr
 	}
 	claimRes := api.PerformClaimKeysResponse{}
-	keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
+	if err := keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
 		OneTimeKeys: r.OneTimeKeys,
 		Timeout:     r.GetTimeout(),
-	}, &claimRes)
+	}, &claimRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if claimRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(claimRes.Error).Error("failed to PerformClaimKeys")
 		return jsonerror.InternalServerError()
diff --git a/clientapi/routing/peekroom.go b/clientapi/routing/peekroom.go
index d0eeccf17..9b2592eb5 100644
--- a/clientapi/routing/peekroom.go
+++ b/clientapi/routing/peekroom.go
@@ -17,6 +17,7 @@ package routing
 import (
 	"net/http"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -54,7 +55,9 @@ func PeekRoomByIDOrAlias(
 	}
 
 	// Ask the roomserver to perform the peek.
-	rsAPI.PerformPeek(req.Context(), &peekReq, &peekRes)
+	if err := rsAPI.PerformPeek(req.Context(), &peekReq, &peekRes); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if peekRes.Error != nil {
 		return peekRes.Error.JSONResponse()
 	}
@@ -89,7 +92,9 @@ func UnpeekRoomByID(
 	}
 	unpeekRes := roomserverAPI.PerformUnpeekResponse{}
 
-	rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes)
+	if err := rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if unpeekRes.Error != nil {
 		return unpeekRes.Error.JSONResponse()
 	}
diff --git a/clientapi/routing/upgrade_room.go b/clientapi/routing/upgrade_room.go
index 744e2d889..34c7eb004 100644
--- a/clientapi/routing/upgrade_room.go
+++ b/clientapi/routing/upgrade_room.go
@@ -64,7 +64,9 @@ func UpgradeRoom(
 	}
 	upgradeResp := roomserverAPI.PerformRoomUpgradeResponse{}
 
-	rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp)
+	if err := rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	if upgradeResp.Error != nil {
 		if upgradeResp.Error.Code == roomserverAPI.PerformErrorNoRoom {
diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index 53d4701f3..292ed55ad 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -110,7 +110,7 @@ type FederationClientError struct {
 	Blacklisted bool
 }
 
-func (e *FederationClientError) Error() string {
+func (e FederationClientError) Error() string {
 	return fmt.Sprintf("%s - (retry_after=%s, blacklisted=%v)", e.Err, e.RetryAfter.String(), e.Blacklisted)
 }
 
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index 8884e34c6..bdcb9f57c 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -32,11 +32,12 @@ type fedRoomserverAPI struct {
 }
 
 // PerformJoin will call this function
-func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) error {
 	if f.inputRoomEvents == nil {
-		return
+		return nil
 	}
 	f.inputRoomEvents(ctx, req, res)
+	return nil
 }
 
 // keychange consumer calls this
diff --git a/federationapi/inthttp/client.go b/federationapi/inthttp/client.go
index 295ddc495..812d3c6da 100644
--- a/federationapi/inthttp/client.go
+++ b/federationapi/inthttp/client.go
@@ -10,7 +10,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/opentracing/opentracing-go"
 )
 
 // HTTP paths for the internal HTTP API
@@ -48,7 +47,11 @@ func NewFederationAPIClient(federationSenderURL string, httpClient *http.Client,
 	if httpClient == nil {
 		return nil, errors.New("NewFederationInternalAPIHTTP: httpClient is <nil>")
 	}
-	return &httpFederationInternalAPI{federationSenderURL, httpClient, cache}, nil
+	return &httpFederationInternalAPI{
+		federationAPIURL: federationSenderURL,
+		httpClient:       httpClient,
+		cache:            cache,
+	}, nil
 }
 
 type httpFederationInternalAPI struct {
@@ -63,11 +66,10 @@ func (h *httpFederationInternalAPI) PerformLeave(
 	request *api.PerformLeaveRequest,
 	response *api.PerformLeaveResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIPerformLeaveRequestPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformLeave", h.federationAPIURL+FederationAPIPerformLeaveRequestPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // Handle sending an invite to a remote server.
@@ -76,11 +78,10 @@ func (h *httpFederationInternalAPI) PerformInvite(
 	request *api.PerformInviteRequest,
 	response *api.PerformInviteResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformInviteRequest")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIPerformInviteRequestPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformInvite", h.federationAPIURL+FederationAPIPerformInviteRequestPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // Handle starting a peek on a remote server.
@@ -89,11 +90,10 @@ func (h *httpFederationInternalAPI) PerformOutboundPeek(
 	request *api.PerformOutboundPeekRequest,
 	response *api.PerformOutboundPeekResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformOutboundPeekRequest")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIPerformOutboundPeekRequestPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformOutboundPeek", h.federationAPIURL+FederationAPIPerformOutboundPeekRequestPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryJoinedHostServerNamesInRoom implements FederationInternalAPI
@@ -102,11 +102,10 @@ func (h *httpFederationInternalAPI) QueryJoinedHostServerNamesInRoom(
 	request *api.QueryJoinedHostServerNamesInRoomRequest,
 	response *api.QueryJoinedHostServerNamesInRoomResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIQueryJoinedHostServerNamesInRoomPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryJoinedHostServerNamesInRoom", h.federationAPIURL+FederationAPIQueryJoinedHostServerNamesInRoomPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // Handle an instruction to make_join & send_join with a remote server.
@@ -115,12 +114,10 @@ func (h *httpFederationInternalAPI) PerformJoin(
 	request *api.PerformJoinRequest,
 	response *api.PerformJoinResponse,
 ) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIPerformJoinRequestPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
+	if err := httputil.CallInternalRPCAPI(
+		"PerformJoinRequest", h.federationAPIURL+FederationAPIPerformJoinRequestPath,
+		h.httpClient, ctx, request, response,
+	); err != nil {
 		response.LastError = &gomatrix.HTTPError{
 			Message:      err.Error(),
 			Code:         0,
@@ -135,11 +132,10 @@ func (h *httpFederationInternalAPI) PerformDirectoryLookup(
 	request *api.PerformDirectoryLookupRequest,
 	response *api.PerformDirectoryLookupResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIPerformDirectoryLookupRequestPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformDirectoryLookup", h.federationAPIURL+FederationAPIPerformDirectoryLookupRequestPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // Handle an instruction to broadcast an EDU to all servers in rooms we are joined to.
@@ -148,101 +144,61 @@ func (h *httpFederationInternalAPI) PerformBroadcastEDU(
 	request *api.PerformBroadcastEDURequest,
 	response *api.PerformBroadcastEDUResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformBroadcastEDU")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIPerformBroadcastEDUPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformBroadcastEDU", h.federationAPIURL+FederationAPIPerformBroadcastEDUPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 type getUserDevices struct {
 	S      gomatrixserverlib.ServerName
 	UserID string
-	Res    *gomatrixserverlib.RespUserDevices
-	Err    *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) GetUserDevices(
 	ctx context.Context, s gomatrixserverlib.ServerName, userID string,
 ) (gomatrixserverlib.RespUserDevices, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetUserDevices")
-	defer span.Finish()
-
-	var result gomatrixserverlib.RespUserDevices
-	request := getUserDevices{
-		S:      s,
-		UserID: userID,
-	}
-	var response getUserDevices
-	apiURL := h.federationAPIURL + FederationAPIGetUserDevicesPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return result, err
-	}
-	if response.Err != nil {
-		return result, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[getUserDevices, gomatrixserverlib.RespUserDevices, *api.FederationClientError](
+		"GetUserDevices", h.federationAPIURL+FederationAPIGetUserDevicesPath, h.httpClient,
+		ctx, &getUserDevices{
+			S:      s,
+			UserID: userID,
+		},
+	)
 }
 
 type claimKeys struct {
 	S           gomatrixserverlib.ServerName
 	OneTimeKeys map[string]map[string]string
-	Res         *gomatrixserverlib.RespClaimKeys
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) ClaimKeys(
 	ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
 ) (gomatrixserverlib.RespClaimKeys, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ClaimKeys")
-	defer span.Finish()
-
-	var result gomatrixserverlib.RespClaimKeys
-	request := claimKeys{
-		S:           s,
-		OneTimeKeys: oneTimeKeys,
-	}
-	var response claimKeys
-	apiURL := h.federationAPIURL + FederationAPIClaimKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return result, err
-	}
-	if response.Err != nil {
-		return result, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[claimKeys, gomatrixserverlib.RespClaimKeys, *api.FederationClientError](
+		"ClaimKeys", h.federationAPIURL+FederationAPIClaimKeysPath, h.httpClient,
+		ctx, &claimKeys{
+			S:           s,
+			OneTimeKeys: oneTimeKeys,
+		},
+	)
 }
 
 type queryKeys struct {
 	S    gomatrixserverlib.ServerName
 	Keys map[string][]string
-	Res  *gomatrixserverlib.RespQueryKeys
-	Err  *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) QueryKeys(
 	ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string,
 ) (gomatrixserverlib.RespQueryKeys, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeys")
-	defer span.Finish()
-
-	var result gomatrixserverlib.RespQueryKeys
-	request := queryKeys{
-		S:    s,
-		Keys: keys,
-	}
-	var response queryKeys
-	apiURL := h.federationAPIURL + FederationAPIQueryKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return result, err
-	}
-	if response.Err != nil {
-		return result, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[queryKeys, gomatrixserverlib.RespQueryKeys, *api.FederationClientError](
+		"QueryKeys", h.federationAPIURL+FederationAPIQueryKeysPath, h.httpClient,
+		ctx, &queryKeys{
+			S:    s,
+			Keys: keys,
+		},
+	)
 }
 
 type backfill struct {
@@ -250,32 +206,20 @@ type backfill struct {
 	RoomID   string
 	Limit    int
 	EventIDs []string
-	Res      *gomatrixserverlib.Transaction
-	Err      *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) Backfill(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string,
 ) (gomatrixserverlib.Transaction, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "Backfill")
-	defer span.Finish()
-
-	request := backfill{
-		S:        s,
-		RoomID:   roomID,
-		Limit:    limit,
-		EventIDs: eventIDs,
-	}
-	var response backfill
-	apiURL := h.federationAPIURL + FederationAPIBackfillPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.Transaction{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.Transaction{}, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[backfill, gomatrixserverlib.Transaction, *api.FederationClientError](
+		"Backfill", h.federationAPIURL+FederationAPIBackfillPath, h.httpClient,
+		ctx, &backfill{
+			S:        s,
+			RoomID:   roomID,
+			Limit:    limit,
+			EventIDs: eventIDs,
+		},
+	)
 }
 
 type lookupState struct {
@@ -283,63 +227,39 @@ type lookupState struct {
 	RoomID      string
 	EventID     string
 	RoomVersion gomatrixserverlib.RoomVersion
-	Res         *gomatrixserverlib.RespState
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupState(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
 ) (gomatrixserverlib.RespState, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupState")
-	defer span.Finish()
-
-	request := lookupState{
-		S:           s,
-		RoomID:      roomID,
-		EventID:     eventID,
-		RoomVersion: roomVersion,
-	}
-	var response lookupState
-	apiURL := h.federationAPIURL + FederationAPILookupStatePath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.RespState{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.RespState{}, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[lookupState, gomatrixserverlib.RespState, *api.FederationClientError](
+		"LookupState", h.federationAPIURL+FederationAPILookupStatePath, h.httpClient,
+		ctx, &lookupState{
+			S:           s,
+			RoomID:      roomID,
+			EventID:     eventID,
+			RoomVersion: roomVersion,
+		},
+	)
 }
 
 type lookupStateIDs struct {
 	S       gomatrixserverlib.ServerName
 	RoomID  string
 	EventID string
-	Res     *gomatrixserverlib.RespStateIDs
-	Err     *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupStateIDs(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string,
 ) (gomatrixserverlib.RespStateIDs, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupStateIDs")
-	defer span.Finish()
-
-	request := lookupStateIDs{
-		S:       s,
-		RoomID:  roomID,
-		EventID: eventID,
-	}
-	var response lookupStateIDs
-	apiURL := h.federationAPIURL + FederationAPILookupStateIDsPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.RespStateIDs{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.RespStateIDs{}, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[lookupStateIDs, gomatrixserverlib.RespStateIDs, *api.FederationClientError](
+		"LookupStateIDs", h.federationAPIURL+FederationAPILookupStateIDsPath, h.httpClient,
+		ctx, &lookupStateIDs{
+			S:       s,
+			RoomID:  roomID,
+			EventID: eventID,
+		},
+	)
 }
 
 type lookupMissingEvents struct {
@@ -347,64 +267,38 @@ type lookupMissingEvents struct {
 	RoomID      string
 	Missing     gomatrixserverlib.MissingEvents
 	RoomVersion gomatrixserverlib.RoomVersion
-	Res         struct {
-		Events []gomatrixserverlib.RawJSON `json:"events"`
-	}
-	Err *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupMissingEvents(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID string,
 	missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
 ) (res gomatrixserverlib.RespMissingEvents, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupMissingEvents")
-	defer span.Finish()
-
-	request := lookupMissingEvents{
-		S:           s,
-		RoomID:      roomID,
-		Missing:     missing,
-		RoomVersion: roomVersion,
-	}
-	apiURL := h.federationAPIURL + FederationAPILookupMissingEventsPath
-	err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &request)
-	if err != nil {
-		return res, err
-	}
-	if request.Err != nil {
-		return res, request.Err
-	}
-	res.Events = request.Res.Events
-	return res, nil
+	return httputil.CallInternalProxyAPI[lookupMissingEvents, gomatrixserverlib.RespMissingEvents, *api.FederationClientError](
+		"LookupMissingEvents", h.federationAPIURL+FederationAPILookupMissingEventsPath, h.httpClient,
+		ctx, &lookupMissingEvents{
+			S:           s,
+			RoomID:      roomID,
+			Missing:     missing,
+			RoomVersion: roomVersion,
+		},
+	)
 }
 
 type getEvent struct {
 	S       gomatrixserverlib.ServerName
 	EventID string
-	Res     *gomatrixserverlib.Transaction
-	Err     *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) GetEvent(
 	ctx context.Context, s gomatrixserverlib.ServerName, eventID string,
 ) (gomatrixserverlib.Transaction, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetEvent")
-	defer span.Finish()
-
-	request := getEvent{
-		S:       s,
-		EventID: eventID,
-	}
-	var response getEvent
-	apiURL := h.federationAPIURL + FederationAPIGetEventPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.Transaction{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.Transaction{}, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[getEvent, gomatrixserverlib.Transaction, *api.FederationClientError](
+		"GetEvent", h.federationAPIURL+FederationAPIGetEventPath, h.httpClient,
+		ctx, &getEvent{
+			S:       s,
+			EventID: eventID,
+		},
+	)
 }
 
 type getEventAuth struct {
@@ -412,135 +306,86 @@ type getEventAuth struct {
 	RoomVersion gomatrixserverlib.RoomVersion
 	RoomID      string
 	EventID     string
-	Res         *gomatrixserverlib.RespEventAuth
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) GetEventAuth(
 	ctx context.Context, s gomatrixserverlib.ServerName,
 	roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string,
 ) (gomatrixserverlib.RespEventAuth, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetEventAuth")
-	defer span.Finish()
-
-	request := getEventAuth{
-		S:           s,
-		RoomVersion: roomVersion,
-		RoomID:      roomID,
-		EventID:     eventID,
-	}
-	var response getEventAuth
-	apiURL := h.federationAPIURL + FederationAPIGetEventAuthPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.RespEventAuth{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.RespEventAuth{}, response.Err
-	}
-	return *response.Res, nil
+	return httputil.CallInternalProxyAPI[getEventAuth, gomatrixserverlib.RespEventAuth, *api.FederationClientError](
+		"GetEventAuth", h.federationAPIURL+FederationAPIGetEventAuthPath, h.httpClient,
+		ctx, &getEventAuth{
+			S:           s,
+			RoomVersion: roomVersion,
+			RoomID:      roomID,
+			EventID:     eventID,
+		},
+	)
 }
 
 func (h *httpFederationInternalAPI) QueryServerKeys(
 	ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerKeys")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIQueryServerKeysPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"QueryServerKeys", h.federationAPIURL+FederationAPIQueryServerKeysPath,
+		h.httpClient, ctx, req, res,
+	)
 }
 
 type lookupServerKeys struct {
 	S           gomatrixserverlib.ServerName
 	KeyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp
-	ServerKeys  []gomatrixserverlib.ServerKeys
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupServerKeys(
 	ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
 ) ([]gomatrixserverlib.ServerKeys, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupServerKeys")
-	defer span.Finish()
-
-	request := lookupServerKeys{
-		S:           s,
-		KeyRequests: keyRequests,
-	}
-	var response lookupServerKeys
-	apiURL := h.federationAPIURL + FederationAPILookupServerKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return []gomatrixserverlib.ServerKeys{}, err
-	}
-	if response.Err != nil {
-		return []gomatrixserverlib.ServerKeys{}, response.Err
-	}
-	return response.ServerKeys, nil
+	return httputil.CallInternalProxyAPI[lookupServerKeys, []gomatrixserverlib.ServerKeys, *api.FederationClientError](
+		"LookupServerKeys", h.federationAPIURL+FederationAPILookupServerKeysPath, h.httpClient,
+		ctx, &lookupServerKeys{
+			S:           s,
+			KeyRequests: keyRequests,
+		},
+	)
 }
 
 type eventRelationships struct {
 	S       gomatrixserverlib.ServerName
 	Req     gomatrixserverlib.MSC2836EventRelationshipsRequest
 	RoomVer gomatrixserverlib.RoomVersion
-	Res     gomatrixserverlib.MSC2836EventRelationshipsResponse
-	Err     *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) MSC2836EventRelationships(
 	ctx context.Context, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest,
 	roomVersion gomatrixserverlib.RoomVersion,
 ) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "MSC2836EventRelationships")
-	defer span.Finish()
-
-	request := eventRelationships{
-		S:       s,
-		Req:     r,
-		RoomVer: roomVersion,
-	}
-	var response eventRelationships
-	apiURL := h.federationAPIURL + FederationAPIEventRelationshipsPath
-	err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return res, err
-	}
-	if response.Err != nil {
-		return res, response.Err
-	}
-	return response.Res, nil
+	return httputil.CallInternalProxyAPI[eventRelationships, gomatrixserverlib.MSC2836EventRelationshipsResponse, *api.FederationClientError](
+		"MSC2836EventRelationships", h.federationAPIURL+FederationAPIEventRelationshipsPath, h.httpClient,
+		ctx, &eventRelationships{
+			S:       s,
+			Req:     r,
+			RoomVer: roomVersion,
+		},
+	)
 }
 
 type spacesReq struct {
 	S             gomatrixserverlib.ServerName
 	SuggestedOnly bool
 	RoomID        string
-	Res           gomatrixserverlib.MSC2946SpacesResponse
-	Err           *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) MSC2946Spaces(
 	ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool,
 ) (res gomatrixserverlib.MSC2946SpacesResponse, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "MSC2946Spaces")
-	defer span.Finish()
-
-	request := spacesReq{
-		S:             dst,
-		SuggestedOnly: suggestedOnly,
-		RoomID:        roomID,
-	}
-	var response spacesReq
-	apiURL := h.federationAPIURL + FederationAPISpacesSummaryPath
-	err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return res, err
-	}
-	if response.Err != nil {
-		return res, response.Err
-	}
-	return response.Res, nil
+	return httputil.CallInternalProxyAPI[spacesReq, gomatrixserverlib.MSC2946SpacesResponse, *api.FederationClientError](
+		"MSC2836EventRelationships", h.federationAPIURL+FederationAPISpacesSummaryPath, h.httpClient,
+		ctx, &spacesReq{
+			S:             dst,
+			SuggestedOnly: suggestedOnly,
+			RoomID:        roomID,
+		},
+	)
 }
 
 func (s *httpFederationInternalAPI) KeyRing() *gomatrixserverlib.KeyRing {
@@ -614,11 +459,10 @@ func (h *httpFederationInternalAPI) InputPublicKeys(
 	request *api.InputPublicKeysRequest,
 	response *api.InputPublicKeysResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIInputPublicKeyPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"InputPublicKey", h.federationAPIURL+FederationAPIInputPublicKeyPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpFederationInternalAPI) QueryPublicKeys(
@@ -626,9 +470,8 @@ func (h *httpFederationInternalAPI) QueryPublicKeys(
 	request *api.QueryPublicKeysRequest,
 	response *api.QueryPublicKeysResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey")
-	defer span.Finish()
-
-	apiURL := h.federationAPIURL + FederationAPIQueryPublicKeyPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryPublicKeys", h.federationAPIURL+FederationAPIQueryPublicKeyPath,
+		h.httpClient, ctx, request, response,
+	)
 }
diff --git a/federationapi/inthttp/server.go b/federationapi/inthttp/server.go
index 28e52b32d..a8b829a71 100644
--- a/federationapi/inthttp/server.go
+++ b/federationapi/inthttp/server.go
@@ -1,12 +1,14 @@
 package inthttp
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
@@ -15,372 +17,180 @@ import (
 func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIQueryJoinedHostServerNamesInRoomPath,
-		httputil.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryJoinedHostServerNamesInRoomRequest
-			var response api.QueryJoinedHostServerNamesInRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := intAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(
-		FederationAPIPerformJoinRequestPath,
-		httputil.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse {
-			var request api.PerformJoinRequest
-			var response api.PerformJoinResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			intAPI.PerformJoin(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(
-		FederationAPIPerformLeaveRequestPath,
-		httputil.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse {
-			var request api.PerformLeaveRequest
-			var response api.PerformLeaveResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformLeave(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", intAPI.QueryJoinedHostServerNamesInRoom),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIPerformInviteRequestPath,
-		httputil.MakeInternalAPI("PerformInviteRequest", func(req *http.Request) util.JSONResponse {
-			var request api.PerformInviteRequest
-			var response api.PerformInviteResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformInvite(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", intAPI.PerformInvite),
 	)
+
+	internalAPIMux.Handle(
+		FederationAPIPerformLeaveRequestPath,
+		httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", intAPI.PerformLeave),
+	)
+
 	internalAPIMux.Handle(
 		FederationAPIPerformDirectoryLookupRequestPath,
-		httputil.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse {
-			var request api.PerformDirectoryLookupRequest
-			var response api.PerformDirectoryLookupResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformDirectoryLookup(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", intAPI.PerformDirectoryLookup),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIPerformBroadcastEDUPath,
-		httputil.MakeInternalAPI("PerformBroadcastEDU", func(req *http.Request) util.JSONResponse {
-			var request api.PerformBroadcastEDURequest
-			var response api.PerformBroadcastEDUResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformBroadcastEDU(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", intAPI.PerformBroadcastEDU),
 	)
+
+	internalAPIMux.Handle(
+		FederationAPIPerformJoinRequestPath,
+		httputil.MakeInternalRPCAPI(
+			"FederationAPIPerformJoinRequest",
+			func(ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse) error {
+				intAPI.PerformJoin(ctx, req, res)
+				return nil
+			},
+		),
+	)
+
 	internalAPIMux.Handle(
 		FederationAPIGetUserDevicesPath,
-		httputil.MakeInternalAPI("GetUserDevices", func(req *http.Request) util.JSONResponse {
-			var request getUserDevices
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.GetUserDevices(req.Context(), request.S, request.UserID)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIGetUserDevices",
+			func(ctx context.Context, req *getUserDevices) (*gomatrixserverlib.RespUserDevices, error) {
+				res, err := intAPI.GetUserDevices(ctx, req.S, req.UserID)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIClaimKeysPath,
-		httputil.MakeInternalAPI("ClaimKeys", func(req *http.Request) util.JSONResponse {
-			var request claimKeys
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.ClaimKeys(req.Context(), request.S, request.OneTimeKeys)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIClaimKeys",
+			func(ctx context.Context, req *claimKeys) (*gomatrixserverlib.RespClaimKeys, error) {
+				res, err := intAPI.ClaimKeys(ctx, req.S, req.OneTimeKeys)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIQueryKeysPath,
-		httputil.MakeInternalAPI("QueryKeys", func(req *http.Request) util.JSONResponse {
-			var request queryKeys
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.QueryKeys(req.Context(), request.S, request.Keys)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIQueryKeys",
+			func(ctx context.Context, req *queryKeys) (*gomatrixserverlib.RespQueryKeys, error) {
+				res, err := intAPI.QueryKeys(ctx, req.S, req.Keys)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIBackfillPath,
-		httputil.MakeInternalAPI("Backfill", func(req *http.Request) util.JSONResponse {
-			var request backfill
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.Backfill(req.Context(), request.S, request.RoomID, request.Limit, request.EventIDs)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIBackfill",
+			func(ctx context.Context, req *backfill) (*gomatrixserverlib.Transaction, error) {
+				res, err := intAPI.Backfill(ctx, req.S, req.RoomID, req.Limit, req.EventIDs)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupStatePath,
-		httputil.MakeInternalAPI("LookupState", func(req *http.Request) util.JSONResponse {
-			var request lookupState
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.LookupState(req.Context(), request.S, request.RoomID, request.EventID, request.RoomVersion)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPILookupState",
+			func(ctx context.Context, req *lookupState) (*gomatrixserverlib.RespState, error) {
+				res, err := intAPI.LookupState(ctx, req.S, req.RoomID, req.EventID, req.RoomVersion)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupStateIDsPath,
-		httputil.MakeInternalAPI("LookupStateIDs", func(req *http.Request) util.JSONResponse {
-			var request lookupStateIDs
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.LookupStateIDs(req.Context(), request.S, request.RoomID, request.EventID)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPILookupStateIDs",
+			func(ctx context.Context, req *lookupStateIDs) (*gomatrixserverlib.RespStateIDs, error) {
+				res, err := intAPI.LookupStateIDs(ctx, req.S, req.RoomID, req.EventID)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupMissingEventsPath,
-		httputil.MakeInternalAPI("LookupMissingEvents", func(req *http.Request) util.JSONResponse {
-			var request lookupMissingEvents
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.LookupMissingEvents(req.Context(), request.S, request.RoomID, request.Missing, request.RoomVersion)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			for _, event := range res.Events {
-				js, err := json.Marshal(event)
-				if err != nil {
-					return util.MessageResponse(http.StatusInternalServerError, err.Error())
-				}
-				request.Res.Events = append(request.Res.Events, js)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPILookupMissingEvents",
+			func(ctx context.Context, req *lookupMissingEvents) (*gomatrixserverlib.RespMissingEvents, error) {
+				res, err := intAPI.LookupMissingEvents(ctx, req.S, req.RoomID, req.Missing, req.RoomVersion)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIGetEventPath,
-		httputil.MakeInternalAPI("GetEvent", func(req *http.Request) util.JSONResponse {
-			var request getEvent
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.GetEvent(req.Context(), request.S, request.EventID)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIGetEvent",
+			func(ctx context.Context, req *getEvent) (*gomatrixserverlib.Transaction, error) {
+				res, err := intAPI.GetEvent(ctx, req.S, req.EventID)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIGetEventAuthPath,
-		httputil.MakeInternalAPI("GetEventAuth", func(req *http.Request) util.JSONResponse {
-			var request getEventAuth
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.GetEventAuth(req.Context(), request.S, request.RoomVersion, request.RoomID, request.EventID)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIGetEventAuth",
+			func(ctx context.Context, req *getEventAuth) (*gomatrixserverlib.RespEventAuth, error) {
+				res, err := intAPI.GetEventAuth(ctx, req.S, req.RoomVersion, req.RoomID, req.EventID)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIQueryServerKeysPath,
-		httputil.MakeInternalAPI("QueryServerKeys", func(req *http.Request) util.JSONResponse {
-			var request api.QueryServerKeysRequest
-			var response api.QueryServerKeysResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.QueryServerKeys(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", intAPI.QueryServerKeys),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupServerKeysPath,
-		httputil.MakeInternalAPI("LookupServerKeys", func(req *http.Request) util.JSONResponse {
-			var request lookupServerKeys
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.LookupServerKeys(req.Context(), request.S, request.KeyRequests)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.ServerKeys = res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPILookupServerKeys",
+			func(ctx context.Context, req *lookupServerKeys) (*[]gomatrixserverlib.ServerKeys, error) {
+				res, err := intAPI.LookupServerKeys(ctx, req.S, req.KeyRequests)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIEventRelationshipsPath,
-		httputil.MakeInternalAPI("MSC2836EventRelationships", func(req *http.Request) util.JSONResponse {
-			var request eventRelationships
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.MSC2836EventRelationships(req.Context(), request.S, request.Req, request.RoomVer)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIMSC2836EventRelationships",
+			func(ctx context.Context, req *eventRelationships) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) {
+				res, err := intAPI.MSC2836EventRelationships(ctx, req.S, req.Req, req.RoomVer)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPISpacesSummaryPath,
-		httputil.MakeInternalAPI("MSC2946SpacesSummary", func(req *http.Request) util.JSONResponse {
-			var request spacesReq
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			res, err := intAPI.MSC2946Spaces(req.Context(), request.S, request.RoomID, request.SuggestedOnly)
-			if err != nil {
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
+		httputil.MakeInternalProxyAPI(
+			"FederationAPIMSC2946SpacesSummary",
+			func(ctx context.Context, req *spacesReq) (*gomatrixserverlib.MSC2946SpacesResponse, error) {
+				res, err := intAPI.MSC2946Spaces(ctx, req.S, req.RoomID, req.SuggestedOnly)
+				return &res, federationClientError(err)
+			},
+		),
 	)
+
+	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIQueryPublicKeyPath,
-		httputil.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", func(req *http.Request) util.JSONResponse {
 			request := api.QueryPublicKeysRequest{}
 			response := api.QueryPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -394,8 +204,10 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+
+	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIInputPublicKeyPath,
-		httputil.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIInputPublicKeys", func(req *http.Request) util.JSONResponse {
 			request := api.InputPublicKeysRequest{}
 			response := api.InputPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -408,3 +220,18 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 		}),
 	)
 }
+
+func federationClientError(err error) error {
+	switch ferr := err.(type) {
+	case nil:
+		return nil
+	case api.FederationClientError:
+		return &ferr
+	case *api.FederationClientError:
+		return ferr
+	default:
+		return &api.FederationClientError{
+			Err: err.Error(),
+		}
+	}
+}
diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go
index 2f9da1f25..ce8b06b70 100644
--- a/federationapi/routing/devices.go
+++ b/federationapi/routing/devices.go
@@ -30,9 +30,11 @@ func GetUserDevices(
 	userID string,
 ) util.JSONResponse {
 	var res keyapi.QueryDeviceMessagesResponse
-	keyAPI.QueryDeviceMessages(req.Context(), &keyapi.QueryDeviceMessagesRequest{
+	if err := keyAPI.QueryDeviceMessages(req.Context(), &keyapi.QueryDeviceMessagesRequest{
 		UserID: userID,
-	}, &res)
+	}, &res); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if res.Error != nil {
 		util.GetLogger(req.Context()).WithError(res.Error).Error("keyAPI.QueryDeviceMessages failed")
 		return jsonerror.InternalServerError()
@@ -47,7 +49,9 @@ func GetUserDevices(
 	for _, dev := range res.Devices {
 		sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID))
 	}
-	keyAPI.QuerySignatures(req.Context(), sigReq, sigRes)
+	if err := keyAPI.QuerySignatures(req.Context(), sigReq, sigRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	response := gomatrixserverlib.RespUserDevices{
 		UserID:   userID,
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 9ad3bd8eb..b48eaf78e 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -392,7 +392,7 @@ func SendJoin(
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
 	if !alreadyJoined {
 		var response api.InputRoomEventsResponse
-		rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
+		if err := rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
 			InputRoomEvents: []api.InputRoomEvent{
 				{
 					Kind:          api.KindNew,
@@ -401,7 +401,9 @@ func SendJoin(
 					TransactionID: nil,
 				},
 			},
-		}, &response)
+		}, &response); err != nil {
+			return jsonerror.InternalAPIError(httpReq.Context(), err)
+		}
 		if response.ErrMsg != "" {
 			util.GetLogger(httpReq.Context()).WithField(logrus.ErrorKey, response.ErrMsg).Error("SendEvents failed")
 			if response.NotAllowed {
diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go
index b1a9b6710..b03d4c1d6 100644
--- a/federationapi/routing/keys.go
+++ b/federationapi/routing/keys.go
@@ -19,7 +19,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/httputil"
+	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/keyserver/api"
@@ -61,9 +61,11 @@ func QueryDeviceKeys(
 	}
 
 	var queryRes api.QueryKeysResponse
-	keyAPI.QueryKeys(httpReq.Context(), &api.QueryKeysRequest{
+	if err := keyAPI.QueryKeys(httpReq.Context(), &api.QueryKeysRequest{
 		UserToDevices: qkr.DeviceKeys,
-	}, &queryRes)
+	}, &queryRes); err != nil {
+		return jsonerror.InternalAPIError(httpReq.Context(), err)
+	}
 	if queryRes.Error != nil {
 		util.GetLogger(httpReq.Context()).WithError(queryRes.Error).Error("Failed to QueryKeys")
 		return jsonerror.InternalServerError()
@@ -113,9 +115,11 @@ func ClaimOneTimeKeys(
 	}
 
 	var claimRes api.PerformClaimKeysResponse
-	keyAPI.PerformClaimKeys(httpReq.Context(), &api.PerformClaimKeysRequest{
+	if err := keyAPI.PerformClaimKeys(httpReq.Context(), &api.PerformClaimKeysRequest{
 		OneTimeKeys: cor.OneTimeKeys,
-	}, &claimRes)
+	}, &claimRes); err != nil {
+		return jsonerror.InternalAPIError(httpReq.Context(), err)
+	}
 	if claimRes.Error != nil {
 		util.GetLogger(httpReq.Context()).WithError(claimRes.Error).Error("Failed to PerformClaimKeys")
 		return jsonerror.InternalServerError()
@@ -184,7 +188,7 @@ func NotaryKeys(
 ) util.JSONResponse {
 	if req == nil {
 		req = &gomatrixserverlib.PublicKeyNotaryLookupRequest{}
-		if reqErr := httputil.UnmarshalJSONRequest(httpReq, &req); reqErr != nil {
+		if reqErr := clienthttputil.UnmarshalJSONRequest(httpReq, &req); reqErr != nil {
 			return *reqErr
 		}
 	}
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index dbaf68e5b..8e43ce959 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -277,7 +277,7 @@ func SendLeave(
 	// We are responsible for notifying other servers that the user has left
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
 	var response api.InputRoomEventsResponse
-	rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
+	if err := rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
 		InputRoomEvents: []api.InputRoomEvent{
 			{
 				Kind:          api.KindNew,
@@ -286,7 +286,9 @@ func SendLeave(
 				TransactionID: nil,
 			},
 		},
-	}, &response)
+	}, &response); err != nil {
+		return jsonerror.InternalAPIError(httpReq.Context(), err)
+	}
 
 	if response.ErrMsg != "" {
 		util.GetLogger(httpReq.Context()).WithField(logrus.ErrorKey, response.ErrMsg).WithField("not_allowed", response.NotAllowed).Error("producer.SendEvents failed")
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 43003be38..87b6fa33e 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -458,7 +458,9 @@ func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverli
 		UserID:           updatePayload.UserID,
 	}
 	uploadRes := &keyapi.PerformUploadDeviceKeysResponse{}
-	t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes)
+	if err := t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil {
+		return err
+	}
 	if uploadRes.Error != nil {
 		return uploadRes.Error
 	}
diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go
index a111580c7..1c796f542 100644
--- a/federationapi/routing/send_test.go
+++ b/federationapi/routing/send_test.go
@@ -64,11 +64,12 @@ func (t *testRoomserverAPI) InputRoomEvents(
 	ctx context.Context,
 	request *api.InputRoomEventsRequest,
 	response *api.InputRoomEventsResponse,
-) {
+) error {
 	t.inputRoomEvents = append(t.inputRoomEvents, request.InputRoomEvents...)
 	for _, ire := range request.InputRoomEvents {
 		fmt.Println("InputRoomEvents: ", ire.Event.EventID())
 	}
+	return nil
 }
 
 // Query the latest events and state for a room from the room server.
diff --git a/internal/httputil/http.go b/internal/httputil/http.go
index 4527e2b95..1e07ee33c 100644
--- a/internal/httputil/http.go
+++ b/internal/httputil/http.go
@@ -19,19 +19,21 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
 
-	"github.com/matrix-org/dendrite/userapi/api"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 )
 
-// PostJSON performs a POST request with JSON on an internal HTTP API
-func PostJSON(
+// PostJSON performs a POST request with JSON on an internal HTTP API.
+// The error will match the errtype if returned from the remote API, or
+// will be a different type if there was a problem reaching the API.
+func PostJSON[reqtype, restype any, errtype error](
 	ctx context.Context, span opentracing.Span, httpClient *http.Client,
-	apiURL string, request, response interface{},
+	apiURL string, request *reqtype, response *restype,
 ) error {
 	jsonBytes, err := json.Marshal(request)
 	if err != nil {
@@ -69,17 +71,23 @@ func PostJSON(
 	if err != nil {
 		return err
 	}
-	if res.StatusCode != http.StatusOK {
-		var errorBody struct {
-			Message string `json:"message"`
-		}
-		if _, ok := response.(*api.PerformKeyBackupResponse); ok { // TODO: remove this, once cross-boundary errors are a thing
-			return nil
-		}
-		if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil {
-			return fmt.Errorf("internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message)
-		}
-		return fmt.Errorf("internal API: %d from %s", res.StatusCode, apiURL)
+	var body []byte
+	body, err = io.ReadAll(res.Body)
+	if err != nil {
+		return err
 	}
-	return json.NewDecoder(res.Body).Decode(response)
+	if res.StatusCode != http.StatusOK {
+		if len(body) == 0 {
+			return fmt.Errorf("HTTP %d from %s (no response body)", res.StatusCode, apiURL)
+		}
+		var reserr errtype
+		if err = json.Unmarshal(body, reserr); err != nil {
+			return fmt.Errorf("HTTP %d from %s", res.StatusCode, apiURL)
+		}
+		return reserr
+	}
+	if err = json.Unmarshal(body, response); err != nil {
+		return fmt.Errorf("json.Unmarshal: %w", err)
+	}
+	return nil
 }
diff --git a/internal/httputil/internalapi.go b/internal/httputil/internalapi.go
new file mode 100644
index 000000000..385092d9c
--- /dev/null
+++ b/internal/httputil/internalapi.go
@@ -0,0 +1,93 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"reflect"
+
+	"github.com/matrix-org/util"
+	opentracing "github.com/opentracing/opentracing-go"
+)
+
+type InternalAPIError struct {
+	Type    string
+	Message string
+}
+
+func (e InternalAPIError) Error() string {
+	return fmt.Sprintf("internal API returned %q error: %s", e.Type, e.Message)
+}
+
+func MakeInternalRPCAPI[reqtype, restype any](metricsName string, f func(context.Context, *reqtype, *restype) error) http.Handler {
+	return MakeInternalAPI(metricsName, func(req *http.Request) util.JSONResponse {
+		var request reqtype
+		var response restype
+		if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			return util.MessageResponse(http.StatusBadRequest, err.Error())
+		}
+		if err := f(req.Context(), &request, &response); err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: &InternalAPIError{
+					Type:    reflect.TypeOf(err).String(),
+					Message: fmt.Sprintf("%s", err),
+				},
+			}
+		}
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: &response,
+		}
+	})
+}
+
+func MakeInternalProxyAPI[reqtype, restype any](metricsName string, f func(context.Context, *reqtype) (*restype, error)) http.Handler {
+	return MakeInternalAPI(metricsName, func(req *http.Request) util.JSONResponse {
+		var request reqtype
+		if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			return util.MessageResponse(http.StatusBadRequest, err.Error())
+		}
+		response, err := f(req.Context(), &request)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: err,
+			}
+		}
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: response,
+		}
+	})
+}
+
+func CallInternalRPCAPI[reqtype, restype any](name, url string, client *http.Client, ctx context.Context, request *reqtype, response *restype) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, name)
+	defer span.Finish()
+
+	return PostJSON[reqtype, restype, InternalAPIError](ctx, span, client, url, request, response)
+}
+
+func CallInternalProxyAPI[reqtype, restype any, errtype error](name, url string, client *http.Client, ctx context.Context, request *reqtype) (restype, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, name)
+	defer span.Finish()
+
+	var response restype
+	return response, PostJSON[reqtype, restype, errtype](ctx, span, client, url, request, &response)
+}
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index c0a1eedbb..9ba3988b9 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -38,32 +38,32 @@ type KeyInternalAPI interface {
 
 // API functions required by the clientapi
 type ClientKeyAPI interface {
-	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse)
-	PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse)
-	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse)
-	PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse)
+	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error
+	PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) error
+	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) error
+	PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse) error
 	// PerformClaimKeys claims one-time keys for use in pre-key messages
-	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse)
+	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error
 }
 
 // API functions required by the userapi
 type UserKeyAPI interface {
-	PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse)
-	PerformDeleteKeys(ctx context.Context, req *PerformDeleteKeysRequest, res *PerformDeleteKeysResponse)
+	PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) error
+	PerformDeleteKeys(ctx context.Context, req *PerformDeleteKeysRequest, res *PerformDeleteKeysResponse) error
 }
 
 // API functions required by the syncapi
 type SyncKeyAPI interface {
-	QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse)
-	QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse)
+	QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse) error
+	QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse) error
 }
 
 type FederationKeyAPI interface {
-	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse)
-	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse)
-	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse)
-	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse)
-	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse)
+	QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error
+	QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse) error
+	QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse) error
+	PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) error
+	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error
 }
 
 // KeyError is returned if there was a problem performing/querying the server
diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go
index 08bbfedb8..99859dff6 100644
--- a/keyserver/internal/cross_signing.go
+++ b/keyserver/internal/cross_signing.go
@@ -103,7 +103,7 @@ func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpos
 }
 
 // nolint:gocyclo
-func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) {
+func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) error {
 	// Find the keys to store.
 	byPurpose := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{}
 	toStore := types.CrossSigningKeyMap{}
@@ -115,7 +115,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 				Err:            "Master key sanity check failed: " + err.Error(),
 				IsInvalidParam: true,
 			}
-			return
+			return nil
 		}
 
 		byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster] = req.MasterKey
@@ -131,7 +131,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 				Err:            "Self-signing key sanity check failed: " + err.Error(),
 				IsInvalidParam: true,
 			}
-			return
+			return nil
 		}
 
 		byPurpose[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning] = req.SelfSigningKey
@@ -146,7 +146,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 				Err:            "User-signing key sanity check failed: " + err.Error(),
 				IsInvalidParam: true,
 			}
-			return
+			return nil
 		}
 
 		byPurpose[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = req.UserSigningKey
@@ -161,7 +161,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 			Err:            "No keys were supplied in the request",
 			IsMissingParam: true,
 		}
-		return
+		return nil
 	}
 
 	// We can't have a self-signing or user-signing key without a master
@@ -174,7 +174,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 		res.Error = &api.KeyError{
 			Err: "Retrieving cross-signing keys from database failed: " + err.Error(),
 		}
-		return
+		return nil
 	}
 
 	// If we still can't find a master key for the user then stop the upload.
@@ -185,7 +185,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 				Err:            "No master key was found",
 				IsMissingParam: true,
 			}
-			return
+			return nil
 		}
 	}
 
@@ -212,7 +212,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 		}
 	}
 	if !changed {
-		return
+		return nil
 	}
 
 	// Store the keys.
@@ -220,7 +220,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 		res.Error = &api.KeyError{
 			Err: fmt.Sprintf("a.DB.StoreCrossSigningKeysForUser: %s", err),
 		}
-		return
+		return nil
 	}
 
 	// Now upload any signatures that were included with the keys.
@@ -238,7 +238,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 					res.Error = &api.KeyError{
 						Err: fmt.Sprintf("a.DB.StoreCrossSigningSigsForTarget: %s", err),
 					}
-					return
+					return nil
 				}
 			}
 		}
@@ -255,17 +255,18 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 		update.SelfSigningKey = &ssk
 	}
 	if update.MasterKey == nil && update.SelfSigningKey == nil {
-		return
+		return nil
 	}
 	if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil {
 		res.Error = &api.KeyError{
 			Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err),
 		}
-		return
+		return nil
 	}
+	return nil
 }
 
-func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req *api.PerformUploadDeviceSignaturesRequest, res *api.PerformUploadDeviceSignaturesResponse) {
+func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req *api.PerformUploadDeviceSignaturesRequest, res *api.PerformUploadDeviceSignaturesResponse) error {
 	// Before we do anything, we need the master and self-signing keys for this user.
 	// Then we can verify the signatures make sense.
 	queryReq := &api.QueryKeysRequest{
@@ -276,7 +277,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req
 	for userID := range req.Signatures {
 		queryReq.UserToDevices[userID] = []string{}
 	}
-	a.QueryKeys(ctx, queryReq, queryRes)
+	_ = a.QueryKeys(ctx, queryReq, queryRes)
 
 	selfSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{}
 	otherSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{}
@@ -322,14 +323,14 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req
 		res.Error = &api.KeyError{
 			Err: fmt.Sprintf("a.processSelfSignatures: %s", err),
 		}
-		return
+		return nil
 	}
 
 	if err := a.processOtherSignatures(ctx, req.UserID, queryRes, otherSignatures); err != nil {
 		res.Error = &api.KeyError{
 			Err: fmt.Sprintf("a.processOtherSignatures: %s", err),
 		}
-		return
+		return nil
 	}
 
 	// Finally, generate a notification that we updated the signatures.
@@ -345,9 +346,10 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req
 			res.Error = &api.KeyError{
 				Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err),
 			}
-			return
+			return nil
 		}
 	}
+	return nil
 }
 
 func (a *KeyInternalAPI) processSelfSignatures(
@@ -520,7 +522,7 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase(
 	}
 }
 
-func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySignaturesRequest, res *api.QuerySignaturesResponse) {
+func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySignaturesRequest, res *api.QuerySignaturesResponse) error {
 	for targetUserID, forTargetUser := range req.TargetIDs {
 		keyMap, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID)
 		if err != nil && err != sql.ErrNoRows {
@@ -559,7 +561,7 @@ func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySign
 				res.Error = &api.KeyError{
 					Err: fmt.Sprintf("a.DB.CrossSigningSigsForTarget: %s", err),
 				}
-				return
+				return nil
 			}
 
 			for sourceUserID, forSourceUser := range sigMap {
@@ -581,4 +583,5 @@ func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySign
 			}
 		}
 	}
+	return nil
 }
diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index e317755e4..80efbec51 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -119,7 +119,7 @@ type DeviceListUpdaterDatabase interface {
 }
 
 type DeviceListUpdaterAPI interface {
-	PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse)
+	PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) error
 }
 
 // KeyChangeProducer is the interface for producers.KeyChange useful for testing.
@@ -421,7 +421,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 					uploadReq.SelfSigningKey = *res.SelfSigningKey
 				}
 			}
-			u.api.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes)
+			_ = u.api.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes)
 		}
 		err = u.updateDeviceList(&res)
 		if err != nil {
diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go
index 0c2405f34..0520a9e66 100644
--- a/keyserver/internal/device_list_update_test.go
+++ b/keyserver/internal/device_list_update_test.go
@@ -113,8 +113,8 @@ func (d *mockDeviceListUpdaterDatabase) DeviceKeysJSON(ctx context.Context, keys
 type mockDeviceListUpdaterAPI struct {
 }
 
-func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) {
-
+func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) error {
+	return nil
 }
 
 type roundTripper struct {
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 91f011517..41b4d44a4 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -48,18 +48,20 @@ func (a *KeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) {
 	a.UserAPI = i
 }
 
-func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyChangesRequest, res *api.QueryKeyChangesResponse) {
+func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyChangesRequest, res *api.QueryKeyChangesResponse) error {
 	userIDs, latest, err := a.DB.KeyChanges(ctx, req.Offset, req.ToOffset)
 	if err != nil {
 		res.Error = &api.KeyError{
 			Err: err.Error(),
 		}
+		return nil
 	}
 	res.Offset = latest
 	res.UserIDs = userIDs
+	return nil
 }
 
-func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) {
+func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) error {
 	res.KeyErrors = make(map[string]map[string]*api.KeyError)
 	if len(req.DeviceKeys) > 0 {
 		a.uploadLocalDeviceKeys(ctx, req, res)
@@ -67,9 +69,10 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform
 	if len(req.OneTimeKeys) > 0 {
 		a.uploadOneTimeKeys(ctx, req, res)
 	}
+	return nil
 }
 
-func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformClaimKeysRequest, res *api.PerformClaimKeysResponse) {
+func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformClaimKeysRequest, res *api.PerformClaimKeysResponse) error {
 	res.OneTimeKeys = make(map[string]map[string]map[string]json.RawMessage)
 	res.Failures = make(map[string]interface{})
 	// wrap request map in a top-level by-domain map
@@ -113,6 +116,7 @@ func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformC
 	if len(domainToDeviceKeys) > 0 {
 		a.claimRemoteKeys(ctx, req.Timeout, res, domainToDeviceKeys)
 	}
+	return nil
 }
 
 func (a *KeyInternalAPI) claimRemoteKeys(
@@ -172,32 +176,34 @@ func (a *KeyInternalAPI) claimRemoteKeys(
 	util.GetLogger(ctx).WithField("num_keys", keysClaimed).Info("Claimed remote keys")
 }
 
-func (a *KeyInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.PerformDeleteKeysRequest, res *api.PerformDeleteKeysResponse) {
+func (a *KeyInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.PerformDeleteKeysRequest, res *api.PerformDeleteKeysResponse) error {
 	if err := a.DB.DeleteDeviceKeys(ctx, req.UserID, req.KeyIDs); err != nil {
 		res.Error = &api.KeyError{
 			Err: fmt.Sprintf("Failed to delete device keys: %s", err),
 		}
 	}
+	return nil
 }
 
-func (a *KeyInternalAPI) QueryOneTimeKeys(ctx context.Context, req *api.QueryOneTimeKeysRequest, res *api.QueryOneTimeKeysResponse) {
+func (a *KeyInternalAPI) QueryOneTimeKeys(ctx context.Context, req *api.QueryOneTimeKeysRequest, res *api.QueryOneTimeKeysResponse) error {
 	count, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID)
 	if err != nil {
 		res.Error = &api.KeyError{
 			Err: fmt.Sprintf("Failed to query OTK counts: %s", err),
 		}
-		return
+		return nil
 	}
 	res.Count = *count
+	return nil
 }
 
-func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.QueryDeviceMessagesRequest, res *api.QueryDeviceMessagesResponse) {
+func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.QueryDeviceMessagesRequest, res *api.QueryDeviceMessagesResponse) error {
 	msgs, err := a.DB.DeviceKeysForUser(ctx, req.UserID, nil, false)
 	if err != nil {
 		res.Error = &api.KeyError{
 			Err: fmt.Sprintf("failed to query DB for device keys: %s", err),
 		}
-		return
+		return nil
 	}
 	maxStreamID := int64(0)
 	for _, m := range msgs {
@@ -215,10 +221,11 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query
 	}
 	res.Devices = result
 	res.StreamID = maxStreamID
+	return nil
 }
 
 // nolint:gocyclo
-func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) {
+func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) error {
 	res.DeviceKeys = make(map[string]map[string]json.RawMessage)
 	res.MasterKeys = make(map[string]gomatrixserverlib.CrossSigningKey)
 	res.SelfSigningKeys = make(map[string]gomatrixserverlib.CrossSigningKey)
@@ -244,7 +251,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 				res.Error = &api.KeyError{
 					Err: fmt.Sprintf("failed to query local device keys: %s", err),
 				}
-				return
+				return nil
 			}
 
 			// pull out display names after we have the keys so we handle wildcards correctly
@@ -318,7 +325,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 				// Stop executing the function if the context was canceled/the deadline was exceeded,
 				// as we can't continue without a valid context.
 				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-					return
+					return nil
 				}
 				logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
 				continue
@@ -344,7 +351,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 				// Stop executing the function if the context was canceled/the deadline was exceeded,
 				// as we can't continue without a valid context.
 				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-					return
+					return nil
 				}
 				logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed")
 				continue
@@ -372,6 +379,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques
 			}
 		}
 	}
+	return nil
 }
 
 func (a *KeyInternalAPI) remoteKeysFromDatabase(
diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go
index dac61d1ea..7a7131145 100644
--- a/keyserver/inthttp/client.go
+++ b/keyserver/inthttp/client.go
@@ -22,7 +22,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/opentracing/opentracing-go"
 )
 
 // HTTP paths for the internal HTTP APIs
@@ -68,168 +67,108 @@ func (h *httpKeyInternalAPI) PerformClaimKeys(
 	ctx context.Context,
 	request *api.PerformClaimKeysRequest,
 	response *api.PerformClaimKeysResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformClaimKeys")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformClaimKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformClaimKeys", h.apiURL+PerformClaimKeysPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) PerformDeleteKeys(
 	ctx context.Context,
 	request *api.PerformDeleteKeysRequest,
 	response *api.PerformDeleteKeysResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformClaimKeys")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformClaimKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformDeleteKeys", h.apiURL+PerformDeleteKeysPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) PerformUploadKeys(
 	ctx context.Context,
 	request *api.PerformUploadKeysRequest,
 	response *api.PerformUploadKeysResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUploadKeys")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformUploadKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformUploadKeys", h.apiURL+PerformUploadKeysPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) QueryKeys(
 	ctx context.Context,
 	request *api.QueryKeysRequest,
 	response *api.QueryKeysResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeys")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryKeys", h.apiURL+QueryKeysPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) QueryOneTimeKeys(
 	ctx context.Context,
 	request *api.QueryOneTimeKeysRequest,
 	response *api.QueryOneTimeKeysResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryOneTimeKeys")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryOneTimeKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryOneTimeKeys", h.apiURL+QueryOneTimeKeysPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) QueryDeviceMessages(
 	ctx context.Context,
 	request *api.QueryDeviceMessagesRequest,
 	response *api.QueryDeviceMessagesResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryDeviceMessages")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryDeviceMessagesPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryDeviceMessages", h.apiURL+QueryDeviceMessagesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) QueryKeyChanges(
 	ctx context.Context,
 	request *api.QueryKeyChangesRequest,
 	response *api.QueryKeyChangesResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeyChanges")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryKeyChangesPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryKeyChanges", h.apiURL+QueryKeyChangesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) PerformUploadDeviceKeys(
 	ctx context.Context,
 	request *api.PerformUploadDeviceKeysRequest,
 	response *api.PerformUploadDeviceKeysResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUploadDeviceKeys")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformUploadDeviceKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformUploadDeviceKeys", h.apiURL+PerformUploadDeviceKeysPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) PerformUploadDeviceSignatures(
 	ctx context.Context,
 	request *api.PerformUploadDeviceSignaturesRequest,
 	response *api.PerformUploadDeviceSignaturesResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUploadDeviceSignatures")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformUploadDeviceSignaturesPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformUploadDeviceSignatures", h.apiURL+PerformUploadDeviceSignaturesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpKeyInternalAPI) QuerySignatures(
 	ctx context.Context,
 	request *api.QuerySignaturesRequest,
 	response *api.QuerySignaturesResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QuerySignatures")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QuerySignaturesPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.KeyError{
-			Err: err.Error(),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QuerySignatures", h.apiURL+QuerySignaturesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go
index 5bf5976a8..4e5f9fba4 100644
--- a/keyserver/inthttp/server.go
+++ b/keyserver/inthttp/server.go
@@ -15,124 +15,59 @@
 package inthttp
 
 import (
-	"encoding/json"
-	"net/http"
-
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver/api"
-	"github.com/matrix-org/util"
 )
 
 func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) {
-	internalAPIMux.Handle(PerformClaimKeysPath,
-		httputil.MakeInternalAPI("performClaimKeys", func(req *http.Request) util.JSONResponse {
-			request := api.PerformClaimKeysRequest{}
-			response := api.PerformClaimKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.PerformClaimKeys(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		PerformClaimKeysPath,
+		httputil.MakeInternalRPCAPI("KeyserverPerformClaimKeys", s.PerformClaimKeys),
 	)
-	internalAPIMux.Handle(PerformDeleteKeysPath,
-		httputil.MakeInternalAPI("performDeleteKeys", func(req *http.Request) util.JSONResponse {
-			request := api.PerformDeleteKeysRequest{}
-			response := api.PerformDeleteKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.PerformDeleteKeys(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		PerformDeleteKeysPath,
+		httputil.MakeInternalRPCAPI("KeyserverPerformDeleteKeys", s.PerformDeleteKeys),
 	)
-	internalAPIMux.Handle(PerformUploadKeysPath,
-		httputil.MakeInternalAPI("performUploadKeys", func(req *http.Request) util.JSONResponse {
-			request := api.PerformUploadKeysRequest{}
-			response := api.PerformUploadKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.PerformUploadKeys(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		PerformUploadKeysPath,
+		httputil.MakeInternalRPCAPI("KeyserverPerformUploadKeys", s.PerformUploadKeys),
 	)
-	internalAPIMux.Handle(PerformUploadDeviceKeysPath,
-		httputil.MakeInternalAPI("performUploadDeviceKeys", func(req *http.Request) util.JSONResponse {
-			request := api.PerformUploadDeviceKeysRequest{}
-			response := api.PerformUploadDeviceKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.PerformUploadDeviceKeys(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		PerformUploadDeviceKeysPath,
+		httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceKeys", s.PerformUploadDeviceKeys),
 	)
-	internalAPIMux.Handle(PerformUploadDeviceSignaturesPath,
-		httputil.MakeInternalAPI("performUploadDeviceSignatures", func(req *http.Request) util.JSONResponse {
-			request := api.PerformUploadDeviceSignaturesRequest{}
-			response := api.PerformUploadDeviceSignaturesResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.PerformUploadDeviceSignatures(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		PerformUploadDeviceSignaturesPath,
+		httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceSignatures", s.PerformUploadDeviceSignatures),
 	)
-	internalAPIMux.Handle(QueryKeysPath,
-		httputil.MakeInternalAPI("queryKeys", func(req *http.Request) util.JSONResponse {
-			request := api.QueryKeysRequest{}
-			response := api.QueryKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.QueryKeys(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryKeysPath,
+		httputil.MakeInternalRPCAPI("KeyserverQueryKeys", s.QueryKeys),
 	)
-	internalAPIMux.Handle(QueryOneTimeKeysPath,
-		httputil.MakeInternalAPI("queryOneTimeKeys", func(req *http.Request) util.JSONResponse {
-			request := api.QueryOneTimeKeysRequest{}
-			response := api.QueryOneTimeKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.QueryOneTimeKeys(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryOneTimeKeysPath,
+		httputil.MakeInternalRPCAPI("KeyserverQueryOneTimeKeys", s.QueryOneTimeKeys),
 	)
-	internalAPIMux.Handle(QueryDeviceMessagesPath,
-		httputil.MakeInternalAPI("queryDeviceMessages", func(req *http.Request) util.JSONResponse {
-			request := api.QueryDeviceMessagesRequest{}
-			response := api.QueryDeviceMessagesResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.QueryDeviceMessages(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryDeviceMessagesPath,
+		httputil.MakeInternalRPCAPI("KeyserverQueryDeviceMessages", s.QueryDeviceMessages),
 	)
-	internalAPIMux.Handle(QueryKeyChangesPath,
-		httputil.MakeInternalAPI("queryKeyChanges", func(req *http.Request) util.JSONResponse {
-			request := api.QueryKeyChangesRequest{}
-			response := api.QueryKeyChangesResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.QueryKeyChanges(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryKeyChangesPath,
+		httputil.MakeInternalRPCAPI("KeyserverQueryKeyChanges", s.QueryKeyChanges),
 	)
-	internalAPIMux.Handle(QuerySignaturesPath,
-		httputil.MakeInternalAPI("querySignatures", func(req *http.Request) util.JSONResponse {
-			request := api.QuerySignaturesRequest{}
-			response := api.QuerySignaturesResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.QuerySignatures(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QuerySignaturesPath,
+		httputil.MakeInternalRPCAPI("KeyserverQuerySignatures", s.QuerySignatures),
 	)
 }
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 38baa617f..ee0212ecf 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -40,7 +40,7 @@ type InputRoomEventsAPI interface {
 		ctx context.Context,
 		req *InputRoomEventsRequest,
 		res *InputRoomEventsResponse,
-	)
+	) error
 }
 
 // Query the latest events and state for a room from the room server.
@@ -139,15 +139,15 @@ type ClientRoomserverAPI interface {
 	GetAliasesForRoomID(ctx context.Context, req *GetAliasesForRoomIDRequest, res *GetAliasesForRoomIDResponse) error
 
 	// PerformRoomUpgrade upgrades a room to a newer version
-	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse)
-	PerformAdminEvacuateRoom(ctx context.Context, req *PerformAdminEvacuateRoomRequest, res *PerformAdminEvacuateRoomResponse)
-	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse)
-	PerformPeek(ctx context.Context, req *PerformPeekRequest, res *PerformPeekResponse)
-	PerformUnpeek(ctx context.Context, req *PerformUnpeekRequest, res *PerformUnpeekResponse)
+	PerformRoomUpgrade(ctx context.Context, req *PerformRoomUpgradeRequest, resp *PerformRoomUpgradeResponse) error
+	PerformAdminEvacuateRoom(ctx context.Context, req *PerformAdminEvacuateRoomRequest, res *PerformAdminEvacuateRoomResponse) error
+	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse) error
+	PerformPeek(ctx context.Context, req *PerformPeekRequest, res *PerformPeekResponse) error
+	PerformUnpeek(ctx context.Context, req *PerformUnpeekRequest, res *PerformUnpeekResponse) error
 	PerformInvite(ctx context.Context, req *PerformInviteRequest, res *PerformInviteResponse) error
-	PerformJoin(ctx context.Context, req *PerformJoinRequest, res *PerformJoinResponse)
+	PerformJoin(ctx context.Context, req *PerformJoinRequest, res *PerformJoinResponse) error
 	PerformLeave(ctx context.Context, req *PerformLeaveRequest, res *PerformLeaveResponse) error
-	PerformPublish(ctx context.Context, req *PerformPublishRequest, res *PerformPublishResponse)
+	PerformPublish(ctx context.Context, req *PerformPublishRequest, res *PerformPublishResponse) error
 	// PerformForget forgets a rooms history for a specific user
 	PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error
 	SetRoomAlias(ctx context.Context, req *SetRoomAliasRequest, res *SetRoomAliasResponse) error
@@ -158,7 +158,7 @@ type UserRoomserverAPI interface {
 	QueryLatestEventsAndStateAPI
 	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
 	QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error
-	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse)
+	PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse) error
 }
 
 type FederationRoomserverAPI interface {
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index 211f320ff..18a617331 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -35,9 +35,10 @@ func (t *RoomserverInternalAPITrace) InputRoomEvents(
 	ctx context.Context,
 	req *InputRoomEventsRequest,
 	res *InputRoomEventsResponse,
-) {
-	t.Impl.InputRoomEvents(ctx, req, res)
-	util.GetLogger(ctx).Infof("InputRoomEvents req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.InputRoomEvents(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("InputRoomEvents req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformInvite(
@@ -45,44 +46,49 @@ func (t *RoomserverInternalAPITrace) PerformInvite(
 	req *PerformInviteRequest,
 	res *PerformInviteResponse,
 ) error {
-	util.GetLogger(ctx).Infof("PerformInvite req=%+v res=%+v", js(req), js(res))
-	return t.Impl.PerformInvite(ctx, req, res)
+	err := t.Impl.PerformInvite(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformInvite req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformPeek(
 	ctx context.Context,
 	req *PerformPeekRequest,
 	res *PerformPeekResponse,
-) {
-	t.Impl.PerformPeek(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformPeek req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.PerformPeek(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformPeek req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformUnpeek(
 	ctx context.Context,
 	req *PerformUnpeekRequest,
 	res *PerformUnpeekResponse,
-) {
-	t.Impl.PerformUnpeek(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.PerformUnpeek(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformRoomUpgrade(
 	ctx context.Context,
 	req *PerformRoomUpgradeRequest,
 	res *PerformRoomUpgradeResponse,
-) {
-	t.Impl.PerformRoomUpgrade(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformRoomUpgrade req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.PerformRoomUpgrade(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformRoomUpgrade req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformJoin(
 	ctx context.Context,
 	req *PerformJoinRequest,
 	res *PerformJoinResponse,
-) {
-	t.Impl.PerformJoin(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformJoin req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.PerformJoin(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformJoin req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformLeave(
@@ -99,27 +105,30 @@ func (t *RoomserverInternalAPITrace) PerformPublish(
 	ctx context.Context,
 	req *PerformPublishRequest,
 	res *PerformPublishResponse,
-) {
-	t.Impl.PerformPublish(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformPublish req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.PerformPublish(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformPublish req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformAdminEvacuateRoom(
 	ctx context.Context,
 	req *PerformAdminEvacuateRoomRequest,
 	res *PerformAdminEvacuateRoomResponse,
-) {
-	t.Impl.PerformAdminEvacuateRoom(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformAdminEvacuateRoom req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.PerformAdminEvacuateRoom(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformAdminEvacuateRoom req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformAdminEvacuateUser(
 	ctx context.Context,
 	req *PerformAdminEvacuateUserRequest,
 	res *PerformAdminEvacuateUserResponse,
-) {
-	t.Impl.PerformAdminEvacuateUser(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformAdminEvacuateUser req=%+v res=%+v", js(req), js(res))
+) error {
+	err := t.Impl.PerformAdminEvacuateUser(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("PerformAdminEvacuateUser req=%+v res=%+v", js(req), js(res))
+	return err
 }
 
 func (t *RoomserverInternalAPITrace) PerformInboundPeek(
@@ -128,7 +137,7 @@ func (t *RoomserverInternalAPITrace) PerformInboundPeek(
 	res *PerformInboundPeekResponse,
 ) error {
 	err := t.Impl.PerformInboundPeek(ctx, req, res)
-	util.GetLogger(ctx).Infof("PerformInboundPeek req=%+v res=%+v", js(req), js(res))
+	util.GetLogger(ctx).WithError(err).Infof("PerformInboundPeek req=%+v res=%+v", js(req), js(res))
 	return err
 }
 
diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go
index 344e9b079..bc2f28176 100644
--- a/roomserver/api/wrapper.go
+++ b/roomserver/api/wrapper.go
@@ -90,7 +90,9 @@ func SendInputRoomEvents(
 		Asynchronous:    async,
 	}
 	var response InputRoomEventsResponse
-	rsAPI.InputRoomEvents(ctx, &request, &response)
+	if err := rsAPI.InputRoomEvents(ctx, &request, &response); err != nil {
+		return err
+	}
 	return response.Err()
 }
 
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 339c9796c..8d24f3c59 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -337,18 +337,18 @@ func (r *Inputer) InputRoomEvents(
 	ctx context.Context,
 	request *api.InputRoomEventsRequest,
 	response *api.InputRoomEventsResponse,
-) {
+) error {
 	// Queue up the event into the roomserver.
 	replySub, err := r.queueInputRoomEvents(ctx, request)
 	if err != nil {
 		response.ErrMsg = err.Error()
-		return
+		return nil
 	}
 
 	// If we aren't waiting for synchronous responses then we can
 	// give up here, there is nothing further to do.
 	if replySub == nil {
-		return
+		return nil
 	}
 
 	// Otherwise, we'll want to sit and wait for the responses
@@ -360,12 +360,14 @@ func (r *Inputer) InputRoomEvents(
 		msg, err := replySub.NextMsgWithContext(ctx)
 		if err != nil {
 			response.ErrMsg = err.Error()
-			return
+			return nil
 		}
 		if len(msg.Data) > 0 {
 			response.ErrMsg = string(msg.Data)
 		}
 	}
+
+	return nil
 }
 
 var roomserverInputBackpressure = prometheus.NewGaugeVec(
diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go
index 6c7c6c98b..cb6b22d32 100644
--- a/roomserver/internal/perform/perform_admin.go
+++ b/roomserver/internal/perform/perform_admin.go
@@ -43,21 +43,21 @@ func (r *Admin) PerformAdminEvacuateRoom(
 	ctx context.Context,
 	req *api.PerformAdminEvacuateRoomRequest,
 	res *api.PerformAdminEvacuateRoomResponse,
-) {
+) error {
 	roomInfo, err := r.DB.RoomInfo(ctx, req.RoomID)
 	if err != nil {
 		res.Error = &api.PerformError{
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("r.DB.RoomInfo: %s", err),
 		}
-		return
+		return nil
 	}
 	if roomInfo == nil || roomInfo.IsStub() {
 		res.Error = &api.PerformError{
 			Code: api.PerformErrorNoRoom,
 			Msg:  fmt.Sprintf("Room %s not found", req.RoomID),
 		}
-		return
+		return nil
 	}
 
 	memberNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, roomInfo.RoomNID, true, true)
@@ -66,7 +66,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("r.DB.GetMembershipEventNIDsForRoom: %s", err),
 		}
-		return
+		return nil
 	}
 
 	memberEvents, err := r.DB.Events(ctx, memberNIDs)
@@ -75,7 +75,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("r.DB.Events: %s", err),
 		}
-		return
+		return nil
 	}
 
 	inputEvents := make([]api.InputRoomEvent, 0, len(memberEvents))
@@ -89,7 +89,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("r.Queryer.QueryLatestEventsAndState: %s", err),
 		}
-		return
+		return nil
 	}
 
 	prevEvents := latestRes.LatestEvents
@@ -104,7 +104,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 				Code: api.PerformErrorBadRequest,
 				Msg:  fmt.Sprintf("json.Unmarshal: %s", err),
 			}
-			return
+			return nil
 		}
 		memberContent.Membership = gomatrixserverlib.Leave
 
@@ -122,7 +122,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 				Code: api.PerformErrorBadRequest,
 				Msg:  fmt.Sprintf("json.Marshal: %s", err),
 			}
-			return
+			return nil
 		}
 
 		eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(fledglingEvent)
@@ -131,7 +131,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 				Code: api.PerformErrorBadRequest,
 				Msg:  fmt.Sprintf("gomatrixserverlib.StateNeededForEventBuilder: %s", err),
 			}
-			return
+			return nil
 		}
 
 		event, err := eventutil.BuildEvent(ctx, fledglingEvent, r.Cfg.Matrix, time.Now(), &eventsNeeded, latestRes)
@@ -140,7 +140,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 				Code: api.PerformErrorBadRequest,
 				Msg:  fmt.Sprintf("eventutil.BuildEvent: %s", err),
 			}
-			return
+			return nil
 		}
 
 		inputEvents = append(inputEvents, api.InputRoomEvent{
@@ -160,28 +160,28 @@ func (r *Admin) PerformAdminEvacuateRoom(
 		Asynchronous:    true,
 	}
 	inputRes := &api.InputRoomEventsResponse{}
-	r.Inputer.InputRoomEvents(ctx, inputReq, inputRes)
+	return r.Inputer.InputRoomEvents(ctx, inputReq, inputRes)
 }
 
 func (r *Admin) PerformAdminEvacuateUser(
 	ctx context.Context,
 	req *api.PerformAdminEvacuateUserRequest,
 	res *api.PerformAdminEvacuateUserResponse,
-) {
+) error {
 	_, domain, err := gomatrixserverlib.SplitID('@', req.UserID)
 	if err != nil {
 		res.Error = &api.PerformError{
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("Malformed user ID: %s", err),
 		}
-		return
+		return nil
 	}
 	if domain != r.Cfg.Matrix.ServerName {
 		res.Error = &api.PerformError{
 			Code: api.PerformErrorBadRequest,
 			Msg:  "Can only evacuate local users using this endpoint",
 		}
-		return
+		return nil
 	}
 
 	roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, gomatrixserverlib.Join)
@@ -190,7 +190,7 @@ func (r *Admin) PerformAdminEvacuateUser(
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("r.DB.GetRoomsByMembership: %s", err),
 		}
-		return
+		return nil
 	}
 
 	inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, gomatrixserverlib.Invite)
@@ -199,7 +199,7 @@ func (r *Admin) PerformAdminEvacuateUser(
 			Code: api.PerformErrorBadRequest,
 			Msg:  fmt.Sprintf("r.DB.GetRoomsByMembership: %s", err),
 		}
-		return
+		return nil
 	}
 
 	for _, roomID := range append(roomIDs, inviteRoomIDs...) {
@@ -214,7 +214,7 @@ func (r *Admin) PerformAdminEvacuateUser(
 				Code: api.PerformErrorBadRequest,
 				Msg:  fmt.Sprintf("r.Leaver.PerformLeave: %s", err),
 			}
-			return
+			return nil
 		}
 		if len(outputEvents) == 0 {
 			continue
@@ -224,9 +224,10 @@ func (r *Admin) PerformAdminEvacuateUser(
 				Code: api.PerformErrorBadRequest,
 				Msg:  fmt.Sprintf("r.Inputer.WriteOutputEvents: %s", err),
 			}
-			return
+			return nil
 		}
 
 		res.Affected = append(res.Affected, roomID)
 	}
+	return nil
 }
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index e1ff4eabb..483e78c3f 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -241,7 +241,9 @@ func (r *Inviter) PerformInvite(
 		},
 	}
 	inputRes := &api.InputRoomEventsResponse{}
-	r.Inputer.InputRoomEvents(context.Background(), inputReq, inputRes)
+	if err = r.Inputer.InputRoomEvents(context.Background(), inputReq, inputRes); err != nil {
+		return nil, fmt.Errorf("r.Inputer.InputRoomEvents: %w", err)
+	}
 	if err = inputRes.Err(); err != nil {
 		res.Error = &api.PerformError{
 			Msg:  fmt.Sprintf("r.InputRoomEvents: %s", err.Error()),
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 1445b4088..43be54beb 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -52,7 +52,7 @@ func (r *Joiner) PerformJoin(
 	ctx context.Context,
 	req *rsAPI.PerformJoinRequest,
 	res *rsAPI.PerformJoinResponse,
-) {
+) error {
 	logger := logrus.WithContext(ctx).WithFields(logrus.Fields{
 		"room_id": req.RoomIDOrAlias,
 		"user_id": req.UserID,
@@ -71,11 +71,12 @@ func (r *Joiner) PerformJoin(
 				Msg: err.Error(),
 			}
 		}
-		return
+		return nil
 	}
 	logger.Info("User joined room successfully")
 	res.RoomID = roomID
 	res.JoinedVia = joinedVia
+	return nil
 }
 
 func (r *Joiner) performJoin(
@@ -291,7 +292,12 @@ func (r *Joiner) performJoinRoomByID(
 				},
 			}
 			inputRes := rsAPI.InputRoomEventsResponse{}
-			r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes)
+			if err = r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes); err != nil {
+				return "", "", &rsAPI.PerformError{
+					Code: rsAPI.PerformErrorNoOperation,
+					Msg:  fmt.Sprintf("InputRoomEvents failed: %s", err),
+				}
+			}
 			if err = inputRes.Err(); err != nil {
 				return "", "", &rsAPI.PerformError{
 					Code: rsAPI.PerformErrorNotAllowed,
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index 56e7240d0..036404cd2 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -186,7 +186,9 @@ func (r *Leaver) performLeaveRoomByID(
 		},
 	}
 	inputRes := api.InputRoomEventsResponse{}
-	r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes)
+	if err = r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes); err != nil {
+		return nil, fmt.Errorf("r.Inputer.InputRoomEvents: %w", err)
+	}
 	if err = inputRes.Err(); err != nil {
 		return nil, fmt.Errorf("r.InputRoomEvents: %w", err)
 	}
diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go
index 5560916b2..74d87a5b4 100644
--- a/roomserver/internal/perform/perform_peek.go
+++ b/roomserver/internal/perform/perform_peek.go
@@ -44,7 +44,7 @@ func (r *Peeker) PerformPeek(
 	ctx context.Context,
 	req *api.PerformPeekRequest,
 	res *api.PerformPeekResponse,
-) {
+) error {
 	roomID, err := r.performPeek(ctx, req)
 	if err != nil {
 		perr, ok := err.(*api.PerformError)
@@ -57,6 +57,7 @@ func (r *Peeker) PerformPeek(
 		}
 	}
 	res.RoomID = roomID
+	return nil
 }
 
 func (r *Peeker) performPeek(
diff --git a/roomserver/internal/perform/perform_publish.go b/roomserver/internal/perform/perform_publish.go
index 6ff42ac1a..1631fc657 100644
--- a/roomserver/internal/perform/perform_publish.go
+++ b/roomserver/internal/perform/perform_publish.go
@@ -29,11 +29,12 @@ func (r *Publisher) PerformPublish(
 	ctx context.Context,
 	req *api.PerformPublishRequest,
 	res *api.PerformPublishResponse,
-) {
+) error {
 	err := r.DB.PublishRoom(ctx, req.RoomID, req.Visibility == "public")
 	if err != nil {
 		res.Error = &api.PerformError{
 			Msg: err.Error(),
 		}
 	}
+	return nil
 }
diff --git a/roomserver/internal/perform/perform_unpeek.go b/roomserver/internal/perform/perform_unpeek.go
index 1fe8d5a0f..49e9067c9 100644
--- a/roomserver/internal/perform/perform_unpeek.go
+++ b/roomserver/internal/perform/perform_unpeek.go
@@ -41,7 +41,7 @@ func (r *Unpeeker) PerformUnpeek(
 	ctx context.Context,
 	req *api.PerformUnpeekRequest,
 	res *api.PerformUnpeekResponse,
-) {
+) error {
 	if err := r.performUnpeek(ctx, req); err != nil {
 		perr, ok := err.(*api.PerformError)
 		if ok {
@@ -52,6 +52,7 @@ func (r *Unpeeker) PerformUnpeek(
 			}
 		}
 	}
+	return nil
 }
 
 func (r *Unpeeker) performUnpeek(
diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go
index 393d7dd14..d6dc9708c 100644
--- a/roomserver/internal/perform/perform_upgrade.go
+++ b/roomserver/internal/perform/perform_upgrade.go
@@ -45,12 +45,13 @@ func (r *Upgrader) PerformRoomUpgrade(
 	ctx context.Context,
 	req *api.PerformRoomUpgradeRequest,
 	res *api.PerformRoomUpgradeResponse,
-) {
+) error {
 	res.NewRoomID, res.Error = r.performRoomUpgrade(ctx, req)
 	if res.Error != nil {
 		res.NewRoomID = ""
 		logrus.WithContext(ctx).WithError(res.Error).Error("Room upgrade failed")
 	}
+	return nil
 }
 
 func (r *Upgrader) performRoomUpgrade(
@@ -286,22 +287,24 @@ func publishNewRoomAndUnpublishOldRoom(
 ) {
 	// expose this room in the published room list
 	var pubNewRoomRes api.PerformPublishResponse
-	URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
+	if err := URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
 		RoomID:     newRoomID,
 		Visibility: "public",
-	}, &pubNewRoomRes)
-	if pubNewRoomRes.Error != nil {
+	}, &pubNewRoomRes); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("failed to reach internal API")
+	} else if pubNewRoomRes.Error != nil {
 		// treat as non-fatal since the room is already made by this point
 		util.GetLogger(ctx).WithError(pubNewRoomRes.Error).Error("failed to visibility:public")
 	}
 
 	var unpubOldRoomRes api.PerformPublishResponse
 	// remove the old room from the published room list
-	URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
+	if err := URSAPI.PerformPublish(ctx, &api.PerformPublishRequest{
 		RoomID:     oldRoomID,
 		Visibility: "private",
-	}, &unpubOldRoomRes)
-	if unpubOldRoomRes.Error != nil {
+	}, &unpubOldRoomRes); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("failed to reach internal API")
+	} else if unpubOldRoomRes.Error != nil {
 		// treat as non-fatal since the room is already made by this point
 		util.GetLogger(ctx).WithError(unpubOldRoomRes.Error).Error("failed to visibility:private")
 	}
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index 2fa8afc49..d16f67c69 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -3,7 +3,6 @@ package inthttp
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/http"
 
 	asAPI "github.com/matrix-org/dendrite/appservice/api"
@@ -14,7 +13,6 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/opentracing/opentracing-go"
 )
 
 const (
@@ -106,11 +104,10 @@ func (h *httpRoomserverInternalAPI) SetRoomAlias(
 	request *api.SetRoomAliasRequest,
 	response *api.SetRoomAliasResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "SetRoomAlias")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverSetRoomAliasPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"SetRoomAlias", h.roomserverURL+RoomserverSetRoomAliasPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // GetRoomIDForAlias implements RoomserverAliasAPI
@@ -119,11 +116,10 @@ func (h *httpRoomserverInternalAPI) GetRoomIDForAlias(
 	request *api.GetRoomIDForAliasRequest,
 	response *api.GetRoomIDForAliasResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetRoomIDForAlias")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverGetRoomIDForAliasPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"GetRoomIDForAlias", h.roomserverURL+RoomserverGetRoomIDForAliasPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // GetAliasesForRoomID implements RoomserverAliasAPI
@@ -132,11 +128,10 @@ func (h *httpRoomserverInternalAPI) GetAliasesForRoomID(
 	request *api.GetAliasesForRoomIDRequest,
 	response *api.GetAliasesForRoomIDResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetAliasesForRoomID")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverGetAliasesForRoomIDPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"GetAliasesForRoomID", h.roomserverURL+RoomserverGetAliasesForRoomIDPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // RemoveRoomAlias implements RoomserverAliasAPI
@@ -145,11 +140,10 @@ func (h *httpRoomserverInternalAPI) RemoveRoomAlias(
 	request *api.RemoveRoomAliasRequest,
 	response *api.RemoveRoomAliasResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "RemoveRoomAlias")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverRemoveRoomAliasPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"RemoveRoomAlias", h.roomserverURL+RoomserverRemoveRoomAliasPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // InputRoomEvents implements RoomserverInputAPI
@@ -157,15 +151,14 @@ func (h *httpRoomserverInternalAPI) InputRoomEvents(
 	ctx context.Context,
 	request *api.InputRoomEventsRequest,
 	response *api.InputRoomEventsResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputRoomEvents")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverInputRoomEventsPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
+) error {
+	if err := httputil.CallInternalRPCAPI(
+		"InputRoomEvents", h.roomserverURL+RoomserverInputRoomEventsPath,
+		h.httpClient, ctx, request, response,
+	); err != nil {
 		response.ErrMsg = err.Error()
 	}
+	return nil
 }
 
 func (h *httpRoomserverInternalAPI) PerformInvite(
@@ -173,45 +166,32 @@ func (h *httpRoomserverInternalAPI) PerformInvite(
 	request *api.PerformInviteRequest,
 	response *api.PerformInviteResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformInvite")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformInvitePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformInvite", h.roomserverURL+RoomserverPerformInvitePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformJoin(
 	ctx context.Context,
 	request *api.PerformJoinRequest,
 	response *api.PerformJoinResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoin")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformJoinPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.PerformError{
-			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformJoin", h.roomserverURL+RoomserverPerformJoinPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformPeek(
 	ctx context.Context,
 	request *api.PerformPeekRequest,
 	response *api.PerformPeekResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPeek")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformPeekPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.PerformError{
-			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformPeek", h.roomserverURL+RoomserverPerformPeekPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformInboundPeek(
@@ -219,45 +199,32 @@ func (h *httpRoomserverInternalAPI) PerformInboundPeek(
 	request *api.PerformInboundPeekRequest,
 	response *api.PerformInboundPeekResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformInboundPeek")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformInboundPeekPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformInboundPeek", h.roomserverURL+RoomserverPerformInboundPeekPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformUnpeek(
 	ctx context.Context,
 	request *api.PerformUnpeekRequest,
 	response *api.PerformUnpeekResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUnpeek")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformUnpeekPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.PerformError{
-			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformUnpeek", h.roomserverURL+RoomserverPerformUnpeekPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformRoomUpgrade(
 	ctx context.Context,
 	request *api.PerformRoomUpgradeRequest,
 	response *api.PerformRoomUpgradeResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformRoomUpgrade")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformRoomUpgradePath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
-		response.Error = &api.PerformError{
-			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
-		}
-	}
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformRoomUpgrade", h.roomserverURL+RoomserverPerformRoomUpgradePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformLeave(
@@ -265,62 +232,43 @@ func (h *httpRoomserverInternalAPI) PerformLeave(
 	request *api.PerformLeaveRequest,
 	response *api.PerformLeaveResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeave")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformLeavePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformLeave", h.roomserverURL+RoomserverPerformLeavePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformPublish(
 	ctx context.Context,
-	req *api.PerformPublishRequest,
-	res *api.PerformPublishResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPublish")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformPublishPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
-	if err != nil {
-		res.Error = &api.PerformError{
-			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
-		}
-	}
+	request *api.PerformPublishRequest,
+	response *api.PerformPublishResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformPublish", h.roomserverURL+RoomserverPerformPublishPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom(
 	ctx context.Context,
-	req *api.PerformAdminEvacuateRoomRequest,
-	res *api.PerformAdminEvacuateRoomResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAdminEvacuateRoom")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformAdminEvacuateRoomPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
-	if err != nil {
-		res.Error = &api.PerformError{
-			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
-		}
-	}
+	request *api.PerformAdminEvacuateRoomRequest,
+	response *api.PerformAdminEvacuateRoomResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformAdminEvacuateRoom", h.roomserverURL+RoomserverPerformAdminEvacuateRoomPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) PerformAdminEvacuateUser(
 	ctx context.Context,
-	req *api.PerformAdminEvacuateUserRequest,
-	res *api.PerformAdminEvacuateUserResponse,
-) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAdminEvacuateUser")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformAdminEvacuateUserPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
-	if err != nil {
-		res.Error = &api.PerformError{
-			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
-		}
-	}
+	request *api.PerformAdminEvacuateUserRequest,
+	response *api.PerformAdminEvacuateUserResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformAdminEvacuateUser", h.roomserverURL+RoomserverPerformAdminEvacuateUserPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryLatestEventsAndState implements RoomserverQueryAPI
@@ -329,11 +277,10 @@ func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
 	request *api.QueryLatestEventsAndStateRequest,
 	response *api.QueryLatestEventsAndStateResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryLatestEventsAndState")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryLatestEventsAndStatePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryLatestEventsAndState", h.roomserverURL+RoomserverQueryLatestEventsAndStatePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryStateAfterEvents implements RoomserverQueryAPI
@@ -342,11 +289,10 @@ func (h *httpRoomserverInternalAPI) QueryStateAfterEvents(
 	request *api.QueryStateAfterEventsRequest,
 	response *api.QueryStateAfterEventsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateAfterEvents")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryStateAfterEventsPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryStateAfterEvents", h.roomserverURL+RoomserverQueryStateAfterEventsPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryEventsByID implements RoomserverQueryAPI
@@ -355,11 +301,10 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID(
 	request *api.QueryEventsByIDRequest,
 	response *api.QueryEventsByIDResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryEventsByID")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryEventsByIDPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryEventsByID", h.roomserverURL+RoomserverQueryEventsByIDPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryPublishedRooms(
@@ -367,11 +312,10 @@ func (h *httpRoomserverInternalAPI) QueryPublishedRooms(
 	request *api.QueryPublishedRoomsRequest,
 	response *api.QueryPublishedRoomsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublishedRooms")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryPublishedRoomsPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryPublishedRooms", h.roomserverURL+RoomserverQueryPublishedRoomsPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryMembershipForUser implements RoomserverQueryAPI
@@ -380,11 +324,10 @@ func (h *httpRoomserverInternalAPI) QueryMembershipForUser(
 	request *api.QueryMembershipForUserRequest,
 	response *api.QueryMembershipForUserResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMembershipForUser")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryMembershipForUserPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryMembershipForUser", h.roomserverURL+RoomserverQueryMembershipForUserPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryMembershipsForRoom implements RoomserverQueryAPI
@@ -393,11 +336,10 @@ func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom(
 	request *api.QueryMembershipsForRoomRequest,
 	response *api.QueryMembershipsForRoomResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMembershipsForRoom")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryMembershipsForRoomPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryMembershipsForRoom", h.roomserverURL+RoomserverQueryMembershipsForRoomPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryMembershipsForRoom implements RoomserverQueryAPI
@@ -406,11 +348,10 @@ func (h *httpRoomserverInternalAPI) QueryServerJoinedToRoom(
 	request *api.QueryServerJoinedToRoomRequest,
 	response *api.QueryServerJoinedToRoomResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerJoinedToRoom")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryServerJoinedToRoomPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryServerJoinedToRoom", h.roomserverURL+RoomserverQueryServerJoinedToRoomPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI
@@ -419,11 +360,10 @@ func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent(
 	request *api.QueryServerAllowedToSeeEventRequest,
 	response *api.QueryServerAllowedToSeeEventResponse,
 ) (err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerAllowedToSeeEvent")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryServerAllowedToSeeEventPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryServerAllowedToSeeEvent", h.roomserverURL+RoomserverQueryServerAllowedToSeeEventPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryMissingEvents implements RoomServerQueryAPI
@@ -432,11 +372,10 @@ func (h *httpRoomserverInternalAPI) QueryMissingEvents(
 	request *api.QueryMissingEventsRequest,
 	response *api.QueryMissingEventsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMissingEvents")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryMissingEvents", h.roomserverURL+RoomserverQueryMissingEventsPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryStateAndAuthChain implements RoomserverQueryAPI
@@ -445,11 +384,10 @@ func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain(
 	request *api.QueryStateAndAuthChainRequest,
 	response *api.QueryStateAndAuthChainResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateAndAuthChain")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryStateAndAuthChainPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryStateAndAuthChain", h.roomserverURL+RoomserverQueryStateAndAuthChainPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // PerformBackfill implements RoomServerQueryAPI
@@ -458,11 +396,10 @@ func (h *httpRoomserverInternalAPI) PerformBackfill(
 	request *api.PerformBackfillRequest,
 	response *api.PerformBackfillResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformBackfill")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformBackfillPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformBackfill", h.roomserverURL+RoomserverPerformBackfillPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryRoomVersionCapabilities implements RoomServerQueryAPI
@@ -471,11 +408,10 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities(
 	request *api.QueryRoomVersionCapabilitiesRequest,
 	response *api.QueryRoomVersionCapabilitiesResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionCapabilities")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryRoomVersionCapabilities", h.roomserverURL+RoomserverQueryRoomVersionCapabilitiesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 // QueryRoomVersionForRoom implements RoomServerQueryAPI
@@ -488,16 +424,10 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom(
 		response.RoomVersion = roomVersion
 		return nil
 	}
-
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionForRoom")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err == nil {
-		h.cache.StoreRoomVersion(request.RoomID, response.RoomVersion)
-	}
-	return err
+	return httputil.CallInternalRPCAPI(
+		"QueryRoomVersionForRoom", h.roomserverURL+RoomserverQueryRoomVersionForRoomPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryCurrentState(
@@ -505,11 +435,10 @@ func (h *httpRoomserverInternalAPI) QueryCurrentState(
 	request *api.QueryCurrentStateRequest,
 	response *api.QueryCurrentStateResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryCurrentState")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryCurrentStatePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryCurrentState", h.roomserverURL+RoomserverQueryCurrentStatePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryRoomsForUser(
@@ -517,11 +446,10 @@ func (h *httpRoomserverInternalAPI) QueryRoomsForUser(
 	request *api.QueryRoomsForUserRequest,
 	response *api.QueryRoomsForUserResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomsForUser")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryRoomsForUserPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryRoomsForUser", h.roomserverURL+RoomserverQueryRoomsForUserPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryBulkStateContent(
@@ -529,68 +457,75 @@ func (h *httpRoomserverInternalAPI) QueryBulkStateContent(
 	request *api.QueryBulkStateContentRequest,
 	response *api.QueryBulkStateContentResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBulkStateContent")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryBulkStateContentPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryBulkStateContent", h.roomserverURL+RoomserverQueryBulkStateContentPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QuerySharedUsers(
-	ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse,
+	ctx context.Context,
+	request *api.QuerySharedUsersRequest,
+	response *api.QuerySharedUsersResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QuerySharedUsers")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQuerySharedUsersPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"QuerySharedUsers", h.roomserverURL+RoomserverQuerySharedUsersPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryKnownUsers(
-	ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse,
+	ctx context.Context,
+	request *api.QueryKnownUsersRequest,
+	response *api.QueryKnownUsersResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKnownUsers")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryKnownUsersPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"QueryKnownUsers", h.roomserverURL+RoomserverQueryKnownUsersPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryAuthChain(
-	ctx context.Context, req *api.QueryAuthChainRequest, res *api.QueryAuthChainResponse,
+	ctx context.Context,
+	request *api.QueryAuthChainRequest,
+	response *api.QueryAuthChainResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryAuthChain")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryAuthChainPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"QueryAuthChain", h.roomserverURL+RoomserverQueryAuthChainPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom(
-	ctx context.Context, req *api.QueryServerBannedFromRoomRequest, res *api.QueryServerBannedFromRoomResponse,
+	ctx context.Context,
+	request *api.QueryServerBannedFromRoomRequest,
+	response *api.QueryServerBannedFromRoomResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerBannedFromRoom")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryServerBannedFromRoomPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"QueryServerBannedFromRoom", h.roomserverURL+RoomserverQueryServerBannedFromRoomPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpRoomserverInternalAPI) QueryRestrictedJoinAllowed(
-	ctx context.Context, req *api.QueryRestrictedJoinAllowedRequest, res *api.QueryRestrictedJoinAllowedResponse,
+	ctx context.Context,
+	request *api.QueryRestrictedJoinAllowedRequest,
+	response *api.QueryRestrictedJoinAllowedResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRestrictedJoinAllowed")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverQueryRestrictedJoinAllowed
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"QueryRestrictedJoinAllowed", h.roomserverURL+RoomserverQueryRestrictedJoinAllowed,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpRoomserverInternalAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, res *api.PerformForgetResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformForget")
-	defer span.Finish()
-
-	apiURL := h.roomserverURL + RoomserverPerformForgetPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpRoomserverInternalAPI) PerformForget(
+	ctx context.Context,
+	request *api.PerformForgetRequest,
+	response *api.PerformForgetResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformForget", h.roomserverURL+RoomserverPerformForgetPath,
+		h.httpClient, ctx, request, response,
+	)
 
 }
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index 993381585..e325d76a5 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -1,499 +1,196 @@
 package inthttp
 
 import (
-	"encoding/json"
-	"net/http"
-
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/util"
 )
 
 // AddRoutes adds the RoomserverInternalAPI handlers to the http.ServeMux.
 // nolint: gocyclo
 func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
-	internalAPIMux.Handle(RoomserverInputRoomEventsPath,
-		httputil.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse {
-			var request api.InputRoomEventsRequest
-			var response api.InputRoomEventsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.InputRoomEvents(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		RoomserverInputRoomEventsPath,
+		httputil.MakeInternalRPCAPI("RoomserverInputRoomEvents", r.InputRoomEvents),
 	)
-	internalAPIMux.Handle(RoomserverPerformInvitePath,
-		httputil.MakeInternalAPI("performInvite", func(req *http.Request) util.JSONResponse {
-			var request api.PerformInviteRequest
-			var response api.PerformInviteResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.PerformInvite(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformInvitePath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformInvite", r.PerformInvite),
 	)
-	internalAPIMux.Handle(RoomserverPerformJoinPath,
-		httputil.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse {
-			var request api.PerformJoinRequest
-			var response api.PerformJoinResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.PerformJoin(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformJoinPath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformJoin", r.PerformJoin),
 	)
-	internalAPIMux.Handle(RoomserverPerformLeavePath,
-		httputil.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse {
-			var request api.PerformLeaveRequest
-			var response api.PerformLeaveResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.PerformLeave(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformLeavePath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformLeave", r.PerformLeave),
 	)
-	internalAPIMux.Handle(RoomserverPerformPeekPath,
-		httputil.MakeInternalAPI("performPeek", func(req *http.Request) util.JSONResponse {
-			var request api.PerformPeekRequest
-			var response api.PerformPeekResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.PerformPeek(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformPeekPath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformPeek", r.PerformPeek),
 	)
-	internalAPIMux.Handle(RoomserverPerformInboundPeekPath,
-		httputil.MakeInternalAPI("performInboundPeek", func(req *http.Request) util.JSONResponse {
-			var request api.PerformInboundPeekRequest
-			var response api.PerformInboundPeekResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.PerformInboundPeek(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformInboundPeekPath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformInboundPeek", r.PerformInboundPeek),
 	)
-	internalAPIMux.Handle(RoomserverPerformPeekPath,
-		httputil.MakeInternalAPI("performUnpeek", func(req *http.Request) util.JSONResponse {
-			var request api.PerformUnpeekRequest
-			var response api.PerformUnpeekResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.PerformUnpeek(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformUnpeekPath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformUnpeek", r.PerformUnpeek),
 	)
-	internalAPIMux.Handle(RoomserverPerformRoomUpgradePath,
-		httputil.MakeInternalAPI("performRoomUpgrade", func(req *http.Request) util.JSONResponse {
-			var request api.PerformRoomUpgradeRequest
-			var response api.PerformRoomUpgradeResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.PerformRoomUpgrade(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformRoomUpgradePath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformRoomUpgrade", r.PerformRoomUpgrade),
 	)
-	internalAPIMux.Handle(RoomserverPerformPublishPath,
-		httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse {
-			var request api.PerformPublishRequest
-			var response api.PerformPublishResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.PerformPublish(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformPublishPath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformPublish", r.PerformPublish),
 	)
-	internalAPIMux.Handle(RoomserverPerformAdminEvacuateRoomPath,
-		httputil.MakeInternalAPI("performAdminEvacuateRoom", func(req *http.Request) util.JSONResponse {
-			var request api.PerformAdminEvacuateRoomRequest
-			var response api.PerformAdminEvacuateRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.PerformAdminEvacuateRoom(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformAdminEvacuateRoomPath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateRoom", r.PerformAdminEvacuateRoom),
 	)
-	internalAPIMux.Handle(RoomserverPerformAdminEvacuateUserPath,
-		httputil.MakeInternalAPI("performAdminEvacuateUser", func(req *http.Request) util.JSONResponse {
-			var request api.PerformAdminEvacuateUserRequest
-			var response api.PerformAdminEvacuateUserResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			r.PerformAdminEvacuateUser(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverPerformAdminEvacuateUserPath,
+		httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", r.PerformAdminEvacuateUser),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryPublishedRoomsPath,
-		httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse {
-			var request api.QueryPublishedRoomsRequest
-			var response api.QueryPublishedRoomsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryPublishedRooms(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", r.QueryPublishedRooms),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryLatestEventsAndStatePath,
-		httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {
-			var request api.QueryLatestEventsAndStateRequest
-			var response api.QueryLatestEventsAndStateResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryLatestEventsAndState", r.QueryLatestEventsAndState),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryStateAfterEventsPath,
-		httputil.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse {
-			var request api.QueryStateAfterEventsRequest
-			var response api.QueryStateAfterEventsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryStateAfterEvents", r.QueryStateAfterEvents),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryEventsByIDPath,
-		httputil.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse {
-			var request api.QueryEventsByIDRequest
-			var response api.QueryEventsByIDResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryEventsByID", r.QueryEventsByID),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryMembershipForUserPath,
-		httputil.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse {
-			var request api.QueryMembershipForUserRequest
-			var response api.QueryMembershipForUserResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipForUser", r.QueryMembershipForUser),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryMembershipsForRoomPath,
-		httputil.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryMembershipsForRoomRequest
-			var response api.QueryMembershipsForRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipsForRoom", r.QueryMembershipsForRoom),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryServerJoinedToRoomPath,
-		httputil.MakeInternalAPI("queryServerJoinedToRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryServerJoinedToRoomRequest
-			var response api.QueryServerJoinedToRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryServerJoinedToRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryServerJoinedToRoom", r.QueryServerJoinedToRoom),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryServerAllowedToSeeEventPath,
-		httputil.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse {
-			var request api.QueryServerAllowedToSeeEventRequest
-			var response api.QueryServerAllowedToSeeEventResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryServerAllowedToSeeEvent", r.QueryServerAllowedToSeeEvent),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryMissingEventsPath,
-		httputil.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse {
-			var request api.QueryMissingEventsRequest
-			var response api.QueryMissingEventsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryMissingEvents", r.QueryMissingEvents),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryStateAndAuthChainPath,
-		httputil.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse {
-			var request api.QueryStateAndAuthChainRequest
-			var response api.QueryStateAndAuthChainResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryStateAndAuthChain", r.QueryStateAndAuthChain),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverPerformBackfillPath,
-		httputil.MakeInternalAPI("PerformBackfill", func(req *http.Request) util.JSONResponse {
-			var request api.PerformBackfillRequest
-			var response api.PerformBackfillResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.PerformBackfill(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverPerformBackfill", r.PerformBackfill),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverPerformForgetPath,
-		httputil.MakeInternalAPI("PerformForget", func(req *http.Request) util.JSONResponse {
-			var request api.PerformForgetRequest
-			var response api.PerformForgetResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.PerformForget(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverPerformForget", r.PerformForget),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryRoomVersionCapabilitiesPath,
-		httputil.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse {
-			var request api.QueryRoomVersionCapabilitiesRequest
-			var response api.QueryRoomVersionCapabilitiesResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionCapabilities", r.QueryRoomVersionCapabilities),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverQueryRoomVersionForRoomPath,
-		httputil.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse {
-			var request api.QueryRoomVersionForRoomRequest
-			var response api.QueryRoomVersionForRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionForRoom", r.QueryRoomVersionForRoom),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverSetRoomAliasPath,
-		httputil.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse {
-			var request api.SetRoomAliasRequest
-			var response api.SetRoomAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverSetRoomAlias", r.SetRoomAlias),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverGetRoomIDForAliasPath,
-		httputil.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse {
-			var request api.GetRoomIDForAliasRequest
-			var response api.GetRoomIDForAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverGetRoomIDForAlias", r.GetRoomIDForAlias),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverGetAliasesForRoomIDPath,
-		httputil.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse {
-			var request api.GetAliasesForRoomIDRequest
-			var response api.GetAliasesForRoomIDResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverGetAliasesForRoomID", r.GetAliasesForRoomID),
 	)
+
 	internalAPIMux.Handle(
 		RoomserverRemoveRoomAliasPath,
-		httputil.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse {
-			var request api.RemoveRoomAliasRequest
-			var response api.RemoveRoomAliasResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+		httputil.MakeInternalRPCAPI("RoomserverRemoveRoomAlias", r.RemoveRoomAlias),
 	)
-	internalAPIMux.Handle(RoomserverQueryCurrentStatePath,
-		httputil.MakeInternalAPI("queryCurrentState", func(req *http.Request) util.JSONResponse {
-			request := api.QueryCurrentStateRequest{}
-			response := api.QueryCurrentStateResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QueryCurrentState(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQueryCurrentStatePath,
+		httputil.MakeInternalRPCAPI("RoomserverQueryCurrentState", r.QueryCurrentState),
 	)
-	internalAPIMux.Handle(RoomserverQueryRoomsForUserPath,
-		httputil.MakeInternalAPI("queryRoomsForUser", func(req *http.Request) util.JSONResponse {
-			request := api.QueryRoomsForUserRequest{}
-			response := api.QueryRoomsForUserResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QueryRoomsForUser(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQueryRoomsForUserPath,
+		httputil.MakeInternalRPCAPI("RoomserverQueryRoomsForUser", r.QueryRoomsForUser),
 	)
-	internalAPIMux.Handle(RoomserverQueryBulkStateContentPath,
-		httputil.MakeInternalAPI("queryBulkStateContent", func(req *http.Request) util.JSONResponse {
-			request := api.QueryBulkStateContentRequest{}
-			response := api.QueryBulkStateContentResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QueryBulkStateContent(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQueryBulkStateContentPath,
+		httputil.MakeInternalRPCAPI("RoomserverQueryBulkStateContent", r.QueryBulkStateContent),
 	)
-	internalAPIMux.Handle(RoomserverQuerySharedUsersPath,
-		httputil.MakeInternalAPI("querySharedUsers", func(req *http.Request) util.JSONResponse {
-			request := api.QuerySharedUsersRequest{}
-			response := api.QuerySharedUsersResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QuerySharedUsers(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQuerySharedUsersPath,
+		httputil.MakeInternalRPCAPI("RoomserverQuerySharedUsers", r.QuerySharedUsers),
 	)
-	internalAPIMux.Handle(RoomserverQueryKnownUsersPath,
-		httputil.MakeInternalAPI("queryKnownUsers", func(req *http.Request) util.JSONResponse {
-			request := api.QueryKnownUsersRequest{}
-			response := api.QueryKnownUsersResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QueryKnownUsers(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQueryKnownUsersPath,
+		httputil.MakeInternalRPCAPI("RoomserverQueryKnownUsers", r.QueryKnownUsers),
 	)
-	internalAPIMux.Handle(RoomserverQueryServerBannedFromRoomPath,
-		httputil.MakeInternalAPI("queryServerBannedFromRoom", func(req *http.Request) util.JSONResponse {
-			request := api.QueryServerBannedFromRoomRequest{}
-			response := api.QueryServerBannedFromRoomResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QueryServerBannedFromRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQueryServerBannedFromRoomPath,
+		httputil.MakeInternalRPCAPI("RoomserverQueryServerBannedFromRoom", r.QueryServerBannedFromRoom),
 	)
-	internalAPIMux.Handle(RoomserverQueryAuthChainPath,
-		httputil.MakeInternalAPI("queryAuthChain", func(req *http.Request) util.JSONResponse {
-			request := api.QueryAuthChainRequest{}
-			response := api.QueryAuthChainResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QueryAuthChain(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQueryAuthChainPath,
+		httputil.MakeInternalRPCAPI("RoomserverQueryAuthChain", r.QueryAuthChain),
 	)
-	internalAPIMux.Handle(RoomserverQueryRestrictedJoinAllowed,
-		httputil.MakeInternalAPI("queryRestrictedJoinAllowed", func(req *http.Request) util.JSONResponse {
-			request := api.QueryRestrictedJoinAllowedRequest{}
-			response := api.QueryRestrictedJoinAllowedResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := r.QueryRestrictedJoinAllowed(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		RoomserverQueryRestrictedJoinAllowed,
+		httputil.MakeInternalRPCAPI("RoomserverQueryRestrictedJoinAllowed", r.QueryRestrictedJoinAllowed),
 	)
 }
diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go
index eeded4275..3e9d90a1f 100644
--- a/setup/mscs/msc2836/msc2836_test.go
+++ b/setup/mscs/msc2836/msc2836_test.go
@@ -164,9 +164,9 @@ func TestMSC2836(t *testing.T) {
 	// make everyone joined to each other's rooms
 	nopRsAPI := &testRoomserverAPI{
 		userToJoinedRooms: map[string][]string{
-			alice:   []string{roomID},
-			bob:     []string{roomID},
-			charlie: []string{roomID},
+			alice:   {roomID},
+			bob:     {roomID},
+			charlie: {roomID},
 		},
 		events: map[string]*gomatrixserverlib.HeaderedEvent{
 			eventA.EventID(): eventA,
diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go
index 4bf54cae0..23824e366 100644
--- a/syncapi/internal/keychange.go
+++ b/syncapi/internal/keychange.go
@@ -31,7 +31,7 @@ import (
 // DeviceOTKCounts adds one-time key counts to the /sync response
 func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, deviceID string, res *types.Response) error {
 	var queryRes keyapi.QueryOneTimeKeysResponse
-	keyAPI.QueryOneTimeKeys(ctx, &keyapi.QueryOneTimeKeysRequest{
+	_ = keyAPI.QueryOneTimeKeys(ctx, &keyapi.QueryOneTimeKeysRequest{
 		UserID:   userID,
 		DeviceID: deviceID,
 	}, &queryRes)
@@ -73,7 +73,7 @@ func DeviceListCatchup(
 		offset = int64(from)
 	}
 	var queryRes keyapi.QueryKeyChangesResponse
-	keyAPI.QueryKeyChanges(ctx, &keyapi.QueryKeyChangesRequest{
+	_ = keyAPI.QueryKeyChanges(ctx, &keyapi.QueryKeyChangesRequest{
 		Offset:   offset,
 		ToOffset: toOffset,
 	}, &queryRes)
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index c7d8df740..80d2811be 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -22,31 +22,41 @@ var (
 
 type mockKeyAPI struct{}
 
-func (k *mockKeyAPI) PerformUploadKeys(ctx context.Context, req *keyapi.PerformUploadKeysRequest, res *keyapi.PerformUploadKeysResponse) {
+func (k *mockKeyAPI) PerformUploadKeys(ctx context.Context, req *keyapi.PerformUploadKeysRequest, res *keyapi.PerformUploadKeysResponse) error {
+	return nil
 }
 
 func (k *mockKeyAPI) SetUserAPI(i userapi.UserInternalAPI) {}
 
 // PerformClaimKeys claims one-time keys for use in pre-key messages
-func (k *mockKeyAPI) PerformClaimKeys(ctx context.Context, req *keyapi.PerformClaimKeysRequest, res *keyapi.PerformClaimKeysResponse) {
+func (k *mockKeyAPI) PerformClaimKeys(ctx context.Context, req *keyapi.PerformClaimKeysRequest, res *keyapi.PerformClaimKeysResponse) error {
+	return nil
 }
-func (k *mockKeyAPI) PerformDeleteKeys(ctx context.Context, req *keyapi.PerformDeleteKeysRequest, res *keyapi.PerformDeleteKeysResponse) {
+func (k *mockKeyAPI) PerformDeleteKeys(ctx context.Context, req *keyapi.PerformDeleteKeysRequest, res *keyapi.PerformDeleteKeysResponse) error {
+	return nil
 }
-func (k *mockKeyAPI) PerformUploadDeviceKeys(ctx context.Context, req *keyapi.PerformUploadDeviceKeysRequest, res *keyapi.PerformUploadDeviceKeysResponse) {
+func (k *mockKeyAPI) PerformUploadDeviceKeys(ctx context.Context, req *keyapi.PerformUploadDeviceKeysRequest, res *keyapi.PerformUploadDeviceKeysResponse) error {
+	return nil
 }
-func (k *mockKeyAPI) PerformUploadDeviceSignatures(ctx context.Context, req *keyapi.PerformUploadDeviceSignaturesRequest, res *keyapi.PerformUploadDeviceSignaturesResponse) {
+func (k *mockKeyAPI) PerformUploadDeviceSignatures(ctx context.Context, req *keyapi.PerformUploadDeviceSignaturesRequest, res *keyapi.PerformUploadDeviceSignaturesResponse) error {
+	return nil
 }
-func (k *mockKeyAPI) QueryKeys(ctx context.Context, req *keyapi.QueryKeysRequest, res *keyapi.QueryKeysResponse) {
+func (k *mockKeyAPI) QueryKeys(ctx context.Context, req *keyapi.QueryKeysRequest, res *keyapi.QueryKeysResponse) error {
+	return nil
 }
-func (k *mockKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) {
+func (k *mockKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) error {
+	return nil
 }
-func (k *mockKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) {
+func (k *mockKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) error {
+	return nil
 
 }
-func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *keyapi.QueryDeviceMessagesRequest, res *keyapi.QueryDeviceMessagesResponse) {
+func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *keyapi.QueryDeviceMessagesRequest, res *keyapi.QueryDeviceMessagesResponse) error {
+	return nil
 
 }
-func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *keyapi.QuerySignaturesRequest, res *keyapi.QuerySignaturesResponse) {
+func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *keyapi.QuerySignaturesRequest, res *keyapi.QuerySignaturesResponse) error {
+	return nil
 }
 
 type mockRoomserverAPI struct {
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index b10864ff5..931fef883 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -78,10 +78,11 @@ type syncKeyAPI struct {
 	keyapi.SyncKeyAPI
 }
 
-func (s *syncKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) {
+func (s *syncKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) error {
+	return nil
 }
-func (s *syncKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) {
-
+func (s *syncKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) error {
+	return nil
 }
 
 func TestSyncAPIAccessTokens(t *testing.T) {
diff --git a/userapi/api/api.go b/userapi/api/api.go
index df9408acb..388f97cb4 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -100,7 +100,7 @@ type ClientUserAPI interface {
 	QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error
 	InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error
 	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error
-	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse)
+	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) error
 
 	QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error
 	QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error
diff --git a/userapi/api/api_trace.go b/userapi/api/api_trace.go
index 6d8d28007..7e2f69615 100644
--- a/userapi/api/api_trace.go
+++ b/userapi/api/api_trace.go
@@ -94,9 +94,10 @@ func (t *UserInternalAPITrace) PerformPushRulesPut(ctx context.Context, req *Per
 	util.GetLogger(ctx).Infof("PerformPushRulesPut req=%+v res=%+v", js(req), js(res))
 	return err
 }
-func (t *UserInternalAPITrace) QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) {
-	t.Impl.QueryKeyBackup(ctx, req, res)
+func (t *UserInternalAPITrace) QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) error {
+	err := t.Impl.QueryKeyBackup(ctx, req, res)
 	util.GetLogger(ctx).Infof("QueryKeyBackup req=%+v res=%+v", js(req), js(res))
+	return err
 }
 func (t *UserInternalAPITrace) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error {
 	err := t.Impl.QueryProfile(ctx, req, res)
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 422eb076e..78b226d46 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -192,7 +192,9 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe
 		deleteReq.KeyIDs = append(deleteReq.KeyIDs, gomatrixserverlib.KeyID(keyID))
 	}
 	deleteRes := &keyapi.PerformDeleteKeysResponse{}
-	a.KeyAPI.PerformDeleteKeys(ctx, deleteReq, deleteRes)
+	if err := a.KeyAPI.PerformDeleteKeys(ctx, deleteReq, deleteRes); err != nil {
+		return err
+	}
 	if err := deleteRes.Error; err != nil {
 		return fmt.Errorf("a.KeyAPI.PerformDeleteKeys: %w", err)
 	}
@@ -211,10 +213,12 @@ func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) er
 	}
 
 	var uploadRes keyapi.PerformUploadKeysResponse
-	a.KeyAPI.PerformUploadKeys(context.Background(), &keyapi.PerformUploadKeysRequest{
+	if err := a.KeyAPI.PerformUploadKeys(context.Background(), &keyapi.PerformUploadKeysRequest{
 		UserID:     userID,
 		DeviceKeys: deviceKeys,
-	}, &uploadRes)
+	}, &uploadRes); err != nil {
+		return err
+	}
 	if uploadRes.Error != nil {
 		return fmt.Errorf("failed to delete device keys: %v", uploadRes.Error)
 	}
@@ -268,7 +272,7 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf
 	if req.DisplayName != nil && dev.DisplayName != *req.DisplayName {
 		// display name has changed: update the device key
 		var uploadRes keyapi.PerformUploadKeysResponse
-		a.KeyAPI.PerformUploadKeys(context.Background(), &keyapi.PerformUploadKeysRequest{
+		if err := a.KeyAPI.PerformUploadKeys(context.Background(), &keyapi.PerformUploadKeysRequest{
 			UserID: req.RequestingUserID,
 			DeviceKeys: []keyapi.DeviceKeys{
 				{
@@ -279,7 +283,9 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf
 				},
 			},
 			OnlyDisplayNameUpdates: true,
-		}, &uploadRes)
+		}, &uploadRes); err != nil {
+			return err
+		}
 		if uploadRes.Error != nil {
 			return fmt.Errorf("failed to update device key display name: %v", uploadRes.Error)
 		}
@@ -479,7 +485,9 @@ func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *a
 		UserID: fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName),
 	}
 	evacuateRes := &rsapi.PerformAdminEvacuateUserResponse{}
-	a.RSAPI.PerformAdminEvacuateUser(ctx, evacuateReq, evacuateRes)
+	if err := a.RSAPI.PerformAdminEvacuateUser(ctx, evacuateReq, evacuateRes); err != nil {
+		return err
+	}
 	if err := evacuateRes.Error; err != nil {
 		logrus.WithError(err).Errorf("Failed to evacuate user after account deactivation")
 	}
@@ -538,9 +546,6 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		if req.Version == "" {
 			res.BadInput = true
 			res.Error = "must specify a version to delete"
-			if res.Error != "" {
-				return fmt.Errorf(res.Error)
-			}
 			return nil
 		}
 		exists, err := a.DB.DeleteKeyBackup(ctx, req.UserID, req.Version)
@@ -549,9 +554,6 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = exists
 		res.Version = req.Version
-		if res.Error != "" {
-			return fmt.Errorf(res.Error)
-		}
 		return nil
 	}
 	// Create metadata
@@ -562,9 +564,6 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = err == nil
 		res.Version = version
-		if res.Error != "" {
-			return fmt.Errorf(res.Error)
-		}
 		return nil
 	}
 	// Update metadata
@@ -575,16 +574,10 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = err == nil
 		res.Version = req.Version
-		if res.Error != "" {
-			return fmt.Errorf(res.Error)
-		}
 		return nil
 	}
 	// Upload Keys for a specific version metadata
 	a.uploadBackupKeys(ctx, req, res)
-	if res.Error != "" {
-		return fmt.Errorf(res.Error)
-	}
 	return nil
 }
 
@@ -627,16 +620,16 @@ func (a *UserInternalAPI) uploadBackupKeys(ctx context.Context, req *api.Perform
 	res.KeyETag = etag
 }
 
-func (a *UserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) {
+func (a *UserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) error {
 	version, algorithm, authData, etag, deleted, err := a.DB.GetKeyBackup(ctx, req.UserID, req.Version)
 	res.Version = version
 	if err != nil {
 		if err == sql.ErrNoRows {
 			res.Exists = false
-			return
+			return nil
 		}
 		res.Error = fmt.Sprintf("failed to query key backup: %s", err)
-		return
+		return nil
 	}
 	res.Algorithm = algorithm
 	res.AuthData = authData
@@ -648,15 +641,16 @@ func (a *UserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyB
 		if err != nil {
 			res.Error = fmt.Sprintf("failed to count keys: %s", err)
 		}
-		return
+		return nil
 	}
 
 	result, err := a.DB.GetBackupKeys(ctx, version, req.UserID, req.KeysForRoomID, req.KeysForSessionID)
 	if err != nil {
 		res.Error = fmt.Sprintf("failed to query keys: %s", err)
-		return
+		return nil
 	}
 	res.Keys = result
+	return nil
 }
 
 func (a *UserInternalAPI) QueryNotifications(ctx context.Context, req *api.QueryNotificationsRequest, res *api.QueryNotificationsResponse) error {
diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go
index 23c335cf2..a375d6caa 100644
--- a/userapi/inthttp/client.go
+++ b/userapi/inthttp/client.go
@@ -21,7 +21,6 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/opentracing/opentracing-go"
 )
 
 // HTTP paths for the internal HTTP APIs
@@ -84,11 +83,10 @@ type httpUserInternalAPI struct {
 }
 
 func (h *httpUserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputAccountData")
-	defer span.Finish()
-
-	apiURL := h.apiURL + InputAccountDataPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"InputAccountData", h.apiURL+InputAccountDataPath,
+		h.httpClient, ctx, req, res,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformAccountCreation(
@@ -96,11 +94,10 @@ func (h *httpUserInternalAPI) PerformAccountCreation(
 	request *api.PerformAccountCreationRequest,
 	response *api.PerformAccountCreationResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAccountCreation")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformAccountCreationPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformAccountCreation", h.apiURL+PerformAccountCreationPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformPasswordUpdate(
@@ -108,11 +105,10 @@ func (h *httpUserInternalAPI) PerformPasswordUpdate(
 	request *api.PerformPasswordUpdateRequest,
 	response *api.PerformPasswordUpdateResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPasswordUpdate")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformPasswordUpdatePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformPasswordUpdate", h.apiURL+PerformPasswordUpdatePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformDeviceCreation(
@@ -120,11 +116,10 @@ func (h *httpUserInternalAPI) PerformDeviceCreation(
 	request *api.PerformDeviceCreationRequest,
 	response *api.PerformDeviceCreationResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDeviceCreation")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformDeviceCreationPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformDeviceCreation", h.apiURL+PerformDeviceCreationPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformDeviceDeletion(
@@ -132,47 +127,54 @@ func (h *httpUserInternalAPI) PerformDeviceDeletion(
 	request *api.PerformDeviceDeletionRequest,
 	response *api.PerformDeviceDeletionResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDeviceDeletion")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformDeviceDeletionPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformDeviceDeletion", h.apiURL+PerformDeviceDeletionPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformLastSeenUpdate(
 	ctx context.Context,
-	req *api.PerformLastSeenUpdateRequest,
-	res *api.PerformLastSeenUpdateResponse,
+	request *api.PerformLastSeenUpdateRequest,
+	response *api.PerformLastSeenUpdateResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLastSeen")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformLastSeenUpdatePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	return httputil.CallInternalRPCAPI(
+		"PerformLastSeen", h.apiURL+PerformLastSeenUpdatePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.PerformDeviceUpdateRequest, res *api.PerformDeviceUpdateResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDeviceUpdate")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformDeviceUpdatePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) PerformDeviceUpdate(
+	ctx context.Context,
+	request *api.PerformDeviceUpdateRequest,
+	response *api.PerformDeviceUpdateResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformDeviceUpdate", h.apiURL+PerformDeviceUpdatePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *api.PerformAccountDeactivationRequest, res *api.PerformAccountDeactivationResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAccountDeactivation")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformAccountDeactivationPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) PerformAccountDeactivation(
+	ctx context.Context,
+	request *api.PerformAccountDeactivationRequest,
+	response *api.PerformAccountDeactivationResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformAccountDeactivation", h.apiURL+PerformAccountDeactivationPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) PerformOpenIDTokenCreation(ctx context.Context, request *api.PerformOpenIDTokenCreationRequest, response *api.PerformOpenIDTokenCreationResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformOpenIDTokenCreation")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformOpenIDTokenCreationPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+func (h *httpUserInternalAPI) PerformOpenIDTokenCreation(
+	ctx context.Context,
+	request *api.PerformOpenIDTokenCreationRequest,
+	response *api.PerformOpenIDTokenCreationResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformOpenIDTokenCreation", h.apiURL+PerformOpenIDTokenCreationPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) QueryProfile(
@@ -180,11 +182,10 @@ func (h *httpUserInternalAPI) QueryProfile(
 	request *api.QueryProfileRequest,
 	response *api.QueryProfileResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryProfile")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryProfilePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryProfile", h.apiURL+QueryProfilePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) QueryDeviceInfos(
@@ -192,11 +193,10 @@ func (h *httpUserInternalAPI) QueryDeviceInfos(
 	request *api.QueryDeviceInfosRequest,
 	response *api.QueryDeviceInfosResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryDeviceInfos")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryDeviceInfosPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryDeviceInfos", h.apiURL+QueryDeviceInfosPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) QueryAccessToken(
@@ -204,72 +204,87 @@ func (h *httpUserInternalAPI) QueryAccessToken(
 	request *api.QueryAccessTokenRequest,
 	response *api.QueryAccessTokenResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryAccessToken")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryAccessTokenPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryAccessToken", h.apiURL+QueryAccessTokenPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryDevices(ctx context.Context, req *api.QueryDevicesRequest, res *api.QueryDevicesResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryDevices")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryDevicesPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryDevices(
+	ctx context.Context,
+	request *api.QueryDevicesRequest,
+	response *api.QueryDevicesResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryDevices", h.apiURL+QueryDevicesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAccountDataRequest, res *api.QueryAccountDataResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryAccountData")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryAccountDataPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryAccountData(
+	ctx context.Context,
+	request *api.QueryAccountDataRequest,
+	response *api.QueryAccountDataResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryAccountData", h.apiURL+QueryAccountDataPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QuerySearchProfiles(ctx context.Context, req *api.QuerySearchProfilesRequest, res *api.QuerySearchProfilesResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QuerySearchProfiles")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QuerySearchProfilesPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QuerySearchProfiles(
+	ctx context.Context,
+	request *api.QuerySearchProfilesRequest,
+	response *api.QuerySearchProfilesResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QuerySearchProfiles", h.apiURL+QuerySearchProfilesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryOpenIDToken(ctx context.Context, req *api.QueryOpenIDTokenRequest, res *api.QueryOpenIDTokenResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryOpenIDToken")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryOpenIDTokenPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryOpenIDToken(
+	ctx context.Context,
+	request *api.QueryOpenIDTokenRequest,
+	response *api.QueryOpenIDTokenResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryOpenIDToken", h.apiURL+QueryOpenIDTokenPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformKeyBackup")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformKeyBackupPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
-	if err != nil {
-		res.Error = err.Error()
-	}
-	return nil
-}
-func (h *httpUserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeyBackup")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryKeyBackupPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
-	if err != nil {
-		res.Error = err.Error()
-	}
+func (h *httpUserInternalAPI) PerformKeyBackup(
+	ctx context.Context,
+	request *api.PerformKeyBackupRequest,
+	response *api.PerformKeyBackupResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformKeyBackup", h.apiURL+PerformKeyBackupPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryNotifications(ctx context.Context, req *api.QueryNotificationsRequest, res *api.QueryNotificationsResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryNotifications")
-	defer span.Finish()
+func (h *httpUserInternalAPI) QueryKeyBackup(
+	ctx context.Context,
+	request *api.QueryKeyBackupRequest,
+	response *api.QueryKeyBackupResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryKeyBackup", h.apiURL+QueryKeyBackupPath,
+		h.httpClient, ctx, request, response,
+	)
+}
 
-	return httputil.PostJSON(ctx, span, h.httpClient, h.apiURL+QueryNotificationsPath, req, res)
+func (h *httpUserInternalAPI) QueryNotifications(
+	ctx context.Context,
+	request *api.QueryNotificationsRequest,
+	response *api.QueryNotificationsResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryNotifications", h.apiURL+QueryNotificationsPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformPusherSet(
@@ -277,27 +292,32 @@ func (h *httpUserInternalAPI) PerformPusherSet(
 	request *api.PerformPusherSetRequest,
 	response *struct{},
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPusherSet")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformPusherSetPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformPusherSet", h.apiURL+PerformPusherSetPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) PerformPusherDeletion(ctx context.Context, req *api.PerformPusherDeletionRequest, res *struct{}) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPusherDeletion")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformPusherDeletionPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) PerformPusherDeletion(
+	ctx context.Context,
+	request *api.PerformPusherDeletionRequest,
+	response *struct{},
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformPusherDeletion", h.apiURL+PerformPusherDeletionPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryPushers(ctx context.Context, req *api.QueryPushersRequest, res *api.QueryPushersResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPushers")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryPushersPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryPushers(
+	ctx context.Context,
+	request *api.QueryPushersRequest,
+	response *api.QueryPushersResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryPushers", h.apiURL+QueryPushersPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformPushRulesPut(
@@ -305,89 +325,117 @@ func (h *httpUserInternalAPI) PerformPushRulesPut(
 	request *api.PerformPushRulesPutRequest,
 	response *struct{},
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPushRulesPut")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformPushRulesPutPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformPushRulesPut", h.apiURL+PerformPushRulesPutPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryPushRules(ctx context.Context, req *api.QueryPushRulesRequest, res *api.QueryPushRulesResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPushRules")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryPushRulesPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryPushRules(
+	ctx context.Context,
+	request *api.QueryPushRulesRequest,
+	response *api.QueryPushRulesResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryPushRules", h.apiURL+QueryPushRulesPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) SetAvatarURL(ctx context.Context, req *api.PerformSetAvatarURLRequest, res *api.PerformSetAvatarURLResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, PerformSetAvatarURLPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformSetAvatarURLPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) SetAvatarURL(
+	ctx context.Context,
+	request *api.PerformSetAvatarURLRequest,
+	response *api.PerformSetAvatarURLResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"SetAvatarURL", h.apiURL+PerformSetAvatarURLPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.QueryNumericLocalpartResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, QueryNumericLocalpartPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryNumericLocalpartPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, struct{}{}, res)
+func (h *httpUserInternalAPI) QueryNumericLocalpart(
+	ctx context.Context,
+	response *api.QueryNumericLocalpartResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryNumericLocalpart", h.apiURL+QueryNumericLocalpartPath,
+		h.httpClient, ctx, &struct{}{}, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryAccountAvailability(ctx context.Context, req *api.QueryAccountAvailabilityRequest, res *api.QueryAccountAvailabilityResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, QueryAccountAvailabilityPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryAccountAvailabilityPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryAccountAvailability(
+	ctx context.Context,
+	request *api.QueryAccountAvailabilityRequest,
+	response *api.QueryAccountAvailabilityResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryAccountAvailability", h.apiURL+QueryAccountAvailabilityPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, QueryAccountByPasswordPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryAccountByPasswordPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryAccountByPassword(
+	ctx context.Context,
+	request *api.QueryAccountByPasswordRequest,
+	response *api.QueryAccountByPasswordResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryAccountByPassword", h.apiURL+QueryAccountByPasswordPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, res *struct{}) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, PerformSetDisplayNamePath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformSetDisplayNamePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) SetDisplayName(
+	ctx context.Context,
+	request *api.PerformUpdateDisplayNameRequest,
+	response *struct{},
+) error {
+	return httputil.CallInternalRPCAPI(
+		"SetDisplayName", h.apiURL+PerformSetDisplayNamePath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryLocalpartForThreePID(ctx context.Context, req *api.QueryLocalpartForThreePIDRequest, res *api.QueryLocalpartForThreePIDResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, QueryLocalpartForThreePIDPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryLocalpartForThreePIDPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryLocalpartForThreePID(
+	ctx context.Context,
+	request *api.QueryLocalpartForThreePIDRequest,
+	response *api.QueryLocalpartForThreePIDResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryLocalpartForThreePID", h.apiURL+QueryLocalpartForThreePIDPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) QueryThreePIDsForLocalpart(ctx context.Context, req *api.QueryThreePIDsForLocalpartRequest, res *api.QueryThreePIDsForLocalpartResponse) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, QueryThreePIDsForLocalpartPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryThreePIDsForLocalpartPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) QueryThreePIDsForLocalpart(
+	ctx context.Context,
+	request *api.QueryThreePIDsForLocalpartRequest,
+	response *api.QueryThreePIDsForLocalpartResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryThreePIDsForLocalpart", h.apiURL+QueryThreePIDsForLocalpartPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) PerformForgetThreePID(ctx context.Context, req *api.PerformForgetThreePIDRequest, res *struct{}) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, PerformForgetThreePIDPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformForgetThreePIDPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) PerformForgetThreePID(
+	ctx context.Context,
+	request *api.PerformForgetThreePIDRequest,
+	response *struct{},
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformForgetThreePID", h.apiURL+PerformForgetThreePIDPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
-func (h *httpUserInternalAPI) PerformSaveThreePIDAssociation(ctx context.Context, req *api.PerformSaveThreePIDAssociationRequest, res *struct{}) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, PerformSaveThreePIDAssociationPath)
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformSaveThreePIDAssociationPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+func (h *httpUserInternalAPI) PerformSaveThreePIDAssociation(
+	ctx context.Context,
+	request *api.PerformSaveThreePIDAssociationRequest,
+	response *struct{},
+) error {
+	return httputil.CallInternalRPCAPI(
+		"PerformSaveThreePIDAssociation", h.apiURL+PerformSaveThreePIDAssociationPath,
+		h.httpClient, ctx, request, response,
+	)
 }
diff --git a/userapi/inthttp/client_logintoken.go b/userapi/inthttp/client_logintoken.go
index 366a97099..211b1b7a1 100644
--- a/userapi/inthttp/client_logintoken.go
+++ b/userapi/inthttp/client_logintoken.go
@@ -19,7 +19,6 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/opentracing/opentracing-go"
 )
 
 const (
@@ -33,11 +32,10 @@ func (h *httpUserInternalAPI) PerformLoginTokenCreation(
 	request *api.PerformLoginTokenCreationRequest,
 	response *api.PerformLoginTokenCreationResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLoginTokenCreation")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformLoginTokenCreationPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformLoginTokenCreation", h.apiURL+PerformLoginTokenCreationPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) PerformLoginTokenDeletion(
@@ -45,11 +43,10 @@ func (h *httpUserInternalAPI) PerformLoginTokenDeletion(
 	request *api.PerformLoginTokenDeletionRequest,
 	response *api.PerformLoginTokenDeletionResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLoginTokenDeletion")
-	defer span.Finish()
-
-	apiURL := h.apiURL + PerformLoginTokenDeletionPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"PerformLoginTokenDeletion", h.apiURL+PerformLoginTokenDeletionPath,
+		h.httpClient, ctx, request, response,
+	)
 }
 
 func (h *httpUserInternalAPI) QueryLoginToken(
@@ -57,9 +54,8 @@ func (h *httpUserInternalAPI) QueryLoginToken(
 	request *api.QueryLoginTokenRequest,
 	response *api.QueryLoginTokenResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryLoginToken")
-	defer span.Finish()
-
-	apiURL := h.apiURL + QueryLoginTokenPath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	return httputil.CallInternalRPCAPI(
+		"QueryLoginToken", h.apiURL+QueryLoginTokenPath,
+		h.httpClient, ctx, request, response,
+	)
 }
diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go
index ad532b901..99148b760 100644
--- a/userapi/inthttp/server.go
+++ b/userapi/inthttp/server.go
@@ -15,8 +15,6 @@
 package inthttp
 
 import (
-	"encoding/json"
-	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -29,339 +27,134 @@ import (
 func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 	addRoutesLoginToken(internalAPIMux, s)
 
-	internalAPIMux.Handle(PerformAccountCreationPath,
-		httputil.MakeInternalAPI("performAccountCreation", func(req *http.Request) util.JSONResponse {
-			request := api.PerformAccountCreationRequest{}
-			response := api.PerformAccountCreationResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformAccountCreation(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformPasswordUpdatePath,
-		httputil.MakeInternalAPI("performPasswordUpdate", func(req *http.Request) util.JSONResponse {
-			request := api.PerformPasswordUpdateRequest{}
-			response := api.PerformPasswordUpdateResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformPasswordUpdate(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformDeviceCreationPath,
-		httputil.MakeInternalAPI("performDeviceCreation", func(req *http.Request) util.JSONResponse {
-			request := api.PerformDeviceCreationRequest{}
-			response := api.PerformDeviceCreationResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformDeviceCreation(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformLastSeenUpdatePath,
-		httputil.MakeInternalAPI("performLastSeenUpdate", func(req *http.Request) util.JSONResponse {
-			request := api.PerformLastSeenUpdateRequest{}
-			response := api.PerformLastSeenUpdateResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformLastSeenUpdate(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformDeviceUpdatePath,
-		httputil.MakeInternalAPI("performDeviceUpdate", func(req *http.Request) util.JSONResponse {
-			request := api.PerformDeviceUpdateRequest{}
-			response := api.PerformDeviceUpdateResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformDeviceUpdate(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformDeviceDeletionPath,
-		httputil.MakeInternalAPI("performDeviceDeletion", func(req *http.Request) util.JSONResponse {
-			request := api.PerformDeviceDeletionRequest{}
-			response := api.PerformDeviceDeletionResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformDeviceDeletion(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformAccountDeactivationPath,
-		httputil.MakeInternalAPI("performAccountDeactivation", func(req *http.Request) util.JSONResponse {
-			request := api.PerformAccountDeactivationRequest{}
-			response := api.PerformAccountDeactivationResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformAccountDeactivation(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformOpenIDTokenCreationPath,
-		httputil.MakeInternalAPI("performOpenIDTokenCreation", func(req *http.Request) util.JSONResponse {
-			request := api.PerformOpenIDTokenCreationRequest{}
-			response := api.PerformOpenIDTokenCreationResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformOpenIDTokenCreation(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryProfilePath,
-		httputil.MakeInternalAPI("queryProfile", func(req *http.Request) util.JSONResponse {
-			request := api.QueryProfileRequest{}
-			response := api.QueryProfileResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryProfile(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryAccessTokenPath,
-		httputil.MakeInternalAPI("queryAccessToken", func(req *http.Request) util.JSONResponse {
-			request := api.QueryAccessTokenRequest{}
-			response := api.QueryAccessTokenResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryAccessToken(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryDevicesPath,
-		httputil.MakeInternalAPI("queryDevices", func(req *http.Request) util.JSONResponse {
-			request := api.QueryDevicesRequest{}
-			response := api.QueryDevicesResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryDevices(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryAccountDataPath,
-		httputil.MakeInternalAPI("queryAccountData", func(req *http.Request) util.JSONResponse {
-			request := api.QueryAccountDataRequest{}
-			response := api.QueryAccountDataResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryAccountData(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryDeviceInfosPath,
-		httputil.MakeInternalAPI("queryDeviceInfos", func(req *http.Request) util.JSONResponse {
-			request := api.QueryDeviceInfosRequest{}
-			response := api.QueryDeviceInfosResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryDeviceInfos(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QuerySearchProfilesPath,
-		httputil.MakeInternalAPI("querySearchProfiles", func(req *http.Request) util.JSONResponse {
-			request := api.QuerySearchProfilesRequest{}
-			response := api.QuerySearchProfilesResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QuerySearchProfiles(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryOpenIDTokenPath,
-		httputil.MakeInternalAPI("queryOpenIDToken", func(req *http.Request) util.JSONResponse {
-			request := api.QueryOpenIDTokenRequest{}
-			response := api.QueryOpenIDTokenResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryOpenIDToken(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(InputAccountDataPath,
-		httputil.MakeInternalAPI("inputAccountDataPath", func(req *http.Request) util.JSONResponse {
-			request := api.InputAccountDataRequest{}
-			response := api.InputAccountDataResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.InputAccountData(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryKeyBackupPath,
-		httputil.MakeInternalAPI("queryKeyBackup", func(req *http.Request) util.JSONResponse {
-			request := api.QueryKeyBackupRequest{}
-			response := api.QueryKeyBackupResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			s.QueryKeyBackup(req.Context(), &request, &response)
-			if response.Error != "" {
-				return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", response.Error))
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformKeyBackupPath,
-		httputil.MakeInternalAPI("performKeyBackup", func(req *http.Request) util.JSONResponse {
-			request := api.PerformKeyBackupRequest{}
-			response := api.PerformKeyBackupResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			err := s.PerformKeyBackup(req.Context(), &request, &response)
-			if err != nil {
-				return util.JSONResponse{Code: http.StatusBadRequest, JSON: &response}
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(QueryNotificationsPath,
-		httputil.MakeInternalAPI("queryNotifications", func(req *http.Request) util.JSONResponse {
-			var request api.QueryNotificationsRequest
-			var response api.QueryNotificationsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryNotifications(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		PerformAccountCreationPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformAccountCreation", s.PerformAccountCreation),
 	)
 
-	internalAPIMux.Handle(PerformPusherSetPath,
-		httputil.MakeInternalAPI("performPusherSet", func(req *http.Request) util.JSONResponse {
-			request := api.PerformPusherSetRequest{}
-			response := struct{}{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformPusherSet(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(PerformPusherDeletionPath,
-		httputil.MakeInternalAPI("performPusherDeletion", func(req *http.Request) util.JSONResponse {
-			request := api.PerformPusherDeletionRequest{}
-			response := struct{}{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformPusherDeletion(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		PerformPasswordUpdatePath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformPasswordUpdate", s.PerformPasswordUpdate),
 	)
 
-	internalAPIMux.Handle(QueryPushersPath,
-		httputil.MakeInternalAPI("queryPushers", func(req *http.Request) util.JSONResponse {
-			request := api.QueryPushersRequest{}
-			response := api.QueryPushersResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryPushers(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		PerformDeviceCreationPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceCreation", s.PerformDeviceCreation),
 	)
 
-	internalAPIMux.Handle(PerformPushRulesPutPath,
-		httputil.MakeInternalAPI("performPushRulesPut", func(req *http.Request) util.JSONResponse {
-			request := api.PerformPushRulesPutRequest{}
-			response := struct{}{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformPushRulesPut(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		PerformLastSeenUpdatePath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformLastSeenUpdate", s.PerformLastSeenUpdate),
 	)
 
-	internalAPIMux.Handle(QueryPushRulesPath,
-		httputil.MakeInternalAPI("queryPushRules", func(req *http.Request) util.JSONResponse {
-			request := api.QueryPushRulesRequest{}
-			response := api.QueryPushRulesResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryPushRules(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		PerformDeviceUpdatePath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceUpdate", s.PerformDeviceUpdate),
 	)
-	internalAPIMux.Handle(PerformSetAvatarURLPath,
-		httputil.MakeInternalAPI("performSetAvatarURL", func(req *http.Request) util.JSONResponse {
-			request := api.PerformSetAvatarURLRequest{}
-			response := api.PerformSetAvatarURLResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.SetAvatarURL(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		PerformDeviceDeletionPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformDeviceDeletion", s.PerformDeviceDeletion),
 	)
+
+	internalAPIMux.Handle(
+		PerformAccountDeactivationPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformAccountDeactivation", s.PerformAccountDeactivation),
+	)
+
+	internalAPIMux.Handle(
+		PerformOpenIDTokenCreationPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformOpenIDTokenCreation", s.PerformOpenIDTokenCreation),
+	)
+
+	internalAPIMux.Handle(
+		QueryProfilePath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryProfile", s.QueryProfile),
+	)
+
+	internalAPIMux.Handle(
+		QueryAccessTokenPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccessToken", s.QueryAccessToken),
+	)
+
+	internalAPIMux.Handle(
+		QueryDevicesPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryDevices", s.QueryDevices),
+	)
+
+	internalAPIMux.Handle(
+		QueryAccountDataPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccountData", s.QueryAccountData),
+	)
+
+	internalAPIMux.Handle(
+		QueryDeviceInfosPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryDeviceInfos", s.QueryDeviceInfos),
+	)
+
+	internalAPIMux.Handle(
+		QuerySearchProfilesPath,
+		httputil.MakeInternalRPCAPI("UserAPIQuerySearchProfiles", s.QuerySearchProfiles),
+	)
+
+	internalAPIMux.Handle(
+		QueryOpenIDTokenPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryOpenIDToken", s.QueryOpenIDToken),
+	)
+
+	internalAPIMux.Handle(
+		InputAccountDataPath,
+		httputil.MakeInternalRPCAPI("UserAPIInputAccountData", s.InputAccountData),
+	)
+
+	internalAPIMux.Handle(
+		QueryKeyBackupPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryKeyBackup", s.QueryKeyBackup),
+	)
+
+	internalAPIMux.Handle(
+		PerformKeyBackupPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformKeyBackup", s.PerformKeyBackup),
+	)
+
+	internalAPIMux.Handle(
+		QueryNotificationsPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryNotifications", s.QueryNotifications),
+	)
+
+	internalAPIMux.Handle(
+		PerformPusherSetPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformPusherSet", s.PerformPusherSet),
+	)
+
+	internalAPIMux.Handle(
+		PerformPusherDeletionPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformPusherDeletion", s.PerformPusherDeletion),
+	)
+
+	internalAPIMux.Handle(
+		QueryPushersPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryPushers", s.QueryPushers),
+	)
+
+	internalAPIMux.Handle(
+		PerformPushRulesPutPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformPushRulesPut", s.PerformPushRulesPut),
+	)
+
+	internalAPIMux.Handle(
+		QueryPushRulesPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryPushRules", s.QueryPushRules),
+	)
+
+	internalAPIMux.Handle(
+		PerformSetAvatarURLPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformSetAvatarURL", s.SetAvatarURL),
+	)
+
+	// TODO: Look at the shape of this
 	internalAPIMux.Handle(QueryNumericLocalpartPath,
-		httputil.MakeInternalAPI("queryNumericLocalpart", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("UserAPIQueryNumericLocalpart", func(req *http.Request) util.JSONResponse {
 			response := api.QueryNumericLocalpartResponse{}
 			if err := s.QueryNumericLocalpart(req.Context(), &response); err != nil {
 				return util.ErrorResponse(err)
@@ -369,92 +162,39 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	internalAPIMux.Handle(QueryAccountAvailabilityPath,
-		httputil.MakeInternalAPI("queryAccountAvailability", func(req *http.Request) util.JSONResponse {
-			request := api.QueryAccountAvailabilityRequest{}
-			response := api.QueryAccountAvailabilityResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryAccountAvailability(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryAccountAvailabilityPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccountAvailability", s.QueryAccountAvailability),
 	)
-	internalAPIMux.Handle(QueryAccountByPasswordPath,
-		httputil.MakeInternalAPI("queryAccountByPassword", func(req *http.Request) util.JSONResponse {
-			request := api.QueryAccountByPasswordRequest{}
-			response := api.QueryAccountByPasswordResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryAccountByPassword(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryAccountByPasswordPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryAccountByPassword", s.QueryAccountByPassword),
 	)
-	internalAPIMux.Handle(PerformSetDisplayNamePath,
-		httputil.MakeInternalAPI("performSetDisplayName", func(req *http.Request) util.JSONResponse {
-			request := api.PerformUpdateDisplayNameRequest{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.SetDisplayName(req.Context(), &request, &struct{}{}); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}}
-		}),
+
+	internalAPIMux.Handle(
+		PerformSetDisplayNamePath,
+		httputil.MakeInternalRPCAPI("UserAPISetDisplayName", s.SetDisplayName),
 	)
-	internalAPIMux.Handle(QueryLocalpartForThreePIDPath,
-		httputil.MakeInternalAPI("queryLocalpartForThreePID", func(req *http.Request) util.JSONResponse {
-			request := api.QueryLocalpartForThreePIDRequest{}
-			response := api.QueryLocalpartForThreePIDResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryLocalpartForThreePID(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryLocalpartForThreePIDPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryLocalpartForThreePID", s.QueryLocalpartForThreePID),
 	)
-	internalAPIMux.Handle(QueryThreePIDsForLocalpartPath,
-		httputil.MakeInternalAPI("queryThreePIDsForLocalpart", func(req *http.Request) util.JSONResponse {
-			request := api.QueryThreePIDsForLocalpartRequest{}
-			response := api.QueryThreePIDsForLocalpartResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryThreePIDsForLocalpart(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryThreePIDsForLocalpartPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryThreePIDsForLocalpart", s.QueryThreePIDsForLocalpart),
 	)
-	internalAPIMux.Handle(PerformForgetThreePIDPath,
-		httputil.MakeInternalAPI("performForgetThreePID", func(req *http.Request) util.JSONResponse {
-			request := api.PerformForgetThreePIDRequest{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformForgetThreePID(req.Context(), &request, &struct{}{}); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}}
-		}),
+
+	internalAPIMux.Handle(
+		PerformForgetThreePIDPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformForgetThreePID", s.PerformForgetThreePID),
 	)
-	internalAPIMux.Handle(PerformSaveThreePIDAssociationPath,
-		httputil.MakeInternalAPI("performSaveThreePIDAssociation", func(req *http.Request) util.JSONResponse {
-			request := api.PerformSaveThreePIDAssociationRequest{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformSaveThreePIDAssociation(req.Context(), &request, &struct{}{}); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &struct{}{}}
-		}),
+
+	internalAPIMux.Handle(
+		PerformSaveThreePIDAssociationPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformSaveThreePIDAssociation", s.PerformSaveThreePIDAssociation),
 	)
 }
diff --git a/userapi/inthttp/server_logintoken.go b/userapi/inthttp/server_logintoken.go
index 1f2eb34b9..b57348413 100644
--- a/userapi/inthttp/server_logintoken.go
+++ b/userapi/inthttp/server_logintoken.go
@@ -15,54 +15,25 @@
 package inthttp
 
 import (
-	"encoding/json"
-	"net/http"
-
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/util"
 )
 
 // addRoutesLoginToken adds routes for all login token API calls.
 func addRoutesLoginToken(internalAPIMux *mux.Router, s api.UserInternalAPI) {
-	internalAPIMux.Handle(PerformLoginTokenCreationPath,
-		httputil.MakeInternalAPI("performLoginTokenCreation", func(req *http.Request) util.JSONResponse {
-			request := api.PerformLoginTokenCreationRequest{}
-			response := api.PerformLoginTokenCreationResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformLoginTokenCreation(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+	internalAPIMux.Handle(
+		PerformLoginTokenCreationPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenCreation", s.PerformLoginTokenCreation),
 	)
-	internalAPIMux.Handle(PerformLoginTokenDeletionPath,
-		httputil.MakeInternalAPI("performLoginTokenDeletion", func(req *http.Request) util.JSONResponse {
-			request := api.PerformLoginTokenDeletionRequest{}
-			response := api.PerformLoginTokenDeletionResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.PerformLoginTokenDeletion(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		PerformLoginTokenDeletionPath,
+		httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenDeletion", s.PerformLoginTokenDeletion),
 	)
-	internalAPIMux.Handle(QueryLoginTokenPath,
-		httputil.MakeInternalAPI("queryLoginToken", func(req *http.Request) util.JSONResponse {
-			request := api.QueryLoginTokenRequest{}
-			response := api.QueryLoginTokenResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := s.QueryLoginToken(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
+
+	internalAPIMux.Handle(
+		QueryLoginTokenPath,
+		httputil.MakeInternalRPCAPI("UserAPIQueryLoginToken", s.QueryLoginToken),
 	)
 }
diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index 40e37c5d6..31a69793b 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -117,16 +117,20 @@ func TestQueryProfile(t *testing.T) {
 		},
 	}
 
-	runCases := func(testAPI api.UserInternalAPI) {
+	runCases := func(testAPI api.UserInternalAPI, http bool) {
+		mode := "monolith"
+		if http {
+			mode = "HTTP"
+		}
 		for _, tc := range testCases {
 			var gotRes api.QueryProfileResponse
 			gotErr := testAPI.QueryProfile(context.TODO(), &tc.req, &gotRes)
 			if tc.wantErr == nil && gotErr != nil || tc.wantErr != nil && gotErr == nil {
-				t.Errorf("QueryProfile error, got %s want %s", gotErr, tc.wantErr)
+				t.Errorf("QueryProfile %s error, got %s want %s", mode, gotErr, tc.wantErr)
 				continue
 			}
 			if !reflect.DeepEqual(tc.wantRes, gotRes) {
-				t.Errorf("QueryProfile response got %+v want %+v", gotRes, tc.wantRes)
+				t.Errorf("QueryProfile %s response got %+v want %+v", mode, gotRes, tc.wantRes)
 			}
 		}
 	}
@@ -140,10 +144,10 @@ func TestQueryProfile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("failed to create HTTP client")
 		}
-		runCases(httpAPI)
+		runCases(httpAPI, true)
 	})
 	t.Run("Monolith", func(t *testing.T) {
-		runCases(userAPI)
+		runCases(userAPI, false)
 	})
 }
 

From 25ad40f5e56ea3c84a2bacba3327d6ac65d80e01 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 11 Aug 2022 16:06:13 +0100
Subject: [PATCH 257/391] Remove test from `sytest-blacklist`

---
 sytest-blacklist | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/sytest-blacklist b/sytest-blacklist
index e0b2767b1..bcc345f6e 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -49,7 +49,3 @@ Notifications can be viewed with GET /notifications
 
 If remote user leaves room we no longer receive device updates
 Guest users can join guest_access rooms
-
-# You'll be shocked to discover this is flakey too
-
-Inbound /v1/send_join rejects joins from other servers

From 371336c6b5ffd510802d06b193a48b01a5e78d0c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 11 Aug 2022 16:31:44 +0100
Subject: [PATCH 258/391] Set default room version to 9

---
 roomserver/version/version.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roomserver/version/version.go b/roomserver/version/version.go
index 1f66995d8..729d00a80 100644
--- a/roomserver/version/version.go
+++ b/roomserver/version/version.go
@@ -23,7 +23,7 @@ import (
 // DefaultRoomVersion contains the room version that will, by
 // default, be used to create new rooms on this server.
 func DefaultRoomVersion() gomatrixserverlib.RoomVersion {
-	return gomatrixserverlib.RoomVersionV6
+	return gomatrixserverlib.RoomVersionV9
 }
 
 // RoomVersions returns a map of all known room versions to this

From 05cafbd197c99c0e116c9b61447e70ba5af992a3 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 11 Aug 2022 18:23:35 +0200
Subject: [PATCH 259/391] Implement history visibility on `/messages`,
 `/context`, `/sync` (#2511)

* Add possibility to set history_visibility and user AccountType

* Add new DB queries

* Add actual history_visibility changes for /messages

* Add passing tests

* Extract check function

* Cleanup

* Cleanup

* Fix build on 386

* Move ApplyHistoryVisibilityFilter to internal

* Move queries to topology table

* Add filtering to /sync and /context
Some cleanup

* Add passing tests; Remove failing tests :(

* Re-add passing tests

* Move filtering to own function to avoid duplication

* Re-add passing test

* Use newly added GMSL HistoryVisibility

* Update gomatrixserverlib

* Set the visibility when creating events

* Default to shared history visibility

* Remove unused query

* Update history visibility checks to use gmsl
Update tests

* Remove unused statement

* Update migrations to set "correct" history visibility

* Add method to fetch the membership at a given event

* Tweaks and logging

* Use actual internal rsAPI, default to shared visibility in tests

* Revert "Move queries to topology table"

This reverts commit 4f0d41be9c194a46379796435ce73e79203edbd6.

* Remove noise/unneeded code

* More cleanup

* Try to optimize database requests

* Fix imports

* PR peview fixes/changes

* Move setting history visibility to own migration, be more restrictive

* Fix unit tests

* Lint

* Fix missing entries

* Tweaks for incremental syncs

* Adapt generic changes

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
Co-authored-by: kegsay <kegan@matrix.org>
---
 go.mod                                        |   8 +-
 go.sum                                        |  16 +-
 roomserver/api/api.go                         |   8 +
 roomserver/api/api_trace.go                   |  10 +
 roomserver/api/query.go                       |  14 ++
 roomserver/internal/helpers/helpers.go        |   6 +
 roomserver/internal/input/input_events.go     |   4 +-
 roomserver/internal/query/query.go            |  48 ++++
 roomserver/inthttp/client.go                  |  12 +-
 roomserver/inthttp/server.go                  |   5 +
 roomserver/state/state.go                     |  60 ++++-
 syncapi/internal/history_visibility.go        | 217 ++++++++++++++++++
 syncapi/routing/context.go                    |  99 ++++++--
 syncapi/routing/messages.go                   | 100 ++------
 syncapi/storage/interface.go                  |   4 +
 ...2061412000000_history_visibility_column.go |  55 +++++
 syncapi/storage/postgres/memberships_table.go |  28 ++-
 .../postgres/output_room_events_table.go      |  10 +-
 syncapi/storage/postgres/syncserver.go        |  15 ++
 syncapi/storage/shared/syncserver.go          |  49 ++--
 ...2061412000000_history_visibility_column.go |  55 +++++
 syncapi/storage/sqlite3/memberships_table.go  |  22 ++
 .../sqlite3/output_room_events_table.go       |  10 +-
 syncapi/storage/sqlite3/syncserver.go         |  19 +-
 syncapi/storage/storage_test.go               |  20 +-
 syncapi/storage/tables/interface.go           |   1 +
 syncapi/streams/stream_pdu.go                 | 123 ++++++----
 syncapi/syncapi_test.go                       | 186 ++++++++++++++-
 sytest-whitelist                              |  25 +-
 test/room.go                                  |  28 ++-
 test/user.go                                  |  10 +-
 31 files changed, 1043 insertions(+), 224 deletions(-)
 create mode 100644 syncapi/internal/history_visibility.go

diff --git a/go.mod b/go.mod
index 3559c5bb1..79ef5c3e8 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
@@ -34,7 +34,7 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.2
-	github.com/sirupsen/logrus v1.8.1
+	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tidwall/gjson v1.14.1
 	github.com/tidwall/sjson v1.2.4
@@ -42,7 +42,7 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.3
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
 	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
 	golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
 	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
@@ -99,7 +99,7 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
 	golang.org/x/tools v0.1.10 // indirect
diff --git a/go.sum b/go.sum
index 2c8bb4f18..9d5c50d2c 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771 h1:ZIPHFIPNDS9dmEbPEiJbNmyCGJtn9exfpLC7JOcn/bE=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220725104114-b6003e522771/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839 h1:QEFxKWH8PlEt3ZQKl31yJNAm8lvpNUwT51IMNTl9v1k=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -493,8 +493,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
@@ -569,8 +569,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
-golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -748,8 +748,10 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80=
+golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index ee0212ecf..baf63aa31 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -97,6 +97,14 @@ type SyncRoomserverAPI interface {
 		req *PerformBackfillRequest,
 		res *PerformBackfillResponse,
 	) error
+
+	// QueryMembershipAtEvent queries the memberships at the given events.
+	// Returns a map from eventID to a slice of gomatrixserverlib.HeaderedEvent.
+	QueryMembershipAtEvent(
+		ctx context.Context,
+		request *QueryMembershipAtEventRequest,
+		response *QueryMembershipAtEventResponse,
+	) error
 }
 
 type AppserviceRoomserverAPI interface {
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index 18a617331..8bef35379 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -382,6 +382,16 @@ func (t *RoomserverInternalAPITrace) QueryRestrictedJoinAllowed(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) QueryMembershipAtEvent(
+	ctx context.Context,
+	request *QueryMembershipAtEventRequest,
+	response *QueryMembershipAtEventResponse,
+) error {
+	err := t.Impl.QueryMembershipAtEvent(ctx, request, response)
+	util.GetLogger(ctx).WithError(err).Infof("QueryMembershipAtEvent req=%+v res=%+v", js(request), js(response))
+	return err
+}
+
 func js(thing interface{}) string {
 	b, err := json.Marshal(thing)
 	if err != nil {
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index f157a9025..c8e6f9dc6 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -427,3 +427,17 @@ func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error {
 	}
 	return nil
 }
+
+// QueryMembershipAtEventRequest requests the membership events for a user
+// for a list of eventIDs.
+type QueryMembershipAtEventRequest struct {
+	RoomID   string
+	EventIDs []string
+	UserID   string
+}
+
+// QueryMembershipAtEventResponse is the response to QueryMembershipAtEventRequest.
+type QueryMembershipAtEventResponse struct {
+	// Memberships is a map from eventID to a list of events (if any).
+	Memberships map[string][]*gomatrixserverlib.HeaderedEvent `json:"memberships"`
+}
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index d61aa08cb..6091f8ec2 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -208,6 +208,12 @@ func StateBeforeEvent(ctx context.Context, db storage.Database, info *types.Room
 	return roomState.LoadCombinedStateAfterEvents(ctx, prevState)
 }
 
+func MembershipAtEvent(ctx context.Context, db storage.Database, info *types.RoomInfo, eventIDs []string, stateKeyNID types.EventStateKeyNID) (map[string][]types.StateEntry, error) {
+	roomState := state.NewStateResolution(db, info)
+	// Fetch the state as it was when this event was fired
+	return roomState.LoadMembershipAtEvent(ctx, eventIDs, stateKeyNID)
+}
+
 func LoadEvents(
 	ctx context.Context, db storage.Database, eventNIDs []types.EventNID,
 ) ([]*gomatrixserverlib.Event, error) {
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 866670d7a..81541260c 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -299,7 +299,7 @@ func (r *Inputer) processRoomEvent(
 	// allowed at the time, and also to get the history visibility. We won't
 	// bother doing this if the event was already rejected as it just ends up
 	// burning CPU time.
-	historyVisibility := gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive.
+	historyVisibility := gomatrixserverlib.HistoryVisibilityShared // Default to shared.
 	if rejectionErr == nil && !isRejected && !softfail {
 		var err error
 		historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev)
@@ -429,7 +429,7 @@ func (r *Inputer) processStateBefore(
 	input *api.InputRoomEvent,
 	missingPrev bool,
 ) (historyVisibility gomatrixserverlib.HistoryVisibility, rejectionErr error, err error) {
-	historyVisibility = gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive.
+	historyVisibility = gomatrixserverlib.HistoryVisibilityShared // Default to shared.
 	event := input.Event.Unwrap()
 	isCreateEvent := event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("")
 	var stateBeforeEvent []*gomatrixserverlib.Event
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 5ba59b8f5..c41e1ea67 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -204,6 +204,54 @@ func (r *Queryer) QueryMembershipForUser(
 	return err
 }
 
+func (r *Queryer) QueryMembershipAtEvent(
+	ctx context.Context,
+	request *api.QueryMembershipAtEventRequest,
+	response *api.QueryMembershipAtEventResponse,
+) error {
+	response.Memberships = make(map[string][]*gomatrixserverlib.HeaderedEvent)
+	info, err := r.DB.RoomInfo(ctx, request.RoomID)
+	if err != nil {
+		return fmt.Errorf("unable to get roomInfo: %w", err)
+	}
+	if info == nil {
+		return fmt.Errorf("no roomInfo found")
+	}
+
+	// get the users stateKeyNID
+	stateKeyNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{request.UserID})
+	if err != nil {
+		return fmt.Errorf("unable to get stateKeyNIDs for %s: %w", request.UserID, err)
+	}
+	if _, ok := stateKeyNIDs[request.UserID]; !ok {
+		return fmt.Errorf("requested stateKeyNID for %s was not found", request.UserID)
+	}
+
+	stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, info, request.EventIDs, stateKeyNIDs[request.UserID])
+	if err != nil {
+		return fmt.Errorf("unable to get state before event: %w", err)
+	}
+
+	for _, eventID := range request.EventIDs {
+		stateEntry := stateEntries[eventID]
+		memberships, err := helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false)
+		if err != nil {
+			return fmt.Errorf("unable to get memberships at state: %w", err)
+		}
+		res := make([]*gomatrixserverlib.HeaderedEvent, 0, len(memberships))
+
+		for i := range memberships {
+			ev := memberships[i]
+			if ev.Type() == gomatrixserverlib.MRoomMember && ev.StateKeyEquals(request.UserID) {
+				res = append(res, ev.Headered(info.RoomVersion))
+			}
+		}
+		response.Memberships[eventID] = res
+	}
+
+	return nil
+}
+
 // QueryMembershipsForRoom implements api.RoomserverInternalAPI
 func (r *Queryer) QueryMembershipsForRoom(
 	ctx context.Context,
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index d16f67c69..e9387bd97 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -5,14 +5,14 @@ import (
 	"errors"
 	"net/http"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	asAPI "github.com/matrix-org/dendrite/appservice/api"
 	fsInputAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const (
@@ -61,6 +61,7 @@ const (
 	RoomserverQueryServerBannedFromRoomPath    = "/roomserver/queryServerBannedFromRoom"
 	RoomserverQueryAuthChainPath               = "/roomserver/queryAuthChain"
 	RoomserverQueryRestrictedJoinAllowed       = "/roomserver/queryRestrictedJoinAllowed"
+	RoomserverQueryMembershipAtEventPath       = "/roomserver/queryMembershipAtEvent"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -529,3 +530,10 @@ func (h *httpRoomserverInternalAPI) PerformForget(
 	)
 
 }
+
+func (h *httpRoomserverInternalAPI) QueryMembershipAtEvent(ctx context.Context, request *api.QueryMembershipAtEventRequest, response *api.QueryMembershipAtEventResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryMembershiptAtEvent", h.roomserverURL+RoomserverQueryMembershipAtEventPath,
+		h.httpClient, ctx, request, response,
+	)
+}
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index e325d76a5..3b688174a 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -2,6 +2,7 @@ package inthttp
 
 import (
 	"github.com/gorilla/mux"
+
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 )
@@ -193,4 +194,8 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 		RoomserverQueryRestrictedJoinAllowed,
 		httputil.MakeInternalRPCAPI("RoomserverQueryRestrictedJoinAllowed", r.QueryRestrictedJoinAllowed),
 	)
+	internalAPIMux.Handle(
+		RoomserverQueryMembershipAtEventPath,
+		httputil.MakeInternalRPCAPI("RoomserverQueryMembershipAtEventPath", r.QueryMembershipAtEvent),
+	)
 }
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index ca0c69f27..a40a2e9ba 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -23,12 +23,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type StateResolutionStorage interface {
@@ -124,6 +123,61 @@ func (v *StateResolution) LoadStateAtEvent(
 	return stateEntries, nil
 }
 
+func (v *StateResolution) LoadMembershipAtEvent(
+	ctx context.Context, eventIDs []string, stateKeyNID types.EventStateKeyNID,
+) (map[string][]types.StateEntry, error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadMembershipAtEvent")
+	defer span.Finish()
+
+	// De-dupe snapshotNIDs
+	snapshotNIDMap := make(map[types.StateSnapshotNID][]string) // map from snapshot NID to eventIDs
+	for i := range eventIDs {
+		eventID := eventIDs[i]
+		snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID)
+		if err != nil {
+			return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %w", eventID, err)
+		}
+		if snapshotNID == 0 {
+			return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID)
+		}
+		snapshotNIDMap[snapshotNID] = append(snapshotNIDMap[snapshotNID], eventID)
+	}
+
+	snapshotNIDs := make([]types.StateSnapshotNID, 0, len(snapshotNIDMap))
+	for nid := range snapshotNIDMap {
+		snapshotNIDs = append(snapshotNIDs, nid)
+	}
+
+	stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, snapshotNIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string][]types.StateEntry)
+	for _, stateBlockNIDList := range stateBlockNIDLists {
+		// Query the membership event for the user at the given stateblocks
+		stateEntryLists, err := v.db.StateEntriesForTuples(ctx, stateBlockNIDList.StateBlockNIDs, []types.StateKeyTuple{
+			{
+				EventTypeNID:     types.MRoomMemberNID,
+				EventStateKeyNID: stateKeyNID,
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		evIDs := snapshotNIDMap[stateBlockNIDList.StateSnapshotNID]
+
+		for _, evID := range evIDs {
+			for _, x := range stateEntryLists {
+				result[evID] = append(result[evID], x.StateEntries...)
+			}
+		}
+	}
+
+	return result, nil
+}
+
 // LoadStateAtEvent loads the full state of a room before a particular event.
 func (v *StateResolution) LoadStateAtEventForHistoryVisibility(
 	ctx context.Context, eventID string,
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
new file mode 100644
index 000000000..e73c004e5
--- /dev/null
+++ b/syncapi/internal/history_visibility.go
@@ -0,0 +1,217 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"math"
+	"time"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/tidwall/gjson"
+)
+
+func init() {
+	prometheus.MustRegister(calculateHistoryVisibilityDuration)
+}
+
+// calculateHistoryVisibilityDuration stores the time it takes to
+// calculate the history visibility. In polylith mode the roundtrip
+// to the roomserver is included in this time.
+var calculateHistoryVisibilityDuration = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Namespace: "dendrite",
+		Subsystem: "syncapi",
+		Name:      "calculateHistoryVisibility_duration_millis",
+		Help:      "How long it takes to calculate the history visibility",
+		Buckets: []float64{ // milliseconds
+			5, 10, 25, 50, 75, 100, 250, 500,
+			1000, 2000, 3000, 4000, 5000, 6000,
+			7000, 8000, 9000, 10000, 15000, 20000,
+		},
+	},
+	[]string{"api"},
+)
+
+var historyVisibilityPriority = map[gomatrixserverlib.HistoryVisibility]uint8{
+	gomatrixserverlib.WorldReadable:            0,
+	gomatrixserverlib.HistoryVisibilityShared:  1,
+	gomatrixserverlib.HistoryVisibilityInvited: 2,
+	gomatrixserverlib.HistoryVisibilityJoined:  3,
+}
+
+// eventVisibility contains the history visibility and membership state at a given event
+type eventVisibility struct {
+	visibility        gomatrixserverlib.HistoryVisibility
+	membershipAtEvent string
+	membershipCurrent string
+}
+
+// allowed checks the eventVisibility if the user is allowed to see the event.
+// Rules as defined by https://spec.matrix.org/v1.3/client-server-api/#server-behaviour-5
+func (ev eventVisibility) allowed() (allowed bool) {
+	switch ev.visibility {
+	case gomatrixserverlib.HistoryVisibilityWorldReadable:
+		// If the history_visibility was set to world_readable, allow.
+		return true
+	case gomatrixserverlib.HistoryVisibilityJoined:
+		// If the user’s membership was join, allow.
+		if ev.membershipAtEvent == gomatrixserverlib.Join {
+			return true
+		}
+		return false
+	case gomatrixserverlib.HistoryVisibilityShared:
+		// If the user’s membership was join, allow.
+		// If history_visibility was set to shared, and the user joined the room at any point after the event was sent, allow.
+		if ev.membershipAtEvent == gomatrixserverlib.Join || ev.membershipCurrent == gomatrixserverlib.Join {
+			return true
+		}
+		return false
+	case gomatrixserverlib.HistoryVisibilityInvited:
+		// If the user’s membership was join, allow.
+		if ev.membershipAtEvent == gomatrixserverlib.Join {
+			return true
+		}
+		if ev.membershipAtEvent == gomatrixserverlib.Invite {
+			return true
+		}
+		return false
+	default:
+		return false
+	}
+}
+
+// ApplyHistoryVisibilityFilter applies the room history visibility filter on gomatrixserverlib.HeaderedEvents.
+// Returns the filtered events and an error, if any.
+func ApplyHistoryVisibilityFilter(
+	ctx context.Context,
+	syncDB storage.Database,
+	rsAPI api.SyncRoomserverAPI,
+	events []*gomatrixserverlib.HeaderedEvent,
+	alwaysIncludeEventIDs map[string]struct{},
+	userID, endpoint string,
+) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	if len(events) == 0 {
+		return events, nil
+	}
+	start := time.Now()
+
+	// try to get the current membership of the user
+	membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID(), userID, math.MaxInt64)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the mapping from eventID -> eventVisibility
+	eventsFiltered := make([]*gomatrixserverlib.HeaderedEvent, 0, len(events))
+	visibilities, err := visibilityForEvents(ctx, rsAPI, events, userID, events[0].RoomID())
+	if err != nil {
+		return eventsFiltered, err
+	}
+	for _, ev := range events {
+		evVis := visibilities[ev.EventID()]
+		evVis.membershipCurrent = membershipCurrent
+		// Always include specific state events for /sync responses
+		if alwaysIncludeEventIDs != nil {
+			if _, ok := alwaysIncludeEventIDs[ev.EventID()]; ok {
+				eventsFiltered = append(eventsFiltered, ev)
+				continue
+			}
+		}
+		// NOTSPEC: Always allow user to see their own membership events (spec contains more "rules")
+		if ev.Type() == gomatrixserverlib.MRoomMember && ev.StateKeyEquals(userID) {
+			eventsFiltered = append(eventsFiltered, ev)
+			continue
+		}
+		// Always allow history evVis events on boundaries. This is done
+		// by setting the effective evVis to the least restrictive
+		// of the old vs new.
+		// https://spec.matrix.org/v1.3/client-server-api/#server-behaviour-5
+		if hisVis, err := ev.HistoryVisibility(); err == nil {
+			prevHisVis := gjson.GetBytes(ev.Unsigned(), "prev_content.history_visibility").String()
+			oldPrio, ok := historyVisibilityPriority[gomatrixserverlib.HistoryVisibility(prevHisVis)]
+			// if we can't get the previous history visibility, default to shared.
+			if !ok {
+				oldPrio = historyVisibilityPriority[gomatrixserverlib.HistoryVisibilityShared]
+			}
+			// no OK check, since this should have been validated when setting the value
+			newPrio := historyVisibilityPriority[hisVis]
+			if oldPrio < newPrio {
+				evVis.visibility = gomatrixserverlib.HistoryVisibility(prevHisVis)
+			}
+		}
+		// do the actual check
+		allowed := evVis.allowed()
+		if allowed {
+			eventsFiltered = append(eventsFiltered, ev)
+		}
+	}
+	calculateHistoryVisibilityDuration.With(prometheus.Labels{"api": endpoint}).Observe(float64(time.Since(start).Milliseconds()))
+	return eventsFiltered, nil
+}
+
+// visibilityForEvents returns a map from eventID to eventVisibility containing the visibility and the membership
+// of `userID` at the given event.
+// Returns an error if the roomserver can't calculate the memberships.
+func visibilityForEvents(
+	ctx context.Context,
+	rsAPI api.SyncRoomserverAPI,
+	events []*gomatrixserverlib.HeaderedEvent,
+	userID, roomID string,
+) (map[string]eventVisibility, error) {
+	eventIDs := make([]string, len(events))
+	for i := range events {
+		eventIDs[i] = events[i].EventID()
+	}
+
+	result := make(map[string]eventVisibility, len(eventIDs))
+
+	// get the membership events for all eventIDs
+	membershipResp := &api.QueryMembershipAtEventResponse{}
+	err := rsAPI.QueryMembershipAtEvent(ctx, &api.QueryMembershipAtEventRequest{
+		RoomID:   roomID,
+		EventIDs: eventIDs,
+		UserID:   userID,
+	}, membershipResp)
+	if err != nil {
+		return result, err
+	}
+
+	// Create a map from eventID -> eventVisibility
+	for _, event := range events {
+		eventID := event.EventID()
+		vis := eventVisibility{
+			membershipAtEvent: gomatrixserverlib.Leave, // default to leave, to not expose events by accident
+			visibility:        event.Visibility,
+		}
+		membershipEvs, ok := membershipResp.Memberships[eventID]
+		if !ok {
+			result[eventID] = vis
+			continue
+		}
+		for _, ev := range membershipEvs {
+			membership, err := ev.Membership()
+			if err != nil {
+				return result, err
+			}
+			vis.membershipAtEvent = membership
+		}
+		result[eventID] = vis
+	}
+	return result, nil
+}
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index f6b4d15e0..13c4e9d89 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -21,10 +21,12 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/caching"
 	roomserver "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/internal"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -95,24 +97,6 @@ func Context(
 		ContainsURL:             filter.ContainsURL,
 	}
 
-	// TODO: Get the actual state at the last event returned by SelectContextAfterEvent
-	state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
-	// verify the user is allowed to see the context for this room/event
-	for _, x := range state {
-		var hisVis gomatrixserverlib.HistoryVisibility
-		hisVis, err = x.HistoryVisibility()
-		if err != nil {
-			continue
-		}
-		allowed := hisVis == gomatrixserverlib.WorldReadable || membershipRes.Membership == gomatrixserverlib.Join
-		if !allowed {
-			return util.JSONResponse{
-				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden("User is not allowed to query context"),
-			}
-		}
-	}
-
 	id, requestedEvent, err := syncDB.SelectContextEvent(ctx, roomID, eventID)
 	if err != nil {
 		if err == sql.ErrNoRows {
@@ -125,6 +109,24 @@ func Context(
 		return jsonerror.InternalServerError()
 	}
 
+	// verify the user is allowed to see the context for this room/event
+	startTime := time.Now()
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, []*gomatrixserverlib.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context")
+	if err != nil {
+		logrus.WithError(err).Error("unable to apply history visibility filter")
+		return jsonerror.InternalServerError()
+	}
+	logrus.WithFields(logrus.Fields{
+		"duration": time.Since(startTime),
+		"room_id":  roomID,
+	}).Debug("applied history visibility (context)")
+	if len(filteredEvents) == 0 {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("User is not allowed to query context"),
+		}
+	}
+
 	eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, roomID, filter)
 	if err != nil && err != sql.ErrNoRows {
 		logrus.WithError(err).Error("unable to fetch before events")
@@ -137,8 +139,27 @@ func Context(
 		return jsonerror.InternalServerError()
 	}
 
-	eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatAll)
-	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatAll)
+	startTime = time.Now()
+	eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, syncDB, rsAPI, eventsBefore, eventsAfter, device.UserID)
+	if err != nil {
+		logrus.WithError(err).Error("unable to apply history visibility filter")
+		return jsonerror.InternalServerError()
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"duration": time.Since(startTime),
+		"room_id":  roomID,
+	}).Debug("applied history visibility (context eventsBefore/eventsAfter)")
+
+	// TODO: Get the actual state at the last event returned by SelectContextAfterEvent
+	state, err := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
+	if err != nil {
+		logrus.WithError(err).Error("unable to fetch current room state")
+		return jsonerror.InternalServerError()
+	}
+
+	eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBeforeFiltered, gomatrixserverlib.FormatAll)
+	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfterFiltered, gomatrixserverlib.FormatAll)
 	newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache)
 
 	response := ContextRespsonse{
@@ -162,6 +183,44 @@ func Context(
 	}
 }
 
+// applyHistoryVisibilityOnContextEvents is a helper function to avoid roundtrips to the roomserver
+// by combining the events before and after the context event. Returns the filtered events,
+// and an error, if any.
+func applyHistoryVisibilityOnContextEvents(
+	ctx context.Context, syncDB storage.Database, rsAPI roomserver.SyncRoomserverAPI,
+	eventsBefore, eventsAfter []*gomatrixserverlib.HeaderedEvent,
+	userID string,
+) (filteredBefore, filteredAfter []*gomatrixserverlib.HeaderedEvent, err error) {
+	eventIDsBefore := make(map[string]struct{}, len(eventsBefore))
+	eventIDsAfter := make(map[string]struct{}, len(eventsAfter))
+
+	// Remember before/after eventIDs, so we can restore them
+	// after applying history visibility checks
+	for _, ev := range eventsBefore {
+		eventIDsBefore[ev.EventID()] = struct{}{}
+	}
+	for _, ev := range eventsAfter {
+		eventIDsAfter[ev.EventID()] = struct{}{}
+	}
+
+	allEvents := append(eventsBefore, eventsAfter...)
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, allEvents, nil, userID, "context")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// "Restore" events in the correct context
+	for _, ev := range filteredEvents {
+		if _, ok := eventIDsBefore[ev.EventID()]; ok {
+			filteredBefore = append(filteredBefore, ev)
+		}
+		if _, ok := eventIDsAfter[ev.EventID()]; ok {
+			filteredAfter = append(filteredAfter, ev)
+		}
+	}
+	return filteredBefore, filteredAfter, nil
+}
+
 func getStartEnd(ctx context.Context, syncDB storage.Database, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) {
 	if len(startEvents) > 0 {
 		start, err = syncDB.EventPositionInTopology(ctx, startEvents[0].EventID())
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index b4c9a5423..9db3d8e17 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -28,6 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/internal"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/sync"
 	"github.com/matrix-org/dendrite/syncapi/types"
@@ -324,6 +326,9 @@ func (r *messagesReq) retrieveEvents() (
 	// reliable way to define it), it would be easier and less troublesome to
 	// only have to change it in one place, i.e. the database.
 	start, end, err = r.getStartEnd(events)
+	if err != nil {
+		return []gomatrixserverlib.ClientEvent{}, *r.from, *r.to, err
+	}
 
 	// Sort the events to ensure we send them in the right order.
 	if r.backwardOrdering {
@@ -337,97 +342,18 @@ func (r *messagesReq) retrieveEvents() (
 		}
 		events = reversed(events)
 	}
-	events = r.filterHistoryVisible(events)
 	if len(events) == 0 {
 		return []gomatrixserverlib.ClientEvent{}, *r.from, *r.to, nil
 	}
 
-	// Convert all of the events into client events.
-	clientEvents = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll)
-	return clientEvents, start, end, err
-}
-
-func (r *messagesReq) filterHistoryVisible(events []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
-	// TODO FIXME: We don't fully implement history visibility yet. To avoid leaking events which the
-	// user shouldn't see, we check the recent events and remove any prior to the join event of the user
-	// which is equiv to history_visibility: joined
-	joinEventIndex := -1
-	for i, ev := range events {
-		if ev.Type() == gomatrixserverlib.MRoomMember && ev.StateKeyEquals(r.device.UserID) {
-			membership, _ := ev.Membership()
-			if membership == "join" {
-				joinEventIndex = i
-				break
-			}
-		}
-	}
-
-	var result []*gomatrixserverlib.HeaderedEvent
-	var eventsToCheck []*gomatrixserverlib.HeaderedEvent
-	if joinEventIndex != -1 {
-		if r.backwardOrdering {
-			result = events[:joinEventIndex+1]
-			eventsToCheck = append(eventsToCheck, result[0])
-		} else {
-			result = events[joinEventIndex:]
-			eventsToCheck = append(eventsToCheck, result[len(result)-1])
-		}
-	} else {
-		eventsToCheck = []*gomatrixserverlib.HeaderedEvent{events[0], events[len(events)-1]}
-		result = events
-	}
-	// make sure the user was in the room for both the earliest and latest events, we need this because
-	// some backpagination results will not have the join event (e.g if they hit /messages at the join event itself)
-	wasJoined := true
-	for _, ev := range eventsToCheck {
-		var queryRes api.QueryStateAfterEventsResponse
-		err := r.rsAPI.QueryStateAfterEvents(r.ctx, &api.QueryStateAfterEventsRequest{
-			RoomID:       ev.RoomID(),
-			PrevEventIDs: ev.PrevEventIDs(),
-			StateToFetch: []gomatrixserverlib.StateKeyTuple{
-				{EventType: gomatrixserverlib.MRoomMember, StateKey: r.device.UserID},
-				{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""},
-			},
-		}, &queryRes)
-		if err != nil {
-			wasJoined = false
-			break
-		}
-		var hisVisEvent, membershipEvent *gomatrixserverlib.HeaderedEvent
-		for i := range queryRes.StateEvents {
-			switch queryRes.StateEvents[i].Type() {
-			case gomatrixserverlib.MRoomMember:
-				membershipEvent = queryRes.StateEvents[i]
-			case gomatrixserverlib.MRoomHistoryVisibility:
-				hisVisEvent = queryRes.StateEvents[i]
-			}
-		}
-		if hisVisEvent == nil {
-			return events // apply no filtering as it defaults to Shared.
-		}
-		hisVis, _ := hisVisEvent.HistoryVisibility()
-		if hisVis == "shared" || hisVis == "world_readable" {
-			return events // apply no filtering
-		}
-		if membershipEvent == nil {
-			wasJoined = false
-			break
-		}
-		membership, err := membershipEvent.Membership()
-		if err != nil {
-			wasJoined = false
-			break
-		}
-		if membership != "join" {
-			wasJoined = false
-			break
-		}
-	}
-	if !wasJoined {
-		util.GetLogger(r.ctx).WithField("num_events", len(events)).Warnf("%s was not joined to room during these events, omitting them", r.device.UserID)
-		return []*gomatrixserverlib.HeaderedEvent{}
-	}
-	return result
+	// Apply room history visibility filter
+	startTime := time.Now()
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.db, r.rsAPI, events, nil, r.device.UserID, "messages")
+	logrus.WithFields(logrus.Fields{
+		"duration": time.Since(startTime),
+		"room_id":  r.roomID,
+	}).Debug("applied history visibility (messages)")
+	return gomatrixserverlib.HeaderedToClientEvents(filteredEvents, gomatrixserverlib.FormatAll), start, end, err
 }
 
 func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) {
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 9751670b2..43a75da95 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -161,6 +161,10 @@ type Database interface {
 
 	IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error)
 	UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error
+	// SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found
+	// returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty
+	// string as the membership.
+	SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
 }
 
 type Presence interface {
diff --git a/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
index 29008adef..d68ed8d5f 100644
--- a/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
+++ b/syncapi/storage/postgres/deltas/2022061412000000_history_visibility_column.go
@@ -17,7 +17,10 @@ package deltas
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
+
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func UpAddHistoryVisibilityColumnOutputRoomEvents(ctx context.Context, tx *sql.Tx) error {
@@ -31,6 +34,27 @@ func UpAddHistoryVisibilityColumnOutputRoomEvents(ctx context.Context, tx *sql.T
 	return nil
 }
 
+// UpSetHistoryVisibility sets the history visibility for already stored events.
+// Requires current_room_state and output_room_events to be created.
+func UpSetHistoryVisibility(ctx context.Context, tx *sql.Tx) error {
+	// get the current room history visibilities
+	historyVisibilities, err := currentHistoryVisibilities(ctx, tx)
+	if err != nil {
+		return err
+	}
+
+	// update the history visibility
+	for roomID, hisVis := range historyVisibilities {
+		_, err = tx.ExecContext(ctx, `UPDATE syncapi_output_room_events SET history_visibility = $1 
+                        WHERE type IN ('m.room.message', 'm.room.encrypted') AND room_id = $2 AND history_visibility <> $1`, hisVis, roomID)
+		if err != nil {
+			return fmt.Errorf("failed to update history visibility: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func UpAddHistoryVisibilityColumnCurrentRoomState(ctx context.Context, tx *sql.Tx) error {
 	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS history_visibility SMALLINT NOT NULL DEFAULT 2;
@@ -39,9 +63,40 @@ func UpAddHistoryVisibilityColumnCurrentRoomState(ctx context.Context, tx *sql.T
 	if err != nil {
 		return fmt.Errorf("failed to execute upgrade: %w", err)
 	}
+
 	return nil
 }
 
+// currentHistoryVisibilities returns a map from roomID to current history visibility.
+// If the history visibility was changed after room creation, defaults to joined.
+func currentHistoryVisibilities(ctx context.Context, tx *sql.Tx) (map[string]gomatrixserverlib.HistoryVisibility, error) {
+	rows, err := tx.QueryContext(ctx, `SELECT DISTINCT room_id, headered_event_json FROM syncapi_current_room_state
+		WHERE type = 'm.room.history_visibility' AND state_key = '';
+`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query current room state: %w", err)
+	}
+	defer rows.Close() // nolint: errcheck
+	var eventBytes []byte
+	var roomID string
+	var event gomatrixserverlib.HeaderedEvent
+	var hisVis gomatrixserverlib.HistoryVisibility
+	historyVisibilities := make(map[string]gomatrixserverlib.HistoryVisibility)
+	for rows.Next() {
+		if err = rows.Scan(&roomID, &eventBytes); err != nil {
+			return nil, fmt.Errorf("failed to scan row: %w", err)
+		}
+		if err = json.Unmarshal(eventBytes, &event); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal event: %w", err)
+		}
+		historyVisibilities[roomID] = gomatrixserverlib.HistoryVisibilityJoined
+		if hisVis, err = event.HistoryVisibility(); err == nil && event.Depth() < 10 {
+			historyVisibilities[roomID] = hisVis
+		}
+	}
+	return historyVisibilities, nil
+}
+
 func DownAddHistoryVisibilityColumn(ctx context.Context, tx *sql.Tx) error {
 	_, err := tx.ExecContext(ctx, `
 		ALTER TABLE syncapi_output_room_events DROP COLUMN IF EXISTS history_visibility;
diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go
index 00223c57a..939d6b3f5 100644
--- a/syncapi/storage/postgres/memberships_table.go
+++ b/syncapi/storage/postgres/memberships_table.go
@@ -66,10 +66,14 @@ const selectMembershipCountSQL = "" +
 const selectHeroesSQL = "" +
 	"SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5"
 
+const selectMembershipBeforeSQL = "" +
+	"SELECT membership, topological_pos FROM syncapi_memberships WHERE room_id = $1 and user_id = $2 AND topological_pos <= $3 ORDER BY topological_pos DESC LIMIT 1"
+
 type membershipsStatements struct {
-	upsertMembershipStmt      *sql.Stmt
-	selectMembershipCountStmt *sql.Stmt
-	selectHeroesStmt          *sql.Stmt
+	upsertMembershipStmt        *sql.Stmt
+	selectMembershipCountStmt   *sql.Stmt
+	selectHeroesStmt            *sql.Stmt
+	selectMembershipForUserStmt *sql.Stmt
 }
 
 func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) {
@@ -82,6 +86,7 @@ func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) {
 		{&s.upsertMembershipStmt, upsertMembershipSQL},
 		{&s.selectMembershipCountStmt, selectMembershipCountSQL},
 		{&s.selectHeroesStmt, selectHeroesSQL},
+		{&s.selectMembershipForUserStmt, selectMembershipBeforeSQL},
 	}.Prepare(db)
 }
 
@@ -132,3 +137,20 @@ func (s *membershipsStatements) SelectHeroes(
 	}
 	return heroes, rows.Err()
 }
+
+// SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found
+// returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty
+// string as the membership.
+func (s *membershipsStatements) SelectMembershipForUser(
+	ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64,
+) (membership string, topologyPos int, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt)
+	err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return "leave", 0, nil
+		}
+		return "", 0, err
+	}
+	return membership, topologyPos, nil
+}
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 34ff6700f..8f633640e 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -191,10 +191,12 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
 	}
 
 	m := sqlutil.NewMigrator(db)
-	m.AddMigrations(sqlutil.Migration{
-		Version: "syncapi: add history visibility column (output_room_events)",
-		Up:      deltas.UpAddHistoryVisibilityColumnOutputRoomEvents,
-	})
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "syncapi: add history visibility column (output_room_events)",
+			Up:      deltas.UpAddHistoryVisibilityColumnOutputRoomEvents,
+		},
+	)
 	err = m.Up(context.Background())
 	if err != nil {
 		return nil, err
diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go
index a044716ce..979ff6647 100644
--- a/syncapi/storage/postgres/syncserver.go
+++ b/syncapi/storage/postgres/syncserver.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
 )
 
@@ -97,6 +98,20 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if err != nil {
 		return nil, err
 	}
+
+	// apply migrations which need multiple tables
+	m := sqlutil.NewMigrator(d.db)
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "syncapi: set history visibility for existing events",
+			Up:      deltas.UpSetHistoryVisibility, // Requires current_room_state and output_room_events to be created.
+		},
+	)
+	err = m.Up(base.Context())
+	if err != nil {
+		return nil, err
+	}
+
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Writer:              d.writer,
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 932bda164..a46e55256 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -231,7 +231,7 @@ func (d *Database) AddPeek(
 	return
 }
 
-// DeletePeeks tracks the fact that a user has stopped peeking from the specified
+// DeletePeek tracks the fact that a user has stopped peeking from the specified
 // device. If the peeks was successfully deleted this returns the stream ID it was
 // stored at. Returns an error if there was a problem communicating with the database.
 func (d *Database) DeletePeek(
@@ -372,6 +372,7 @@ func (d *Database) WriteEvent(
 ) (pduPosition types.StreamPosition, returnErr error) {
 	returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		var err error
+		ev.Visibility = historyVisibility
 		pos, err := d.OutputEvents.InsertEvent(
 			ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, historyVisibility,
 		)
@@ -563,7 +564,7 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda
 	return err
 }
 
-// Retrieve the backward topology position, i.e. the position of the
+// GetBackwardTopologyPos retrieves the backward topology position, i.e. the position of the
 // oldest event in the room's topology.
 func (d *Database) GetBackwardTopologyPos(
 	ctx context.Context,
@@ -674,7 +675,7 @@ func (d *Database) fetchMissingStateEvents(
 	return events, nil
 }
 
-// getStateDeltas returns the state deltas between fromPos and toPos,
+// GetStateDeltas returns the state deltas between fromPos and toPos,
 // exclusive of oldPos, inclusive of newPos, for the rooms in which
 // the user has new membership events.
 // A list of joined room IDs is also returned in case the caller needs it.
@@ -812,7 +813,7 @@ func (d *Database) GetStateDeltas(
 	return deltas, joinedRoomIDs, nil
 }
 
-// getStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync
+// GetStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync
 // requests with full_state=true.
 // Fetches full state for all joined rooms and uses selectStateInRange to get
 // updates for other rooms.
@@ -1039,37 +1040,41 @@ func (d *Database) GetUserUnreadNotificationCounts(ctx context.Context, userID s
 	return d.NotificationData.SelectUserUnreadCounts(ctx, userID, from, to)
 }
 
-func (s *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) {
-	return s.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID)
+func (d *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) {
+	return d.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID)
 }
 
-func (s *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) {
-	return s.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter)
+func (d *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	return d.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter)
 }
-func (s *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) {
-	return s.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter)
+func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) {
+	return d.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter)
 }
 
-func (s *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) {
-	return s.Ignores.SelectIgnores(ctx, userID)
+func (d *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) {
+	return d.Ignores.SelectIgnores(ctx, userID)
 }
 
-func (s *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error {
-	return s.Ignores.UpsertIgnores(ctx, userID, ignores)
+func (d *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error {
+	return d.Ignores.UpsertIgnores(ctx, userID, ignores)
 }
 
-func (s *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
-	return s.Presence.UpsertPresence(ctx, nil, userID, statusMsg, presence, lastActiveTS, fromSync)
+func (d *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
+	return d.Presence.UpsertPresence(ctx, nil, userID, statusMsg, presence, lastActiveTS, fromSync)
 }
 
-func (s *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
-	return s.Presence.GetPresenceForUser(ctx, nil, userID)
+func (d *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
+	return d.Presence.GetPresenceForUser(ctx, nil, userID)
 }
 
-func (s *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
-	return s.Presence.GetPresenceAfter(ctx, nil, after, filter)
+func (d *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
+	return d.Presence.GetPresenceAfter(ctx, nil, after, filter)
 }
 
-func (s *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
-	return s.Presence.GetMaxPresenceID(ctx, nil)
+func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
+	return d.Presence.GetMaxPresenceID(ctx, nil)
+}
+
+func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) {
+	return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos)
 }
diff --git a/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
index 079177217..d23f07566 100644
--- a/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
+++ b/syncapi/storage/sqlite3/deltas/2022061412000000_history_visibility_column.go
@@ -17,7 +17,10 @@ package deltas
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
+
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func UpAddHistoryVisibilityColumnOutputRoomEvents(ctx context.Context, tx *sql.Tx) error {
@@ -37,6 +40,27 @@ func UpAddHistoryVisibilityColumnOutputRoomEvents(ctx context.Context, tx *sql.T
 	return nil
 }
 
+// UpSetHistoryVisibility sets the history visibility for already stored events.
+// Requires current_room_state and output_room_events to be created.
+func UpSetHistoryVisibility(ctx context.Context, tx *sql.Tx) error {
+	// get the current room history visibilities
+	historyVisibilities, err := currentHistoryVisibilities(ctx, tx)
+	if err != nil {
+		return err
+	}
+
+	// update the history visibility
+	for roomID, hisVis := range historyVisibilities {
+		_, err = tx.ExecContext(ctx, `UPDATE syncapi_output_room_events SET history_visibility = $1 
+                        WHERE type IN ('m.room.message', 'm.room.encrypted') AND room_id = $2 AND history_visibility <> $1`, hisVis, roomID)
+		if err != nil {
+			return fmt.Errorf("failed to update history visibility: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func UpAddHistoryVisibilityColumnCurrentRoomState(ctx context.Context, tx *sql.Tx) error {
 	// SQLite doesn't have "if exists", so check if the column exists. If the query doesn't return an error, it already exists.
 	// Required for unit tests, as otherwise a duplicate column error will show up.
@@ -51,9 +75,40 @@ func UpAddHistoryVisibilityColumnCurrentRoomState(ctx context.Context, tx *sql.T
 	if err != nil {
 		return fmt.Errorf("failed to execute upgrade: %w", err)
 	}
+
 	return nil
 }
 
+// currentHistoryVisibilities returns a map from roomID to current history visibility.
+// If the history visibility was changed after room creation, defaults to joined.
+func currentHistoryVisibilities(ctx context.Context, tx *sql.Tx) (map[string]gomatrixserverlib.HistoryVisibility, error) {
+	rows, err := tx.QueryContext(ctx, `SELECT DISTINCT room_id, headered_event_json FROM syncapi_current_room_state
+		WHERE type = 'm.room.history_visibility' AND state_key = '';
+`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query current room state: %w", err)
+	}
+	defer rows.Close() // nolint: errcheck
+	var eventBytes []byte
+	var roomID string
+	var event gomatrixserverlib.HeaderedEvent
+	var hisVis gomatrixserverlib.HistoryVisibility
+	historyVisibilities := make(map[string]gomatrixserverlib.HistoryVisibility)
+	for rows.Next() {
+		if err = rows.Scan(&roomID, &eventBytes); err != nil {
+			return nil, fmt.Errorf("failed to scan row: %w", err)
+		}
+		if err = json.Unmarshal(eventBytes, &event); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal event: %w", err)
+		}
+		historyVisibilities[roomID] = gomatrixserverlib.HistoryVisibilityJoined
+		if hisVis, err = event.HistoryVisibility(); err == nil && event.Depth() < 10 {
+			historyVisibilities[roomID] = hisVis
+		}
+	}
+	return historyVisibilities, nil
+}
+
 func DownAddHistoryVisibilityColumn(ctx context.Context, tx *sql.Tx) error {
 	// SQLite doesn't have "if exists", so check if the column exists.
 	_, err := tx.QueryContext(ctx, "SELECT history_visibility FROM syncapi_output_room_events LIMIT 1")
diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go
index e4daa99c1..0c966fca0 100644
--- a/syncapi/storage/sqlite3/memberships_table.go
+++ b/syncapi/storage/sqlite3/memberships_table.go
@@ -66,11 +66,15 @@ const selectMembershipCountSQL = "" +
 const selectHeroesSQL = "" +
 	"SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership IN ($3) LIMIT 5"
 
+const selectMembershipBeforeSQL = "" +
+	"SELECT membership, topological_pos FROM syncapi_memberships WHERE room_id = $1 and user_id = $2 AND topological_pos <= $3 ORDER BY topological_pos DESC LIMIT 1"
+
 type membershipsStatements struct {
 	db                        *sql.DB
 	upsertMembershipStmt      *sql.Stmt
 	selectMembershipCountStmt *sql.Stmt
 	//selectHeroesStmt          *sql.Stmt - prepared at runtime due to variadic
+	selectMembershipForUserStmt *sql.Stmt
 }
 
 func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) {
@@ -84,6 +88,7 @@ func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) {
 	return s, sqlutil.StatementList{
 		{&s.upsertMembershipStmt, upsertMembershipSQL},
 		{&s.selectMembershipCountStmt, selectMembershipCountSQL},
+		{&s.selectMembershipForUserStmt, selectMembershipBeforeSQL},
 		// {&s.selectHeroesStmt, selectHeroesSQL}, - prepared at runtime due to variadic
 	}.Prepare(db)
 }
@@ -148,3 +153,20 @@ func (s *membershipsStatements) SelectHeroes(
 	}
 	return heroes, rows.Err()
 }
+
+// SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found
+// returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty
+// string as the membership.
+func (s *membershipsStatements) SelectMembershipForUser(
+	ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64,
+) (membership string, topologyPos int, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt)
+	err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return "leave", 0, nil
+		}
+		return "", 0, err
+	}
+	return membership, topologyPos, nil
+}
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index de389fa9b..91fd35b5b 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -139,10 +139,12 @@ func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Even
 	}
 
 	m := sqlutil.NewMigrator(db)
-	m.AddMigrations(sqlutil.Migration{
-		Version: "syncapi: add history visibility column (output_room_events)",
-		Up:      deltas.UpAddHistoryVisibilityColumnOutputRoomEvents,
-	})
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "syncapi: add history visibility column (output_room_events)",
+			Up:      deltas.UpAddHistoryVisibilityColumnOutputRoomEvents,
+		},
+	)
 	err = m.Up(context.Background())
 	if err != nil {
 		return nil, err
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index 5c5eb0f55..a84e2bd16 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -16,12 +16,14 @@
 package sqlite3
 
 import (
+	"context"
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas"
 )
 
 // SyncServerDatasource represents a sync server datasource which manages
@@ -41,13 +43,13 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
 		return nil, err
 	}
-	if err = d.prepare(); err != nil {
+	if err = d.prepare(base.Context()); err != nil {
 		return nil, err
 	}
 	return &d, nil
 }
 
-func (d *SyncServerDatasource) prepare() (err error) {
+func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) {
 	if err = d.streamID.Prepare(d.db); err != nil {
 		return err
 	}
@@ -107,6 +109,19 @@ func (d *SyncServerDatasource) prepare() (err error) {
 	if err != nil {
 		return err
 	}
+
+	// apply migrations which need multiple tables
+	m := sqlutil.NewMigrator(d.db)
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "syncapi: set history visibility for existing events",
+			Up:      deltas.UpSetHistoryVisibility, // Requires current_room_state and output_room_events to be created.
+		},
+	)
+	err = m.Up(ctx)
+	if err != nil {
+		return err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		Writer:              d.writer,
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index eda5ef3e6..a62818e9b 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -12,20 +12,22 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 var ctx = context.Background()
 
-func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func(), func()) {
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
-	db, err := storage.NewSyncServerDatasource(nil, &config.DatabaseOptions{
+	base, closeBase := testrig.CreateBaseDendrite(t, dbType)
+	db, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	})
 	if err != nil {
 		t.Fatalf("NewSyncServerDatasource returned %s", err)
 	}
-	return db, close
+	return db, close, closeBase
 }
 
 func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) {
@@ -51,8 +53,9 @@ func TestWriteEvents(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		alice := test.NewUser(t)
 		r := test.NewRoom(t, alice)
-		db, close := MustCreateDatabase(t, dbType)
+		db, close, closeBase := MustCreateDatabase(t, dbType)
 		defer close()
+		defer closeBase()
 		MustWriteEvents(t, db, r.Events())
 	})
 }
@@ -60,8 +63,9 @@ func TestWriteEvents(t *testing.T) {
 // These tests assert basic functionality of RecentEvents for PDUs
 func TestRecentEventsPDU(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		db, close := MustCreateDatabase(t, dbType)
+		db, close, closeBase := MustCreateDatabase(t, dbType)
 		defer close()
+		defer closeBase()
 		alice := test.NewUser(t)
 		// dummy room to make sure SQL queries are filtering on room ID
 		MustWriteEvents(t, db, test.NewRoom(t, alice).Events())
@@ -163,8 +167,9 @@ func TestRecentEventsPDU(t *testing.T) {
 // The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token
 func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		db, close := MustCreateDatabase(t, dbType)
+		db, close, closeBase := MustCreateDatabase(t, dbType)
 		defer close()
+		defer closeBase()
 		alice := test.NewUser(t)
 		r := test.NewRoom(t, alice)
 		for i := 0; i < 10; i++ {
@@ -404,8 +409,9 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 	bob := test.NewUser(t)
 	deviceID := "one"
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		db, close := MustCreateDatabase(t, dbType)
+		db, close, closeBase := MustCreateDatabase(t, dbType)
 		defer close()
+		defer closeBase()
 		// At this point there should be no messages. We haven't sent anything
 		// yet.
 		_, events, err := db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100)
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index d68351d4c..468d26aca 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -185,6 +185,7 @@ type Memberships interface {
 	UpsertMembership(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error
 	SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error)
 	SelectHeroes(ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string) (heroes []string, err error)
+	SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
 }
 
 type NotificationData interface {
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 1ad3adc4c..136cbea5a 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -10,10 +10,13 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/internal"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 	"go.uber.org/atomic"
 
@@ -123,7 +126,7 @@ func (p *PDUStreamProvider) CompleteSync(
 			defer reqWaitGroup.Done()
 
 			jr, jerr := p.getJoinResponseForCompleteSync(
-				ctx, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device,
+				ctx, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, false,
 			)
 			if jerr != nil {
 				req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
@@ -149,7 +152,7 @@ func (p *PDUStreamProvider) CompleteSync(
 		if !peek.Deleted {
 			var jr *types.JoinResponse
 			jr, err = p.getJoinResponseForCompleteSync(
-				ctx, peek.RoomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device,
+				ctx, peek.RoomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, true,
 			)
 			if err != nil {
 				req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed")
@@ -281,12 +284,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 			}
 		}
 	}
-	if len(recentEvents) > 0 {
-		updateLatestPosition(recentEvents[len(recentEvents)-1].EventID())
-	}
-	if len(delta.StateEvents) > 0 {
-		updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID())
-	}
 
 	if stateFilter.LazyLoadMembers {
 		delta.StateEvents, err = p.lazyLoadMembers(
@@ -306,6 +303,19 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		}
 	}
 
+	// Applies the history visibility rules
+	events, err := applyHistoryVisibilityFilter(ctx, p.DB, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents)
+	if err != nil {
+		logrus.WithError(err).Error("unable to apply history visibility filter")
+	}
+
+	if len(events) > 0 {
+		updateLatestPosition(events[len(events)-1].EventID())
+	}
+	if len(delta.StateEvents) > 0 {
+		updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID())
+	}
+
 	switch delta.Membership {
 	case gomatrixserverlib.Join:
 		jr := types.NewJoinResponse()
@@ -313,14 +323,17 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 			p.addRoomSummary(ctx, jr, delta.RoomID, device.UserID, latestPosition)
 		}
 		jr.Timeline.PrevBatch = &prevBatch
-		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
-		jr.Timeline.Limited = limited
+		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync)
+		// If we are limited by the filter AND the history visibility filter
+		// didn't "remove" events, return that the response is limited.
+		jr.Timeline.Limited = limited && len(events) == len(recentEvents)
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
 		res.Rooms.Join[delta.RoomID] = *jr
 
 	case gomatrixserverlib.Peek:
 		jr := types.NewJoinResponse()
 		jr.Timeline.PrevBatch = &prevBatch
+		// TODO: Apply history visibility on peeked rooms
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		jr.Timeline.Limited = limited
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
@@ -330,12 +343,12 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		fallthrough // transitions to leave are the same as ban
 
 	case gomatrixserverlib.Ban:
-		// TODO: recentEvents may contain events that this user is not allowed to see because they are
-		//       no longer in the room.
 		lr := types.NewLeaveResponse()
 		lr.Timeline.PrevBatch = &prevBatch
-		lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
-		lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
+		lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync)
+		// If we are limited by the filter AND the history visibility filter
+		// didn't "remove" events, return that the response is limited.
+		lr.Timeline.Limited = limited && len(events) == len(recentEvents)
 		lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
 		res.Rooms.Leave[delta.RoomID] = *lr
 	}
@@ -343,6 +356,41 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	return latestPosition, nil
 }
 
+// applyHistoryVisibilityFilter gets the current room state and supplies it to ApplyHistoryVisibilityFilter, to make
+// sure we always return the required events in the timeline.
+func applyHistoryVisibilityFilter(
+	ctx context.Context,
+	db storage.Database,
+	rsAPI roomserverAPI.SyncRoomserverAPI,
+	roomID, userID string,
+	limit int,
+	recentEvents []*gomatrixserverlib.HeaderedEvent,
+) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	// We need to make sure we always include the latest states events, if they are in the timeline.
+	// We grep at least limit * 2 events, to ensure we really get the needed events.
+	stateEvents, err := db.CurrentState(ctx, roomID, &gomatrixserverlib.StateFilter{Limit: limit * 2}, nil)
+	if err != nil {
+		// Not a fatal error, we can continue without the stateEvents,
+		// they are only needed if there are state events in the timeline.
+		logrus.WithError(err).Warnf("failed to get current room state")
+	}
+	alwaysIncludeIDs := make(map[string]struct{}, len(stateEvents))
+	for _, ev := range stateEvents {
+		alwaysIncludeIDs[ev.EventID()] = struct{}{}
+	}
+	startTime := time.Now()
+	events, err := internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync")
+	if err != nil {
+
+		return nil, err
+	}
+	logrus.WithFields(logrus.Fields{
+		"duration": time.Since(startTime),
+		"room_id":  roomID,
+	}).Debug("applied history visibility (sync)")
+	return events, nil
+}
+
 func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) {
 	// Work out how many members are in the room.
 	joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition)
@@ -390,6 +438,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	eventFilter *gomatrixserverlib.RoomEventFilter,
 	wantFullState bool,
 	device *userapi.Device,
+	isPeek bool,
 ) (jr *types.JoinResponse, err error) {
 	jr = types.NewJoinResponse()
 	// TODO: When filters are added, we may need to call this multiple times to get enough events.
@@ -404,33 +453,6 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		return
 	}
 
-	// TODO FIXME: We don't fully implement history visibility yet. To avoid leaking events which the
-	// user shouldn't see, we check the recent events and remove any prior to the join event of the user
-	// which is equiv to history_visibility: joined
-	joinEventIndex := -1
-	for i := len(recentStreamEvents) - 1; i >= 0; i-- {
-		ev := recentStreamEvents[i]
-		if ev.Type() == gomatrixserverlib.MRoomMember && ev.StateKeyEquals(device.UserID) {
-			membership, _ := ev.Membership()
-			if membership == "join" {
-				joinEventIndex = i
-				if i > 0 {
-					// the create event happens before the first join, so we should cut it at that point instead
-					if recentStreamEvents[i-1].Type() == gomatrixserverlib.MRoomCreate && recentStreamEvents[i-1].StateKeyEquals("") {
-						joinEventIndex = i - 1
-						break
-					}
-				}
-				break
-			}
-		}
-	}
-	if joinEventIndex != -1 {
-		// cut all events earlier than the join (but not the join itself)
-		recentStreamEvents = recentStreamEvents[joinEventIndex:]
-		limited = false // so clients know not to try to backpaginate
-	}
-
 	// Work our way through the timeline events and pick out the event IDs
 	// of any state events that appear in the timeline. We'll specifically
 	// exclude them at the next step, so that we don't get duplicate state
@@ -474,6 +496,19 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents)
 	stateEvents = removeDuplicates(stateEvents, recentEvents)
 
+	events := recentEvents
+	// Only apply history visibility checks if the response is for joined rooms
+	if !isPeek {
+		events, err = applyHistoryVisibilityFilter(ctx, p.DB, p.rsAPI, roomID, device.UserID, eventFilter.Limit, recentEvents)
+		if err != nil {
+			logrus.WithError(err).Error("unable to apply history visibility filter")
+		}
+	}
+
+	// If we are limited by the filter AND the history visibility filter
+	// didn't "remove" events, return that the response is limited.
+	limited = limited && len(events) == len(recentEvents)
+
 	if stateFilter.LazyLoadMembers {
 		if err != nil {
 			return nil, err
@@ -488,8 +523,10 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	}
 
 	jr.Timeline.PrevBatch = prevBatch
-	jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
-	jr.Timeline.Limited = limited
+	jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync)
+	// If we are limited by the filter AND the history visibility filter
+	// didn't "remove" events, return that the response is limited.
+	jr.Timeline.Limited = limited && len(events) == len(recentEvents)
 	jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync)
 	return jr, nil
 }
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 931fef883..dc073a16e 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -54,6 +55,16 @@ func (s *syncRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *rsap
 	return nil
 }
 
+func (s *syncRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *rsapi.QueryMembershipForUserRequest, res *rsapi.QueryMembershipForUserResponse) error {
+	res.IsRoomForgotten = false
+	res.RoomExists = true
+	return nil
+}
+
+func (s *syncRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, req *rsapi.QueryMembershipAtEventRequest, res *rsapi.QueryMembershipAtEventResponse) error {
+	return nil
+}
+
 type syncUserAPI struct {
 	userapi.SyncUserAPI
 	accounts []userapi.Device
@@ -107,7 +118,7 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
 
 	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
-	msgs := toNATSMsgs(t, base, room.Events())
+	msgs := toNATSMsgs(t, base, room.Events()...)
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
 	testrig.MustPublishMsgs(t, jsctx, msgs...)
 
@@ -200,7 +211,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
 	// m.room.power_levels
 	// m.room.join_rules
 	// m.room.history_visibility
-	msgs := toNATSMsgs(t, base, room.Events())
+	msgs := toNATSMsgs(t, base, room.Events()...)
 	sinceTokens := make([]string, len(msgs))
 	AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{})
 	for i, msg := range msgs {
@@ -315,6 +326,174 @@ func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) {
 
 }
 
+// This is mainly what Sytest is doing in "test_history_visibility"
+func TestMessageHistoryVisibility(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		testHistoryVisibility(t, dbType)
+	})
+}
+
+func testHistoryVisibility(t *testing.T, dbType test.DBType) {
+	type result struct {
+		seeWithoutJoin bool
+		seeBeforeJoin  bool
+		seeAfterInvite bool
+	}
+
+	// create the users
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	bobDev := userapi.Device{
+		ID:          "BOBID",
+		UserID:      bob.ID,
+		AccessToken: "BOD_BEARER_TOKEN",
+		DisplayName: "BOB",
+	}
+
+	ctx := context.Background()
+	// check guest and normal user accounts
+	for _, accType := range []userapi.AccountType{userapi.AccountTypeGuest, userapi.AccountTypeUser} {
+		testCases := []struct {
+			historyVisibility gomatrixserverlib.HistoryVisibility
+			wantResult        result
+		}{
+			{
+				historyVisibility: gomatrixserverlib.HistoryVisibilityWorldReadable,
+				wantResult: result{
+					seeWithoutJoin: true,
+					seeBeforeJoin:  true,
+					seeAfterInvite: true,
+				},
+			},
+			{
+				historyVisibility: gomatrixserverlib.HistoryVisibilityShared,
+				wantResult: result{
+					seeWithoutJoin: false,
+					seeBeforeJoin:  true,
+					seeAfterInvite: true,
+				},
+			},
+			{
+				historyVisibility: gomatrixserverlib.HistoryVisibilityInvited,
+				wantResult: result{
+					seeWithoutJoin: false,
+					seeBeforeJoin:  false,
+					seeAfterInvite: true,
+				},
+			},
+			{
+				historyVisibility: gomatrixserverlib.HistoryVisibilityJoined,
+				wantResult: result{
+					seeWithoutJoin: false,
+					seeBeforeJoin:  false,
+					seeAfterInvite: false,
+				},
+			},
+		}
+
+		bobDev.AccountType = accType
+		userType := "guest"
+		if accType == userapi.AccountTypeUser {
+			userType = "real user"
+		}
+
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
+
+		jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
+
+		// Use the actual internal roomserver API
+		rsAPI := roomserver.NewInternalAPI(base)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{bobDev}}, rsAPI, &syncKeyAPI{})
+
+		for _, tc := range testCases {
+			testname := fmt.Sprintf("%s - %s", tc.historyVisibility, userType)
+			t.Run(testname, func(t *testing.T) {
+				// create a room with the given visibility
+				room := test.NewRoom(t, alice, test.RoomHistoryVisibility(tc.historyVisibility))
+
+				// send the events/messages to NATS to create the rooms
+				beforeJoinEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("Before invite in a %s room", tc.historyVisibility)})
+				eventsToSend := append(room.Events(), beforeJoinEv)
+				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
+					t.Fatalf("failed to send events: %v", err)
+				}
+
+				// There is only one event, we expect only to be able to see this, if the room is world_readable
+				w := httptest.NewRecorder()
+				base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": bobDev.AccessToken,
+					"dir":          "b",
+				})))
+				if w.Code != 200 {
+					t.Logf("%s", w.Body.String())
+					t.Fatalf("got HTTP %d want %d", w.Code, 200)
+				}
+				// We only care about the returned events at this point
+				var res struct {
+					Chunk []gomatrixserverlib.ClientEvent `json:"chunk"`
+				}
+				if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+					t.Errorf("failed to decode response body: %s", err)
+				}
+
+				verifyEventVisible(t, tc.wantResult.seeWithoutJoin, beforeJoinEv, res.Chunk)
+
+				// Create invite, a message, join the room and create another message.
+				inviteEv := room.CreateAndInsert(t, alice, "m.room.member", map[string]interface{}{"membership": "invite"}, test.WithStateKey(bob.ID))
+				afterInviteEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("After invite in a %s room", tc.historyVisibility)})
+				joinEv := room.CreateAndInsert(t, bob, "m.room.member", map[string]interface{}{"membership": "join"}, test.WithStateKey(bob.ID))
+				msgEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("After join in a %s room", tc.historyVisibility)})
+
+				eventsToSend = append([]*gomatrixserverlib.HeaderedEvent{}, inviteEv, afterInviteEv, joinEv, msgEv)
+
+				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
+					t.Fatalf("failed to send events: %v", err)
+				}
+
+				// Verify the messages after/before invite are visible or not
+				w = httptest.NewRecorder()
+				base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": bobDev.AccessToken,
+					"dir":          "b",
+				})))
+				if w.Code != 200 {
+					t.Logf("%s", w.Body.String())
+					t.Fatalf("got HTTP %d want %d", w.Code, 200)
+				}
+				if err := json.NewDecoder(w.Body).Decode(&res); err != nil {
+					t.Errorf("failed to decode response body: %s", err)
+				}
+				// verify results
+				verifyEventVisible(t, tc.wantResult.seeBeforeJoin, beforeJoinEv, res.Chunk)
+				verifyEventVisible(t, tc.wantResult.seeAfterInvite, afterInviteEv, res.Chunk)
+			})
+		}
+	}
+}
+
+func verifyEventVisible(t *testing.T, wantVisible bool, wantVisibleEvent *gomatrixserverlib.HeaderedEvent, chunk []gomatrixserverlib.ClientEvent) {
+	t.Helper()
+	if wantVisible {
+		for _, ev := range chunk {
+			if ev.EventID == wantVisibleEvent.EventID() {
+				return
+			}
+		}
+		t.Fatalf("expected to see event %s but didn't: %+v", wantVisibleEvent.EventID(), chunk)
+	} else {
+		for _, ev := range chunk {
+			if ev.EventID == wantVisibleEvent.EventID() {
+				t.Fatalf("expected not to see event %s: %+v", wantVisibleEvent.EventID(), string(ev.Content))
+			}
+		}
+	}
+}
+
 func TestSendToDevice(t *testing.T) {
 	test.WithAllDatabases(t, testSendToDevice)
 }
@@ -448,7 +627,7 @@ func testSendToDevice(t *testing.T, dbType test.DBType) {
 	}
 }
 
-func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
+func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input ...*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
 	result := make([]*nats.Msg, len(input))
 	for i, ev := range input {
 		var addsStateIDs []string
@@ -460,6 +639,7 @@ func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input []*gomatrixserverli
 			NewRoomEvent: &rsapi.OutputNewRoomEvent{
 				Event:             ev,
 				AddsStateEventIDs: addsStateIDs,
+				HistoryVisibility: ev.Visibility,
 			},
 		})
 	}
diff --git a/sytest-whitelist b/sytest-whitelist
index 2a145291f..88dfe920b 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -110,8 +110,6 @@ Newly joined room is included in an incremental sync
 User is offline if they set_presence=offline in their sync
 Changes to state are included in an incremental sync
 A change to displayname should appear in incremental /sync
-Current state appears in timeline in private history
-Current state appears in timeline in private history with many messages before
 Rooms a user is invited to appear in an initial sync
 Rooms a user is invited to appear in an incremental sync
 Sync can be polled for updates
@@ -458,7 +456,6 @@ After changing password, a different session no longer works by default
 Read markers appear in incremental v2 /sync
 Read markers appear in initial v2 /sync
 Read markers can be updated
-Local users can peek into world_readable rooms by room ID
 We can't peek into rooms with shared history_visibility
 We can't peek into rooms with invited history_visibility
 We can't peek into rooms with joined history_visibility
@@ -720,4 +717,26 @@ Setting state twice is idempotent
 Joining room twice is idempotent
 Inbound federation can return missing events for shared visibility
 Inbound federation ignores redactions from invalid servers room > v3
+Joining room twice is idempotent
+Getting messages going forward is limited for a departed room (SPEC-216)
+m.room.history_visibility == "shared" allows/forbids appropriately for Guest users
+m.room.history_visibility == "invited" allows/forbids appropriately for Guest users
+m.room.history_visibility == "default" allows/forbids appropriately for Guest users
+m.room.history_visibility == "shared" allows/forbids appropriately for Real users
+m.room.history_visibility == "invited" allows/forbids appropriately for Real users
+m.room.history_visibility == "default" allows/forbids appropriately for Real users
+Guest users can sync from world_readable guest_access rooms if joined
+Guest users can sync from shared guest_access rooms if joined
+Guest users can sync from invited guest_access rooms if joined
+Guest users can sync from joined guest_access rooms if joined
+Guest users can sync from default guest_access rooms if joined
+Real users can sync from world_readable guest_access rooms if joined
+Real users can sync from shared guest_access rooms if joined
+Real users can sync from invited guest_access rooms if joined
+Real users can sync from joined guest_access rooms if joined
+Real users can sync from default guest_access rooms if joined
+Only see history_visibility changes on boundaries
+Current state appears in timeline in private history
+Current state appears in timeline in private history with many messages before
+Local users can peek into world_readable rooms by room ID
 Newly joined room includes presence in incremental sync
\ No newline at end of file
diff --git a/test/room.go b/test/room.go
index 6ae403b3f..94eb51bbe 100644
--- a/test/room.go
+++ b/test/room.go
@@ -37,10 +37,11 @@ var (
 )
 
 type Room struct {
-	ID      string
-	Version gomatrixserverlib.RoomVersion
-	preset  Preset
-	creator *User
+	ID         string
+	Version    gomatrixserverlib.RoomVersion
+	preset     Preset
+	visibility gomatrixserverlib.HistoryVisibility
+	creator    *User
 
 	authEvents   gomatrixserverlib.AuthEvents
 	currentState map[string]*gomatrixserverlib.HeaderedEvent
@@ -61,6 +62,7 @@ func NewRoom(t *testing.T, creator *User, modifiers ...roomModifier) *Room {
 		preset:       PresetPublicChat,
 		Version:      gomatrixserverlib.RoomVersionV9,
 		currentState: make(map[string]*gomatrixserverlib.HeaderedEvent),
+		visibility:   gomatrixserverlib.HistoryVisibilityShared,
 	}
 	for _, m := range modifiers {
 		m(t, r)
@@ -97,10 +99,14 @@ func (r *Room) insertCreateEvents(t *testing.T) {
 		fallthrough
 	case PresetPrivateChat:
 		joinRule.JoinRule = "invite"
-		hisVis.HistoryVisibility = "shared"
+		hisVis.HistoryVisibility = gomatrixserverlib.HistoryVisibilityShared
 	case PresetPublicChat:
 		joinRule.JoinRule = "public"
-		hisVis.HistoryVisibility = "shared"
+		hisVis.HistoryVisibility = gomatrixserverlib.HistoryVisibilityShared
+	}
+
+	if r.visibility != "" {
+		hisVis.HistoryVisibility = r.visibility
 	}
 
 	r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomCreate, map[string]interface{}{
@@ -183,7 +189,9 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten
 	if err = gomatrixserverlib.Allowed(ev, &r.authEvents); err != nil {
 		t.Fatalf("CreateEvent[%s]: failed to verify event was allowed: %s", eventType, err)
 	}
-	return ev.Headered(r.Version)
+	headeredEvent := ev.Headered(r.Version)
+	headeredEvent.Visibility = r.visibility
+	return headeredEvent
 }
 
 // Add a new event to this room DAG. Not thread-safe.
@@ -242,6 +250,12 @@ func RoomPreset(p Preset) roomModifier {
 	}
 }
 
+func RoomHistoryVisibility(vis gomatrixserverlib.HistoryVisibility) roomModifier {
+	return func(t *testing.T, r *Room) {
+		r.visibility = vis
+	}
+}
+
 func RoomVersion(ver gomatrixserverlib.RoomVersion) roomModifier {
 	return func(t *testing.T, r *Room) {
 		r.Version = ver
diff --git a/test/user.go b/test/user.go
index 0020098a5..692eae351 100644
--- a/test/user.go
+++ b/test/user.go
@@ -20,6 +20,7 @@ import (
 	"sync/atomic"
 	"testing"
 
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -45,7 +46,8 @@ var (
 )
 
 type User struct {
-	ID string
+	ID          string
+	accountType api.AccountType
 	// key ID and private key of the server who has this user, if known.
 	keyID   gomatrixserverlib.KeyID
 	privKey ed25519.PrivateKey
@@ -62,6 +64,12 @@ func WithSigningServer(srvName gomatrixserverlib.ServerName, keyID gomatrixserve
 	}
 }
 
+func WithAccountType(accountType api.AccountType) UserOpt {
+	return func(u *User) {
+		u.accountType = accountType
+	}
+}
+
 func NewUser(t *testing.T, opts ...UserOpt) *User {
 	counter := atomic.AddInt64(&userIDCounter, 1)
 	var u User

From a01af55ec694f33d68ecf5e2dc6cfc3e4ff3c2b9 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 11 Aug 2022 17:34:09 +0100
Subject: [PATCH 260/391] Restore the room version cache in the roomserver
 internal API HTTP client

---
 roomserver/inthttp/client.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index e9387bd97..a1dfc6aac 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -425,10 +425,14 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom(
 		response.RoomVersion = roomVersion
 		return nil
 	}
-	return httputil.CallInternalRPCAPI(
+	err := httputil.CallInternalRPCAPI(
 		"QueryRoomVersionForRoom", h.roomserverURL+RoomserverQueryRoomVersionForRoomPath,
 		h.httpClient, ctx, request, response,
 	)
+	if err == nil {
+		h.cache.StoreRoomVersion(request.RoomID, response.RoomVersion)
+	}
+	return err
 }
 
 func (h *httpRoomserverInternalAPI) QueryCurrentState(

From 2b352915a1722f4971283272f34d219ac6ee32af Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 11 Aug 2022 17:40:57 +0100
Subject: [PATCH 261/391] Update `golangci-lint` component in GHA workflow

---
 .github/workflows/dendrite.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 19ebd760c..9fa6cf197 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -67,7 +67,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
 
   # run go test with different go versions
   test:

From fad3ac8e787cebd231d6fe115980f6335bc94353 Mon Sep 17 00:00:00 2001
From: Tak Wai Wong <64229756+tak-hntlabs@users.noreply.github.com>
Date: Fri, 12 Aug 2022 01:12:05 -0700
Subject: [PATCH 262/391] Protect user_interactive reads and writes with locks
 (#2635)

* Protect user_interactive reads and writes with locks

* Ignore golangci-lint false positive

* fix lint

Co-authored-by: Tak Wai Wong <tak@hntlabs.com>
---
 .github/workflows/dendrite.yml     | 18 ++++++++++------
 clientapi/auth/user_interactive.go | 34 ++++++++++++++++++++++++------
 internal/caching/impl_ristretto.go |  2 +-
 3 files changed, 40 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 9fa6cf197..3044b0b9d 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ false }} # disable for now
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: 1.18
 
@@ -66,6 +66,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
 
@@ -101,7 +105,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - uses: actions/cache@v3
@@ -133,7 +137,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies x86
@@ -167,7 +171,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup Go ${{ matrix.go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies
@@ -208,7 +212,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: "1.18"
       - uses: actions/cache@v3
@@ -233,7 +237,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: "1.18"
       - uses: actions/cache@v3
diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go
index 82ecf674c..9971bf8a4 100644
--- a/clientapi/auth/user_interactive.go
+++ b/clientapi/auth/user_interactive.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/http"
+	"sync"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -102,6 +103,7 @@ type userInteractiveFlow struct {
 // the user already has a valid access token, but we want to double-check
 // that it isn't stolen by re-authenticating them.
 type UserInteractive struct {
+	sync.RWMutex
 	Flows []userInteractiveFlow
 	// Map of login type to implementation
 	Types map[string]Type
@@ -128,6 +130,8 @@ func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI)
 }
 
 func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
+	u.RLock()
+	defer u.RUnlock()
 	for _, f := range u.Flows {
 		if len(f.Stages) == 1 && f.Stages[0] == authType {
 			return true
@@ -137,8 +141,10 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
 }
 
 func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
+	u.Lock()
 	// TODO: Handle multi-stage flows
 	delete(u.Sessions, sessionID)
+	u.Unlock()
 }
 
 type Challenge struct {
@@ -150,12 +156,17 @@ type Challenge struct {
 }
 
 // Challenge returns an HTTP 401 with the supported flows for authenticating
-func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse {
+func (u *UserInteractive) challenge(sessionID string) *util.JSONResponse {
+	u.RLock()
+	completed := u.Sessions[sessionID]
+	flows := u.Flows
+	u.RUnlock()
+
 	return &util.JSONResponse{
 		Code: 401,
 		JSON: Challenge{
-			Completed: u.Sessions[sessionID],
-			Flows:     u.Flows,
+			Completed: completed,
+			Flows:     flows,
 			Session:   sessionID,
 			Params:    make(map[string]interface{}),
 		},
@@ -170,8 +181,10 @@ func (u *UserInteractive) NewSession() *util.JSONResponse {
 		res := jsonerror.InternalServerError()
 		return &res
 	}
+	u.Lock()
 	u.Sessions[sessionID] = []string{}
-	return u.Challenge(sessionID)
+	u.Unlock()
+	return u.challenge(sessionID)
 }
 
 // ResponseWithChallenge mixes together a JSON body (e.g an error with errcode/message) with the
@@ -184,7 +197,7 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter
 		return &ise
 	}
 	_ = json.Unmarshal(b, &mixedObjects)
-	challenge := u.Challenge(sessionID)
+	challenge := u.challenge(sessionID)
 	b, err = json.Marshal(challenge.JSON)
 	if err != nil {
 		ise := jsonerror.InternalServerError()
@@ -213,7 +226,11 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 
 	// extract the type so we know which login type to use
 	authType := gjson.GetBytes(bodyBytes, "auth.type").Str
+
+	u.RLock()
 	loginType, ok := u.Types[authType]
+	u.RUnlock()
+
 	if !ok {
 		return nil, &util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -223,7 +240,12 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 
 	// retrieve the session
 	sessionID := gjson.GetBytes(bodyBytes, "auth.session").Str
-	if _, ok = u.Sessions[sessionID]; !ok {
+
+	u.RLock()
+	_, ok = u.Sessions[sessionID]
+	u.RUnlock()
+
+	if !ok {
 		// if the login type is part of a single stage flow then allow them to omit the session ID
 		if !u.IsSingleStageFlow(authType) {
 			return nil, &util.JSONResponse{
diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go
index fc0c8cc0f..49292d0dc 100644
--- a/internal/caching/impl_ristretto.go
+++ b/internal/caching/impl_ristretto.go
@@ -146,7 +146,7 @@ func (c *RistrettoCostedCachePartition[K, V]) Set(key K, value V) {
 }
 
 type RistrettoCachePartition[K keyable, V any] struct {
-	cache   *ristretto.Cache
+	cache   *ristretto.Cache //nolint:all,unused
 	Prefix  byte
 	Mutable bool
 	MaxAge  time.Duration

From 48600d554096d54d12a6d3834f8f5f45506e3026 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:00:07 +0200
Subject: [PATCH 263/391] Use `/admin/v1/register` in `create-account` (#2484)

* Get all account data on CompleteSync

* Revert "Get all account data on CompleteSync"

This reverts commit 44a3e566d8fb940b0b757aea9b8408fa19ea9f54.

* Use /_synapse/admin/v1/register to create account

* Linting

* Linter again :)

* Update docs

* Use HTTP API to reset password, add option to User API `PerformPasswordUpdate` to invalidate sessions

* Fix routing name

* Tell me more about what went wrong

* Deprecate the `-reset-password` flag, document the new API

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 clientapi/routing/admin.go           |  78 ++++++++++---
 clientapi/routing/routing.go         |  14 ++-
 cmd/create-account/main.go           | 166 ++++++++++++++++-----------
 docs/administration/1_createusers.md |  12 +-
 docs/administration/4_adminapi.md    |  19 ++-
 internal/httputil/httpapi.go         |  18 +++
 userapi/api/api.go                   |   5 +-
 userapi/internal/api.go              |   5 +
 8 files changed, 227 insertions(+), 90 deletions(-)

diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index a8dd0e64f..0c5f8c167 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -1,23 +1,20 @@
 package routing
 
 import (
+	"encoding/json"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	if device.AccountType != userapi.AccountTypeAdmin {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-		}
-	}
+func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -50,13 +47,7 @@ func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserv
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	if device.AccountType != userapi.AccountTypeAdmin {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-		}
-	}
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -68,6 +59,16 @@ func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserv
 			JSON: jsonerror.MissingArgument("Expecting user ID."),
 		}
 	}
+	_, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return util.MessageResponse(http.StatusBadRequest, err.Error())
+	}
+	if domain != cfg.Matrix.ServerName {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("User ID must belong to this server."),
+		}
+	}
 	res := &roomserverAPI.PerformAdminEvacuateUserResponse{}
 	if err := rsAPI.PerformAdminEvacuateUser(
 		req.Context(),
@@ -88,3 +89,52 @@ func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserv
 		},
 	}
 }
+
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	localpart, ok := vars["localpart"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting user localpart."),
+		}
+	}
+	request := struct {
+		Password string `json:"password"`
+	}{}
+	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
+		}
+	}
+	if request.Password == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting non-empty password."),
+		}
+	}
+	updateReq := &userapi.PerformPasswordUpdateRequest{
+		Localpart:     localpart,
+		Password:      request.Password,
+		LogoutDevices: true,
+	}
+	updateRes := &userapi.PerformPasswordUpdateResponse{}
+	if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Failed to perform password update: " + err.Error()),
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct {
+			Updated bool `json:"password_updated"`
+		}{
+			Updated: updateRes.PasswordUpdated,
+		},
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index ced4fdbcf..2063a008f 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -144,17 +144,23 @@ func Setup(
 	}
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
-		httputil.MakeAuthAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateRoom(req, device, rsAPI)
+		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminEvacuateRoom(req, cfg, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
-		httputil.MakeAuthAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateUser(req, device, rsAPI)
+		httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminEvacuateUser(req, cfg, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/resetPassword/{localpart}",
+		httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminResetPassword(req, cfg, device, userAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 92179a049..44d5691c2 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -15,20 +15,26 @@
 package main
 
 import (
-	"context"
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"regexp"
 	"strings"
+	"time"
+
+	"github.com/tidwall/gjson"
 
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/term"
+
+	"github.com/matrix-org/dendrite/setup"
 )
 
 const usage = `Usage: %s
@@ -58,12 +64,17 @@ var (
 	password           = flag.String("password", "", "The password to associate with the account")
 	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
 	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
-	pwdLess            = flag.Bool("passwordless", false, "Create a passwordless account, e.g. if only an accesstoken is required")
 	isAdmin            = flag.Bool("admin", false, "Create an admin account")
 	resetPassword      = flag.Bool("reset-password", false, "Resets the password for the given username")
+	serverURL          = flag.String("url", "https://localhost:8448", "The URL to connect to.")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
 
+var cl = http.Client{
+	Timeout:   time.Second * 10,
+	Transport: http.DefaultTransport,
+}
+
 func main() {
 	name := os.Args[0]
 	flag.Usage = func() {
@@ -72,15 +83,15 @@ func main() {
 	}
 	cfg := setup.ParseFlags(true)
 
+	if *resetPassword {
+		logrus.Fatalf("The reset-password flag has been replaced by the POST /_dendrite/admin/resetPassword/{localpart} admin API.")
+	}
+
 	if *username == "" {
 		flag.Usage()
 		os.Exit(1)
 	}
 
-	if *pwdLess && *resetPassword {
-		logrus.Fatalf("Can not reset to an empty password, unable to login afterwards.")
-	}
-
 	if !validUsernameRegex.MatchString(*username) {
 		logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
 		os.Exit(1)
@@ -90,67 +101,94 @@ func main() {
 		logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
 	}
 
-	var pass string
-	var err error
-	if !*pwdLess {
-		pass, err = getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
-		if err != nil {
-			logrus.Fatalln(err)
-		}
-	}
-
-	// avoid warning about open registration
-	cfg.ClientAPI.RegistrationDisabled = true
-
-	b := base.NewBaseDendrite(cfg, "")
-	defer b.Close() // nolint: errcheck
-
-	accountDB, err := storage.NewUserAPIDatabase(
-		b,
-		&cfg.UserAPI.AccountDatabase,
-		cfg.Global.ServerName,
-		cfg.UserAPI.BCryptCost,
-		cfg.UserAPI.OpenIDTokenLifetimeMS,
-		0, // TODO
-		cfg.Global.ServerNotices.LocalPart,
-	)
+	pass, err := getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
 	if err != nil {
-		logrus.WithError(err).Fatalln("Failed to connect to the database")
+		logrus.Fatalln(err)
 	}
 
-	accType := api.AccountTypeUser
-	if *isAdmin {
-		accType = api.AccountTypeAdmin
-	}
-
-	available, err := accountDB.CheckAccountAvailability(context.Background(), *username)
-	if err != nil {
-		logrus.Fatalln("Unable check username existence.")
-	}
-	if *resetPassword {
-		if available {
-			logrus.Fatalln("Username could not be found.")
-		}
-		err = accountDB.SetPassword(context.Background(), *username, pass)
-		if err != nil {
-			logrus.Fatalf("Failed to update password for user %s: %s", *username, err.Error())
-		}
-		if _, err = accountDB.RemoveAllDevices(context.Background(), *username, ""); err != nil {
-			logrus.Fatalf("Failed to remove all devices: %s", err.Error())
-		}
-		logrus.Infof("Updated password for user %s and invalidated all logins\n", *username)
-		return
-	}
-	if !available {
-		logrus.Fatalln("Username is already in use.")
-	}
-
-	_, err = accountDB.CreateAccount(context.Background(), *username, pass, "", accType)
+	accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
 	if err != nil {
 		logrus.Fatalln("Failed to create the account:", err.Error())
 	}
 
-	logrus.Infoln("Created account", *username)
+	logrus.Infof("Created account: %s (AccessToken: %s)", *username, accessToken)
+}
+
+type sharedSecretRegistrationRequest struct {
+	User     string `json:"username"`
+	Password string `json:"password"`
+	Nonce    string `json:"nonce"`
+	MacStr   string `json:"mac"`
+	Admin    bool   `json:"admin"`
+}
+
+func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accesToken string, err error) {
+	registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", serverURL)
+	nonceReq, err := http.NewRequest(http.MethodGet, registerURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("unable to create http request: %w", err)
+	}
+	nonceResp, err := cl.Do(nonceReq)
+	if err != nil {
+		return "", fmt.Errorf("unable to get nonce: %w", err)
+	}
+	body, err := io.ReadAll(nonceResp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %w", err)
+	}
+	defer nonceResp.Body.Close() // nolint: errcheck
+
+	nonce := gjson.GetBytes(body, "nonce").Str
+
+	adminStr := "notadmin"
+	if admin {
+		adminStr = "admin"
+	}
+	reg := sharedSecretRegistrationRequest{
+		User:     localpart,
+		Password: password,
+		Nonce:    nonce,
+		Admin:    admin,
+	}
+	macStr, err := getRegisterMac(sharedSecret, nonce, localpart, password, adminStr)
+	if err != nil {
+		return "", err
+	}
+	reg.MacStr = macStr
+
+	js, err := json.Marshal(reg)
+	if err != nil {
+		return "", fmt.Errorf("unable to marshal json: %w", err)
+	}
+	registerReq, err := http.NewRequest(http.MethodPost, registerURL, bytes.NewBuffer(js))
+	if err != nil {
+		return "", fmt.Errorf("unable to create http request: %w", err)
+
+	}
+	regResp, err := cl.Do(registerReq)
+	if err != nil {
+		return "", fmt.Errorf("unable to create account: %w", err)
+	}
+	defer regResp.Body.Close() // nolint: errcheck
+	if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
+		body, _ = io.ReadAll(regResp.Body)
+		return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
+	}
+	r, _ := io.ReadAll(regResp.Body)
+
+	return gjson.GetBytes(r, "access_token").Str, nil
+}
+
+func getRegisterMac(sharedSecret, nonce, localpart, password, adminStr string) (string, error) {
+	joined := strings.Join([]string{nonce, localpart, password, adminStr}, "\x00")
+	mac := hmac.New(sha1.New, []byte(sharedSecret))
+	_, err := mac.Write([]byte(joined))
+	if err != nil {
+		return "", fmt.Errorf("unable to construct mac: %w", err)
+	}
+	regMac := mac.Sum(nil)
+
+	return hex.EncodeToString(regMac), nil
 }
 
 func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string, error) {
diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md
index 61ec2299b..3468398ac 100644
--- a/docs/administration/1_createusers.md
+++ b/docs/administration/1_createusers.md
@@ -14,9 +14,8 @@ User accounts can be created on a Dendrite instance in a number of ways.
 The `create-account` tool is built in the `bin` folder when building Dendrite with
 the `build.sh` script.
 
-It uses the `dendrite.yaml` configuration file to connect to the Dendrite user database
-and create the account entries directly. It can therefore be used even if Dendrite is not
-running yet, as long as the database is up.
+It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires 
+shared secret registration to be enabled as explained below.
 
 An example of using `create-account` to create a **normal account**:
 
@@ -32,6 +31,13 @@ To create a new **admin account**, add the `-admin` flag:
 ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
 ```
 
+By default `create-account` uses `https://localhost:8448` to connect to Dendrite, this can be overwritten using 
+the `-url` flag:
+
+```bash
+./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url http://localhost:8008
+```
+
 An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`:
 
 ```bash
diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index 51f56374b..783fee95a 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -13,19 +13,32 @@ without warning.
 
 More endpoints will be added in the future.
 
-## `/_dendrite/admin/evacuateRoom/{roomID}`
+## GET `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
 in the URL. It may take some time to complete. A JSON body will be returned containing
 the user IDs of all affected users.
 
-## `/_dendrite/admin/evacuateUser/{userID}`
+## GET `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from
 all rooms which they are currently joined. A JSON body will be returned containing
 the room IDs of all affected rooms.
 
-## `/_synapse/admin/v1/register`
+## POST `/_dendrite/admin/resetPassword/{localpart}`
+
+Request body format:
+
+```
+{
+    "password": "new_password_here"
+}
+```
+
+Reset the password of a local user. The `localpart` is the username only, i.e. if
+the full user ID is `@alice:domain.com` then the local part is `alice`.
+
+## GET `/_synapse/admin/v1/register`
 
 Shared secret registration — please see the [user creation page](createusers) for
 guidance on configuring and using this endpoint.
diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index aba50ae4d..e0436c60a 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -83,6 +84,23 @@ func MakeAuthAPI(
 	return MakeExternalAPI(metricsName, h)
 }
 
+// MakeAdminAPI is a wrapper around MakeAuthAPI which enforces that the request can only be
+// completed by a user that is a server administrator.
+func MakeAdminAPI(
+	metricsName string, userAPI userapi.QueryAcccessTokenAPI,
+	f func(*http.Request, *userapi.Device) util.JSONResponse,
+) http.Handler {
+	return MakeAuthAPI(metricsName, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		if device.AccountType != userapi.AccountTypeAdmin {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+			}
+		}
+		return f(req, device)
+	})
+}
+
 // MakeExternalAPI turns a util.JSONRequestHandler function into an http.Handler.
 // This is used for APIs that are called from the internet.
 func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler {
diff --git a/userapi/api/api.go b/userapi/api/api.go
index 388f97cb4..66ee9c7c8 100644
--- a/userapi/api/api.go
+++ b/userapi/api/api.go
@@ -334,8 +334,9 @@ type PerformAccountCreationResponse struct {
 
 // PerformAccountCreationRequest is the request for PerformAccountCreation
 type PerformPasswordUpdateRequest struct {
-	Localpart string // Required: The localpart for this account.
-	Password  string // Required: The new password to set.
+	Localpart     string // Required: The localpart for this account.
+	Password      string // Required: The new password to set.
+	LogoutDevices bool   // Optional: Whether to log out all user devices.
 }
 
 // PerformAccountCreationResponse is the response for PerformAccountCreation
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 78b226d46..6ba469327 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -139,6 +139,11 @@ func (a *UserInternalAPI) PerformPasswordUpdate(ctx context.Context, req *api.Pe
 	if err := a.DB.SetPassword(ctx, req.Localpart, req.Password); err != nil {
 		return err
 	}
+	if req.LogoutDevices {
+		if _, err := a.DB.RemoveAllDevices(context.Background(), req.Localpart, ""); err != nil {
+			return err
+		}
+	}
 	res.PasswordUpdated = true
 	return nil
 }

From b4647fbb7e36014c01d4ec8a65e2644133103683 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:33:31 +0200
Subject: [PATCH 264/391] Show/hide users in user directory (#2637)

* CS API changes

* Query remote profiles

* Add passing tests

* Don't create a new FullyQualifiedProfile

* Handle sql.ErrNoRows

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 clientapi/routing/routing.go       |   4 +-
 clientapi/routing/userdirectory.go | 108 +++++++++++++++++------------
 roomserver/internal/query/query.go |   9 +--
 sytest-whitelist                   |   6 +-
 4 files changed, 75 insertions(+), 52 deletions(-)

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 2063a008f..6904a2b34 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -935,12 +935,12 @@ func Setup(
 			return SearchUserDirectory(
 				req.Context(),
 				device,
-				userAPI,
 				rsAPI,
 				userDirectoryProvider,
-				cfg.Matrix.ServerName,
 				postContent.SearchString,
 				postContent.Limit,
+				federation,
+				cfg.Matrix.ServerName,
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
diff --git a/clientapi/routing/userdirectory.go b/clientapi/routing/userdirectory.go
index f311457a0..d3d1c22e4 100644
--- a/clientapi/routing/userdirectory.go
+++ b/clientapi/routing/userdirectory.go
@@ -18,10 +18,13 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"net/http"
+	"strings"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -34,12 +37,12 @@ type UserDirectoryResponse struct {
 func SearchUserDirectory(
 	ctx context.Context,
 	device *userapi.Device,
-	userAPI userapi.ClientUserAPI,
 	rsAPI api.ClientRoomserverAPI,
 	provider userapi.QuerySearchProfilesAPI,
-	serverName gomatrixserverlib.ServerName,
 	searchString string,
 	limit int,
+	federation *gomatrixserverlib.FederationClient,
+	localServerName gomatrixserverlib.ServerName,
 ) util.JSONResponse {
 	if limit < 10 {
 		limit = 10
@@ -51,59 +54,74 @@ func SearchUserDirectory(
 		Limited: false,
 	}
 
-	// First start searching local users.
-	userReq := &userapi.QuerySearchProfilesRequest{
-		SearchString: searchString,
-		Limit:        limit,
+	// Get users we share a room with
+	knownUsersReq := &api.QueryKnownUsersRequest{
+		UserID: device.UserID,
+		Limit:  limit,
 	}
-	userRes := &userapi.QuerySearchProfilesResponse{}
-	if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
-		return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
+	knownUsersRes := &api.QueryKnownUsersResponse{}
+	if err := rsAPI.QueryKnownUsers(ctx, knownUsersReq, knownUsersRes); err != nil && err != sql.ErrNoRows {
+		return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
 	}
 
-	for _, user := range userRes.Profiles {
+knownUsersLoop:
+	for _, profile := range knownUsersRes.Users {
 		if len(results) == limit {
 			response.Limited = true
 			break
 		}
-
-		var userID string
-		if user.ServerName != "" {
-			userID = fmt.Sprintf("@%s:%s", user.Localpart, user.ServerName)
+		userID := profile.UserID
+		// get the full profile of the local user
+		localpart, serverName, _ := gomatrixserverlib.SplitID('@', userID)
+		if serverName == localServerName {
+			userReq := &userapi.QuerySearchProfilesRequest{
+				SearchString: localpart,
+				Limit:        limit,
+			}
+			userRes := &userapi.QuerySearchProfilesResponse{}
+			if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
+				return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
+			}
+			for _, p := range userRes.Profiles {
+				if strings.Contains(p.DisplayName, searchString) ||
+					strings.Contains(p.Localpart, searchString) {
+					profile.DisplayName = p.DisplayName
+					profile.AvatarURL = p.AvatarURL
+					results[userID] = profile
+					if len(results) == limit {
+						response.Limited = true
+						break knownUsersLoop
+					}
+				}
+			}
 		} else {
-			userID = fmt.Sprintf("@%s:%s", user.Localpart, serverName)
-		}
-		if _, ok := results[userID]; !ok {
-			results[userID] = authtypes.FullyQualifiedProfile{
-				UserID:      userID,
-				DisplayName: user.DisplayName,
-				AvatarURL:   user.AvatarURL,
+			// If the username already contains the search string, don't bother hitting federation.
+			// This will result in missing avatars and displaynames, but saves the federation roundtrip.
+			if strings.Contains(localpart, searchString) {
+				results[userID] = profile
+				if len(results) == limit {
+					response.Limited = true
+					break knownUsersLoop
+				}
+				continue
 			}
-		}
-	}
-
-	// Then, if we have enough room left in the response,
-	// start searching for known users from joined rooms.
-
-	if len(results) <= limit {
-		stateReq := &api.QueryKnownUsersRequest{
-			UserID:       device.UserID,
-			SearchString: searchString,
-			Limit:        limit - len(results),
-		}
-		stateRes := &api.QueryKnownUsersResponse{}
-		if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows {
-			return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
-		}
-
-		for _, user := range stateRes.Users {
-			if len(results) == limit {
-				response.Limited = true
-				break
+			// TODO: We should probably cache/store this
+			fedProfile, fedErr := federation.LookupProfile(ctx, serverName, userID, "")
+			if fedErr != nil {
+				if x, ok := fedErr.(gomatrix.HTTPError); ok {
+					if x.Code == http.StatusNotFound {
+						continue
+					}
+				}
 			}
-
-			if _, ok := results[user.UserID]; !ok {
-				results[user.UserID] = user
+			if strings.Contains(fedProfile.DisplayName, searchString) {
+				profile.DisplayName = fedProfile.DisplayName
+				profile.AvatarURL = fedProfile.AvatarURL
+				results[userID] = profile
+				if len(results) == limit {
+					response.Limited = true
+					break knownUsersLoop
+				}
 			}
 		}
 	}
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index c41e1ea67..f3061c222 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -21,6 +21,10 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/acls"
@@ -30,9 +34,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/roomserver/version"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 type Queryer struct {
@@ -732,7 +733,7 @@ func (r *Queryer) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForU
 
 func (r *Queryer) QueryKnownUsers(ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse) error {
 	users, err := r.DB.GetKnownUsers(ctx, req.UserID, req.SearchString, req.Limit)
-	if err != nil {
+	if err != nil && err != sql.ErrNoRows {
 		return err
 	}
 	for _, user := range users {
diff --git a/sytest-whitelist b/sytest-whitelist
index 88dfe920b..dcffeaffb 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -739,4 +739,8 @@ Only see history_visibility changes on boundaries
 Current state appears in timeline in private history
 Current state appears in timeline in private history with many messages before
 Local users can peek into world_readable rooms by room ID
-Newly joined room includes presence in incremental sync
\ No newline at end of file
+Newly joined room includes presence in incremental sync
+User in private room doesn't appear in user directory
+User joining then leaving public room appears and dissappears from directory
+User in remote room doesn't appear in user directory after server left room
+User in shared private room does appear in user directory until leave
\ No newline at end of file

From f1e9108db8768b37999ee35b2550e795e53edaef Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 12:53:29 +0100
Subject: [PATCH 265/391] Update `create-account` help text

---
 cmd/create-account/main.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 44d5691c2..fbd14b8d7 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -52,8 +52,6 @@ Example:
 	# read password from stdin
 	%s --config dendrite.yaml -username alice -passwordstdin < my.pass
 	cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin
-	# reset password for a user, can be used with a combination above to read the password
-	%s --config dendrite.yaml -reset-password -username alice -password foobarbaz
 
 Arguments:
 
@@ -65,7 +63,7 @@ var (
 	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
 	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
 	isAdmin            = flag.Bool("admin", false, "Create an admin account")
-	resetPassword      = flag.Bool("reset-password", false, "Resets the password for the given username")
+	resetPassword      = flag.Bool("reset-password", false, "Deprecated")
 	serverURL          = flag.String("url", "https://localhost:8448", "The URL to connect to.")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
@@ -78,7 +76,7 @@ var cl = http.Client{
 func main() {
 	name := os.Args[0]
 	flag.Usage = func() {
-		_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name, name)
+		_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name)
 		flag.PrintDefaults()
 	}
 	cfg := setup.ParseFlags(true)

From 20bf00b743c20a8ea57a46737ab366580d219d9e Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:55:45 +0200
Subject: [PATCH 266/391] Update README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 10e2b1b86..638161bf2 100644
--- a/README.md
+++ b/README.md
@@ -89,12 +89,12 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of August 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well, although there are still some missing features (like Search).
 
 We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
-than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API).
+than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API).
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)

From 9fa30f5d3cadcfd15394c61b8058e1e6cda49afc Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:29:09 +0100
Subject: [PATCH 267/391] Update NATS Server and nats.go libraries

---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 79ef5c3e8..108165ba2 100644
--- a/go.mod
+++ b/go.mod
@@ -25,8 +25,8 @@ require (
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
-	github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee
-	github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b
+	github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c
+	github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
diff --git a/go.sum b/go.sum
index 9d5c50d2c..e925d6415 100644
--- a/go.sum
+++ b/go.sum
@@ -385,10 +385,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
 github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
-github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee h1:vAtoZ+LW6eIUjkCWWwO1DZ6o16UGrVOG+ot/AkwejO8=
-github.com/nats-io/nats-server/v2 v2.8.5-0.20220731184415-903a06a5b4ee/go.mod h1:3Yg3ApyQxPlAs1KKHKV5pobV5VtZk+TtOiUJx/iqkkg=
-github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b h1:CE9wSYLvwq8aC/0+6zH8lhhtZYvJ9p8PzwvZeYgdBc0=
-github.com/nats-io/nats.go v1.16.1-0.20220731182438-87bbea85922b/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c h1:U5qngWGZ7E/nQxz0544IpIEdKFUUaOJxQN2LHCYLGhg=
+github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c/go.mod h1:+f++B/5jpr71JATt7b5KCX+G7bt43iWx1OYWGkpE/Kk=
+github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995 h1:CUcSQR8jwa9//qNgN/t3tW53DObnTPQ/G/K+qnS7yRc=
+github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
 github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=

From 6d0d7a0bc34563e272dbc045c13cfadc826946d1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 13:34:05 +0100
Subject: [PATCH 268/391] Update example in readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 638161bf2..34a08be94 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
 
 # Create an user account (add -admin for an admin user).
 # Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
-$ ./bin/create-account --config dendrite.yaml -username alice
+$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice
 ```
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.

From e55cd6ea785e927996cd1e0461d5c1826aa89ef5 Mon Sep 17 00:00:00 2001
From: texuf <texuf.eth@gmail.com>
Date: Fri, 12 Aug 2022 06:07:45 -0700
Subject: [PATCH 269/391] /hierarchy - return public and knockable rooms for
 authed users (#2578)

When requesting the room hierarchy with an authenticated user, return public and knockable rooms.

According to the spec, https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/2946-spaces-summary.md

```
Any child room that the user is joined or is potentially joinable is included in the response.
```

This is currently not the case. See discussion here: https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$t2Csj-6y1PVsn8GOnFZfXzeQW13NfqvrFCxB-XI_uhA?via=matrix.org&via=libera.chat&via=element.io and here: https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$EHp1x1DY7tnYZtx_PVEb-sKB9lmJajqHx2uGlhrRh6k?via=matrix.org&via=libera.chat&via=element.io

Test Plan:
create and register clients bob and alice
have bob create a public space
have bob create a public room parented to the space
have alice join the space(room)
have alice sync the space
expect alice to see two rooms in the space hierarchy, the space and the child room

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 setup/mscs/msc2946/msc2946.go | 41 ++++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go
index 4cffa82ad..a92a16a27 100644
--- a/setup/mscs/msc2946/msc2946.go
+++ b/setup/mscs/msc2946/msc2946.go
@@ -479,7 +479,7 @@ func (w *walker) authorised(roomID, parentRoomID string) (authed, isJoinedOrInvi
 	return w.authorisedServer(roomID), false
 }
 
-// authorisedServer returns true iff the server is joined this room or the room is world_readable
+// authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable
 func (w *walker) authorisedServer(roomID string) bool {
 	// Check history visibility / join rules first
 	hisVisTuple := gomatrixserverlib.StateKeyTuple{
@@ -513,8 +513,21 @@ func (w *walker) authorisedServer(roomID string) bool {
 	// in addition to the actual room ID (but always do the actual one first as it's quicker in the common case)
 	allowJoinedToRoomIDs := []string{roomID}
 	joinRuleEv := queryRoomRes.StateEvents[joinRuleTuple]
+
 	if joinRuleEv != nil {
-		allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...)
+		rule, ruleErr := joinRuleEv.JoinRule()
+		if ruleErr != nil {
+			util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule")
+			return false
+		}
+
+		if rule == gomatrixserverlib.Public || rule == gomatrixserverlib.Knock {
+			return true
+		}
+
+		if rule == gomatrixserverlib.Restricted {
+			allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")...)
+		}
 	}
 
 	// check if server is joined to any allowed room
@@ -537,7 +550,8 @@ func (w *walker) authorisedServer(roomID string) bool {
 	return false
 }
 
-// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable.
+// authorisedUser returns true iff the user is invited/joined this room or the room is world_readable
+// or if the room has a public or knock join rule.
 // Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true.
 func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoinedOrInvited bool) {
 	hisVisTuple := gomatrixserverlib.StateKeyTuple{
@@ -579,13 +593,20 @@ func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoi
 	}
 	joinRuleEv := queryRes.StateEvents[joinRuleTuple]
 	if parentRoomID != "" && joinRuleEv != nil {
-		allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")
-		// check parent is in the allowed set
 		var allowed bool
-		for _, a := range allowedRoomIDs {
-			if parentRoomID == a {
-				allowed = true
-				break
+		rule, ruleErr := joinRuleEv.JoinRule()
+		if ruleErr != nil {
+			util.GetLogger(w.ctx).WithError(ruleErr).WithField("parent_room_id", parentRoomID).Warn("failed to get join rule")
+		} else if rule == gomatrixserverlib.Public || rule == gomatrixserverlib.Knock {
+			allowed = true
+		} else if rule == gomatrixserverlib.Restricted {
+			allowedRoomIDs := w.restrictedJoinRuleAllowedRooms(joinRuleEv, "m.room_membership")
+			// check parent is in the allowed set
+			for _, a := range allowedRoomIDs {
+				if parentRoomID == a {
+					allowed = true
+					break
+				}
 			}
 		}
 		if allowed {
@@ -615,7 +636,7 @@ func (w *walker) authorisedUser(roomID, parentRoomID string) (authed bool, isJoi
 
 func (w *walker) restrictedJoinRuleAllowedRooms(joinRuleEv *gomatrixserverlib.HeaderedEvent, allowType string) (allows []string) {
 	rule, _ := joinRuleEv.JoinRule()
-	if rule != "restricted" {
+	if rule != gomatrixserverlib.Restricted {
 		return nil
 	}
 	var jrContent gomatrixserverlib.JoinRuleContent

From ef7262a7a21f6106b8130a7ad8a1911cbd46a82f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 12 Aug 2022 15:18:31 +0100
Subject: [PATCH 270/391] Version 0.9.2 (#2638)

---
 CHANGES.md          | 27 ++++++++++++++++++++++++++-
 internal/version.go |  2 +-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 5dd8da362..041816ae3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,30 @@
 # Changelog
 
+## Dendrite 0.9.2 (2022-08-12)
+
+### Features
+
+* Dendrite now supports history visibility on the `/sync`, `/messages` and `/context` endpoints
+  * It should now be possible to view the history of a room in more cases (as opposed to limiting scrollback to the join event or defaulting to the restrictive `"join"` visibility rule as before)
+* The default room version for newly created rooms is now room version 9
+* New admin endpoint `/_dendrite/admin/resetPassword/{userID}` has been added, which replaces the `-reset-password` flag in `create-account`
+* The `create-account` binary now uses shared secret registration over HTTP to create new accounts, which fixes a number of problems with account data and push rules not being configured correctly for new accounts
+* The internal HTTP APIs for polylith deployments have been refactored for correctness and consistency
+* The federation API will now automatically clean up some EDUs that have failed to send within a certain period of time
+* The `/hierarchy` endpoint will now return potentially joinable rooms (contributed by [texuf](https://github.com/texuf))
+* The user directory will now show or hide users correctly
+
+### Fixes
+
+* Send-to-device messages should no longer be incorrectly duplicated in `/sync`
+* The federation sender will no longer create unnecessary destination queues as a result of a logic error
+* A bug where database migrations may not execute properly when upgrading from older versions has been fixed
+* A crash when failing to update user account data has been fixed
+* A race condition when generating notification counts has been fixed
+* A race condition when setting up NATS has been fixed (contributed by [brianathere](https://github.com/brianathere))
+* Stale cache data for membership lazy-loading is now correctly invalidated when doing a complete sync
+* Data races within user-interactive authentication have been fixed (contributed by [tak-hntlabs](https://github.com/tak-hntlabs))
+
 ## Dendrite 0.9.1 (2022-08-03)
 
 ### Fixes
@@ -10,7 +35,7 @@
 * The media endpoint now sets the `Cache-Control` header correctly to prevent web-based clients from hitting media endpoints excessively
 * The sync API will now advance the PDU stream position correctly in all cases (contributed by [sergekh2](https://github.com/sergekh2))
 * The sync API will now delete the correct range of send-to-device messages when advancing the stream position
-* The device list `changed` key in the `/sync` response should now return the correct users 
+* The device list `changed` key in the `/sync` response should now return the correct users
 * A data race when looking up missing state has been fixed
 * The `/send_join` API is now applying stronger validation to the received membership event
 
diff --git a/internal/version.go b/internal/version.go
index 38d0864e7..18db2dcc8 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 1
+	VersionPatch = 2
 	VersionTag   = "" // example: "rc1"
 )
 

From 30dbc5ac448e30f69305c81b7049d6bd30ae76d0 Mon Sep 17 00:00:00 2001
From: timaeos <104638316+timaeos@users.noreply.github.com>
Date: Sat, 13 Aug 2022 13:24:52 -0500
Subject: [PATCH 271/391] Adding FAQ Questions with common questions (#2641)

* Adding FAQ Questions with common questions

* changing order of the FAQ to reflect suggestions
---
 docs/FAQ.md                              | 26 +++++++++++++++++++++++-
 docs/administration/5_troubleshooting.md |  9 +++++++-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index f8255684e..85ad83db5 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -12,7 +12,13 @@ Mostly, although there are still bugs and missing features. If you are a confide
 
 ## Is Dendrite feature-complete?
 
-No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information.
+No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information.
+
+## Why doesn't Dendrite have <blank> yet?
+
+Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
+specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 
+the development efforts through [contributing](https://matrix-org.github.io/dendrite/development/contributing). 
 
 ## Is there a migration path from Synapse to Dendrite?
 
@@ -43,6 +49,20 @@ It should do, although we are aware of some minor issues:
 * **Element Android**: registration does not work, but logging in with an existing account does
 * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
 
+## Does Dendrite support Space Summaries?
+
+Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
+
+```
+mscs:
+  mscs:
+    - msc2946
+```
+
+Similarly, [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported.
+
+Please note that MSCs should be considered experimental and can result in significant usability issues when enabled. If you'd like more details on how MSCs are ratified or the current status of MSCs, please see the [Matrix specification documentation](https://spec.matrix.org/proposals/) on the subject.
+
 ## Does Dendrite support push notifications?
 
 Yes, we have experimental support for push notifications. Configure them in the usual way in your Matrix client.
@@ -86,6 +106,10 @@ would be a huge help too, as that will help us to understand where the memory us
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!
 
+## VOIP and Video Calls don't appear to work on Dendrite
+
+There is likely an issue with your STUN/TURN configuration on the server. If you believe your configuration to be correct, please see the [troubleshooting](administration/5_troubleshooting.md) for troubleshooting recommendations.
+
 ## What is being reported when enabling phone-home statistics?
 
 Phone-home statistics contain your server's domain name, some configuration information about
diff --git a/docs/administration/5_troubleshooting.md b/docs/administration/5_troubleshooting.md
index 14df2e3fb..8ba510ef6 100644
--- a/docs/administration/5_troubleshooting.md
+++ b/docs/administration/5_troubleshooting.md
@@ -77,5 +77,12 @@ If there aren't, you will see a log lines like this:
 level=warning msg="IMPORTANT: Process file descriptor limit is currently 65535, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
 ```
 
-Follow the [Optimisation](../installation/10_optimisation.md) instructions to correct the
+Follow the [Optimisation](../installation/11_optimisation.md) instructions to correct the
 available number of file descriptors.
+
+## 6. STUN/TURN Server tester
+
+If you are experiencing problems with VoIP or video calls, you should check that Dendrite
+is able to successfully connect your TURN server using 
+[Matrix VoIP Tester](https://test.voip.librepush.net/). This can highlight any issues
+that the server may encounter so that you can begin the troubleshooting process.

From da7d209b10c01ea3762bf5e524dcc17549de934d Mon Sep 17 00:00:00 2001
From: timaeos <104638316+timaeos@users.noreply.github.com>
Date: Sat, 13 Aug 2022 13:25:39 -0500
Subject: [PATCH 272/391] Admin endpoint documentation (#2640)

* Adding documentation for administration end points that aren't currently documented

* Additional information on using the administration API

* Fix inaccurate information on server notice documentation

* Fix typo in the curl command

* Add details about using the whois endpoint

* fix formatting in documentation for admin whois endpoint

* Fix more documentation formatting

* Additional formatting updates to match other formats

* Add a link to the server notice information in the Matrix Spec

* Fix broken link on server notices

* Adding access token information and admin info

* adding a warning regarding access_token

Co-authored-by: Timothy Arnold <Yourmove@gmail.com>
---
 docs/administration/4_adminapi.md | 46 +++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index 783fee95a..a34bfde1f 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -13,6 +13,25 @@ without warning.
 
 More endpoints will be added in the future.
 
+Endpoints may be used directly through curl:
+
+```
+curl --header "Authorization: Bearer <access_token>" -X <POST|GET|PUT> <Endpoint URI> -d '<Request Body Contents>'
+```
+
+An `access_token` can be obtained through most Element-based matrix clients by going to `Settings` -> `Help & About` -> `Advanced` -> `Access Token`.
+Be aware that an `access_token` allows a client to perform actions as an user and should be kept **secret**.
+
+The user must be an administrator in the `account_accounts` table in order to use these endpoints.
+
+Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `account_accounts`
+
+```
+UPDATE account_accounts SET account_type = 3 WHERE localpart = '$localpart';
+```
+
+Where `$localpart` is the username only (e.g. `alice`).
+
 ## GET `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
@@ -38,7 +57,34 @@ Request body format:
 Reset the password of a local user. The `localpart` is the username only, i.e. if
 the full user ID is `@alice:domain.com` then the local part is `alice`.
 
+## POST `/_synapse/admin/v1/send_server_notice`
+
+Request body format:
+```
+{
+    "user_id": "@target_user:server_name",
+    "content": {
+       "msgtype": "m.text",
+       "body": "This is my message"
+    }
+}
+```
+
+Send a server notice to a specific user. See the [Matrix Spec](https://spec.matrix.org/v1.3/client-server-api/#server-notices) for additional details on server notice behaviour.
+If successfully sent, the API will return the following response:
+
+```
+{
+     "event_id": "<event_id>"
+}
+```
+
 ## GET `/_synapse/admin/v1/register`
 
 Shared secret registration — please see the [user creation page](createusers) for
 guidance on configuring and using this endpoint.
+
+## GET `/_matrix/client/v3/admin/whois/{userId}`
+
+From the [Matrix Spec](https://spec.matrix.org/v1.3/client-server-api/#get_matrixclientv3adminwhoisuserid). 
+Gets information about a particular user. `userId` is the full user ID (e.g. `@alice:domain.com`)

From 65b7db633c07311fea109ca045fcdb20c87d22a3 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Sat, 13 Aug 2022 20:31:43 +0200
Subject: [PATCH 273/391] Fix FAQ question

---
 docs/FAQ.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index 85ad83db5..ca72b151d 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -14,7 +14,7 @@ Mostly, although there are still bugs and missing features. If you are a confide
 
 No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information.
 
-## Why doesn't Dendrite have <blank> yet?
+## Why doesn't Dendrite have "x" yet?
 
 Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
 specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 

From bcdbd5c00a757971f753622b5a9f78347d5160d4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 15 Aug 2022 10:56:54 +0100
Subject: [PATCH 274/391] Version 0.9.3

---
 CHANGES.md          | 10 ++++++++++
 go.mod              |  2 +-
 go.sum              |  4 ++--
 internal/version.go |  2 +-
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 041816ae3..36a1a6311 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,15 @@
 # Changelog
 
+## Dendrite 0.9.3 (2022-08-15)
+
+### Important
+
+* This is a **security release** to fix a vulnerability within event auth, affecting all versions of Dendrite before 0.9.3. Upgrading to this version is highly recommended. For more information, [see here](https://github.com/matrix-org/gomatrixserverlib/security/advisories/GHSA-grvv-h2f9-7v9c).
+
+### Fixes
+
+* Dendrite will now correctly parse the `"events_default"` power level value for event auth.
+
 ## Dendrite 0.9.2 (2022-08-12)
 
 ### Features
diff --git a/go.mod b/go.mod
index 108165ba2..fe4604a03 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index e925d6415..7ac9fc6e2 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839 h1:QEFxKWH8PlEt3ZQKl31yJNAm8lvpNUwT51IMNTl9v1k=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220801083850-5ff38e2c2839/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c h1:GhKmb8s9iXA9qsFD1SbiRo6Ee7cnbfcgJQ/iy43wczM=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/internal/version.go b/internal/version.go
index 18db2dcc8..561e6c06f 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 2
+	VersionPatch = 3
 	VersionTag   = "" // example: "rc1"
 )
 

From 0642ffc0f6c6d21af0e4a17acd84200e35bf279a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 16 Aug 2022 10:29:36 +0200
Subject: [PATCH 275/391] Only return non-retired invites (#2643)

* Only return non-retired invites

* Revert "Only return non-retired invites"

This reverts commit 1150aa7f385b7d7cf5378297f3e17566d5aabcc6.

* Check if we're doing an initial sync in the stream
---
 syncapi/streams/stream_invite.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index ddac9be2c..925da32f2 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -7,8 +7,9 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 type InviteStreamProvider struct {
@@ -62,6 +63,11 @@ func (p *InviteStreamProvider) IncrementalSync(
 		req.Response.Rooms.Invite[roomID] = *ir
 	}
 
+	// When doing an initial sync, we don't want to add retired invites, as this
+	// can add rooms we were invited to, but already left.
+	if from == 0 {
+		return to
+	}
 	for roomID := range retiredInvites {
 		if _, ok := req.Response.Rooms.Join[roomID]; !ok {
 			lr := types.NewLeaveResponse()

From 5424b88f3061dddbeae53df8b67bd064e8cbc400 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 16 Aug 2022 11:55:06 +0100
Subject: [PATCH 276/391] Use `is_direct` flag from `/createRoom`, update
 stripped state (#2644)

* Use `is_direct` flag from `/createRoom`, update stripped state

* Add comment
---
 clientapi/routing/createroom.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go
index 874908639..3e837c864 100644
--- a/clientapi/routing/createroom.go
+++ b/clientapi/routing/createroom.go
@@ -49,6 +49,7 @@ type createRoomRequest struct {
 	GuestCanJoin              bool                          `json:"guest_can_join"`
 	RoomVersion               gomatrixserverlib.RoomVersion `json:"room_version"`
 	PowerLevelContentOverride json.RawMessage               `json:"power_level_content_override"`
+	IsDirect                  bool                          `json:"is_direct"`
 }
 
 const (
@@ -499,9 +500,17 @@ func createRoom(
 		// Build some stripped state for the invite.
 		var globalStrippedState []gomatrixserverlib.InviteV2StrippedState
 		for _, event := range builtEvents {
+			// Chosen events from the spec:
+			// https://spec.matrix.org/v1.3/client-server-api/#stripped-state
 			switch event.Type() {
+			case gomatrixserverlib.MRoomCreate:
+				fallthrough
 			case gomatrixserverlib.MRoomName:
 				fallthrough
+			case gomatrixserverlib.MRoomAvatar:
+				fallthrough
+			case gomatrixserverlib.MRoomTopic:
+				fallthrough
 			case gomatrixserverlib.MRoomCanonicalAlias:
 				fallthrough
 			case gomatrixserverlib.MRoomEncryption:
@@ -522,7 +531,7 @@ func createRoom(
 			// Build the invite event.
 			inviteEvent, err := buildMembershipEvent(
 				ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite,
-				roomID, true, cfg, evTime, rsAPI, asAPI,
+				roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI,
 			)
 			if err != nil {
 				util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")

From 804653e55118a8155dfbfe5187e941a2f8ce8337 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 16 Aug 2022 13:21:22 +0200
Subject: [PATCH 277/391] Verify a shared secret is set in `create-account`
 (#2645)

---
 cmd/create-account/main.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index fbd14b8d7..bd053f2f7 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -85,6 +85,10 @@ func main() {
 		logrus.Fatalf("The reset-password flag has been replaced by the POST /_dendrite/admin/resetPassword/{localpart} admin API.")
 	}
 
+	if cfg.ClientAPI.RegistrationSharedSecret == "" {
+		logrus.Fatalln("Shared secret registration is not enabled, enable it by setting a shared secret in the config: 'client_api.registration_shared_secret'")
+	}
+
 	if *username == "" {
 		flag.Usage()
 		os.Exit(1)

From ec16c944eb646c6ef61f0b7783f2f3869b9bc10c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 16 Aug 2022 14:42:00 +0100
Subject: [PATCH 278/391] Lazy-loading fixes (#2646)

* Use existing current room state if we have it

* Don't dedupe before applying the history vis filter

* Revert "Don't dedupe before applying the history vis filter"

This reverts commit d27c4a0874dabb77c2eda6b23eb7c00478bc9e90.

* Revert "Use existing current room state if we have it"

This reverts commit 5819b4a7ce511204c4fb48d3c4741612b136e2ea.

* Tweaks
---
 syncapi/streams/stream_pdu.go | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 136cbea5a..0e9dda577 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -560,14 +560,13 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 			// If this is a gapped incremental sync, we still want this membership
 			isGappedIncremental := limited && incremental
 			// We want this users membership event, keep it in the list
-			_, ok := timelineUsers[event.Sender()]
-			wantMembership := ok || isGappedIncremental
-			if wantMembership {
+			stateKey := *event.StateKey()
+			if _, ok := timelineUsers[stateKey]; ok || isGappedIncremental {
 				newStateEvents = append(newStateEvents, event)
 				if !includeRedundant {
-					p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, event.Sender(), event.EventID())
+					p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, stateKey, event.EventID())
 				}
-				delete(timelineUsers, event.Sender())
+				delete(timelineUsers, stateKey)
 			}
 		} else {
 			newStateEvents = append(newStateEvents, event)
@@ -578,17 +577,16 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 		wantUsers = append(wantUsers, userID)
 	}
 	// Query missing membership events
-	memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &gomatrixserverlib.StateFilter{
-		Limit:   100,
-		Senders: &wantUsers,
-		Types:   &[]string{gomatrixserverlib.MRoomMember},
-	})
+	filter := gomatrixserverlib.DefaultStateFilter()
+	filter.Senders = &wantUsers
+	filter.Types = &[]string{gomatrixserverlib.MRoomMember}
+	memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &filter)
 	if err != nil {
 		return stateEvents, err
 	}
 	// cache the membership events
 	for _, membership := range memberships {
-		p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, membership.Sender(), membership.EventID())
+		p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, *membership.StateKey(), membership.EventID())
 	}
 	stateEvents = append(newStateEvents, memberships...)
 	return stateEvents, nil

From ad4ac2c016d2916c51695fd11d3a7b52e38d0ca2 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 16 Aug 2022 14:42:35 +0100
Subject: [PATCH 279/391] Stop spamming the logs with `StateBetween: ignoring
 deleted state` event IDs

---
 syncapi/storage/postgres/output_room_events_table.go | 4 ++--
 syncapi/storage/sqlite3/output_room_events_table.go  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 8f633640e..9c758cac1 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -279,8 +279,8 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 			log.WithFields(log.Fields{
 				"since":   r.From,
 				"current": r.To,
-				"adds":    addIDs,
-				"dels":    delIDs,
+				"adds":    len(addIDs),
+				"dels":    len(delIDs),
 			}).Warn("StateBetween: ignoring deleted state")
 		}
 
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 91fd35b5b..cd9a46d4c 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -234,8 +234,8 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 			log.WithFields(log.Fields{
 				"since":   r.From,
 				"current": r.To,
-				"adds":    addIDsJSON,
-				"dels":    delIDsJSON,
+				"adds":    len(addIDsJSON),
+				"dels":    len(delIDsJSON),
 			}).Warn("StateBetween: ignoring deleted state")
 		}
 

From 8d9c8f11c5c5673d48b8e5308e1f565927bdd341 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 18 Aug 2022 08:56:57 +0200
Subject: [PATCH 280/391] Add a delay after sending events to the roomserver

---
 syncapi/syncapi_test.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index dc073a16e..089bdafaf 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -10,6 +10,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver"
@@ -21,9 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/tidwall/gjson"
 )
 
 type syncRoomserverAPI struct {
@@ -422,6 +423,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
 					t.Fatalf("failed to send events: %v", err)
 				}
+				time.Sleep(100 * time.Millisecond) // TODO: find a better way
 
 				// There is only one event, we expect only to be able to see this, if the room is world_readable
 				w := httptest.NewRecorder()
@@ -454,6 +456,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
 					t.Fatalf("failed to send events: %v", err)
 				}
+				time.Sleep(100 * time.Millisecond) // TODO: find a better way
 
 				// Verify the messages after/before invite are visible or not
 				w = httptest.NewRecorder()

From 59bc0a6f4ed0324da0387118e1761b4551aaf103 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 18 Aug 2022 10:37:47 +0100
Subject: [PATCH 281/391] Reprocess rejected input events (#2647)

* Reprocess outliers that were previously rejected

* Might as well do all events this way

* More useful errors

* Fix queries

* Tweak condition

* Don't wrap errors

* Report more useful error

* Flatten error on `r.Queryer.QueryStateAfterEvents`

* Some more debug logging

* Flatten error in `QueryRestrictedJoinAllowed`

* Revert "Flatten error in `QueryRestrictedJoinAllowed`"

This reverts commit 1238b4184c30e0c31ffb0f364806fa1275aba483.

* Tweak `QueryStateAfterEvents`

* Handle MissingStateError too

* Scope to room

* Clean up

* Fix the error

* Only apply rejection check to outliers
---
 roomserver/internal/input/input_events.go   | 47 +++++++++++----------
 roomserver/internal/query/query.go          | 13 +++---
 roomserver/storage/interface.go             |  2 +
 roomserver/storage/postgres/events_table.go | 13 ++++++
 roomserver/storage/shared/storage.go        |  4 ++
 roomserver/storage/sqlite3/events_table.go  | 13 ++++++
 roomserver/storage/tables/interface.go      |  1 +
 7 files changed, 65 insertions(+), 28 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 81541260c..53ccd5973 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -17,8 +17,8 @@
 package input
 
 import (
-	"bytes"
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -107,28 +107,6 @@ func (r *Inputer) processRoomEvent(
 		})
 	}
 
-	// if we have already got this event then do not process it again, if the input kind is an outlier.
-	// Outliers contain no extra information which may warrant a re-processing.
-	if input.Kind == api.KindOutlier {
-		evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
-		if err2 == nil && len(evs) == 1 {
-			// check hash matches if we're on early room versions where the event ID was a random string
-			idFormat, err2 := headered.RoomVersion.EventIDFormat()
-			if err2 == nil {
-				switch idFormat {
-				case gomatrixserverlib.EventIDFormatV1:
-					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
-						logger.Debugf("Already processed event; ignoring")
-						return nil
-					}
-				default:
-					logger.Debugf("Already processed event; ignoring")
-					return nil
-				}
-			}
-		}
-	}
-
 	// Don't waste time processing the event if the room doesn't exist.
 	// A room entry locally will only be created in response to a create
 	// event.
@@ -141,6 +119,29 @@ func (r *Inputer) processRoomEvent(
 		return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID())
 	}
 
+	// If we already know about this outlier and it hasn't been rejected
+	// then we won't attempt to reprocess it. If it was rejected or has now
+	// arrived as a different kind of event, then we can attempt to reprocess,
+	// in case we have learned something new or need to weave the event into
+	// the DAG now.
+	if input.Kind == api.KindOutlier && roomInfo != nil {
+		wasRejected, werr := r.DB.IsEventRejected(ctx, roomInfo.RoomNID, event.EventID())
+		switch {
+		case werr == sql.ErrNoRows:
+			// We haven't seen this event before so continue.
+		case werr != nil:
+			// Something has gone wrong trying to find out if we rejected
+			// this event already.
+			logger.WithError(werr).Errorf("Failed to check if event %q is already seen", event.EventID())
+			return werr
+		case !wasRejected:
+			// We've seen this event before and it wasn't rejected so we
+			// should ignore it.
+			logger.Debugf("Already processed event %q, ignoring", event.EventID())
+			return nil
+		}
+	}
+
 	var missingAuth, missingPrev bool
 	serverRes := &fedapi.QueryJoinedHostServerNamesInRoomResponse{}
 	if !isCreateEvent {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index f3061c222..f5d8c2d49 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -72,13 +72,10 @@ func (r *Queryer) QueryStateAfterEvents(
 
 	prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs)
 	if err != nil {
-		switch err.(type) {
-		case types.MissingEventError:
-			util.GetLogger(ctx).Errorf("QueryStateAfterEvents: MissingEventError: %s", err)
+		if _, ok := err.(types.MissingEventError); ok {
 			return nil
-		default:
-			return err
 		}
+		return err
 	}
 	response.PrevEventsExist = true
 
@@ -95,6 +92,12 @@ func (r *Queryer) QueryStateAfterEvents(
 		)
 	}
 	if err != nil {
+		if _, ok := err.(types.MissingEventError); ok {
+			return nil
+		}
+		if _, ok := err.(types.MissingStateError); ok {
+			return nil
+		}
 		return err
 	}
 
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index b12025c41..5c068873a 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -94,6 +94,8 @@ type Database interface {
 	// Opens and returns a room updater, which locks the room and opens a transaction.
 	// The GetRoomUpdater must have Commit or Rollback called on it if this doesn't return an error.
 	// If this returns an error then no further action is required.
+	// IsEventRejected returns true if the event is known and rejected.
+	IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (rejected bool, err error)
 	GetRoomUpdater(ctx context.Context, roomInfo *types.RoomInfo) (*shared.RoomUpdater, error)
 	// Look up event references for the latest events in the room and the current state snapshot.
 	// Returns the latest events, the current state and the maximum depth of the latest events plus 1.
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index a4d05756d..c7748d2be 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -136,6 +136,9 @@ const selectMaxEventDepthSQL = "" +
 const selectRoomNIDsForEventNIDsSQL = "" +
 	"SELECT event_nid, room_nid FROM roomserver_events WHERE event_nid = ANY($1)"
 
+const selectEventRejectedSQL = "" +
+	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
+
 type eventStatements struct {
 	insertEventStmt                        *sql.Stmt
 	selectEventStmt                        *sql.Stmt
@@ -153,6 +156,7 @@ type eventStatements struct {
 	bulkSelectUnsentEventNIDStmt           *sql.Stmt
 	selectMaxEventDepthStmt                *sql.Stmt
 	selectRoomNIDsForEventNIDsStmt         *sql.Stmt
+	selectEventRejectedStmt                *sql.Stmt
 }
 
 func CreateEventsTable(db *sql.DB) error {
@@ -180,6 +184,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		{&s.bulkSelectUnsentEventNIDStmt, bulkSelectUnsentEventNIDSQL},
 		{&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL},
 		{&s.selectRoomNIDsForEventNIDsStmt, selectRoomNIDsForEventNIDsSQL},
+		{&s.selectEventRejectedStmt, selectEventRejectedSQL},
 	}.Prepare(db)
 }
 
@@ -540,3 +545,11 @@ func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array {
 	}
 	return nids
 }
+
+func (s *eventStatements) SelectEventRejected(
+	ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string,
+) (rejected bool, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectEventRejectedStmt)
+	err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected)
+	return
+}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index cbf9c8b20..4f92adf1f 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -567,6 +567,10 @@ func (d *Database) GetRoomUpdater(
 	return updater, err
 }
 
+func (d *Database) IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) {
+	return d.EventsTable.SelectEventRejected(ctx, nil, roomNID, eventID)
+}
+
 func (d *Database) StoreEvent(
 	ctx context.Context, event *gomatrixserverlib.Event,
 	authEventNIDs []types.EventNID, isRejected bool,
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index 1dda34c36..174e3a9a7 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -109,6 +109,9 @@ const selectMaxEventDepthSQL = "" +
 const selectRoomNIDsForEventNIDsSQL = "" +
 	"SELECT event_nid, room_nid FROM roomserver_events WHERE event_nid IN ($1)"
 
+const selectEventRejectedSQL = "" +
+	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
+
 type eventStatements struct {
 	db                                     *sql.DB
 	insertEventStmt                        *sql.Stmt
@@ -122,6 +125,7 @@ type eventStatements struct {
 	bulkSelectStateAtEventAndReferenceStmt *sql.Stmt
 	bulkSelectEventReferenceStmt           *sql.Stmt
 	bulkSelectEventIDStmt                  *sql.Stmt
+	selectEventRejectedStmt                *sql.Stmt
 	//bulkSelectEventNIDStmt               *sql.Stmt
 	//bulkSelectUnsentEventNIDStmt         *sql.Stmt
 	//selectRoomNIDsForEventNIDsStmt       *sql.Stmt
@@ -152,6 +156,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		//{&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL},
 		//{&s.bulkSelectUnsentEventNIDStmt, bulkSelectUnsentEventNIDSQL},
 		//{&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL},
+		{&s.selectEventRejectedStmt, selectEventRejectedSQL},
 	}.Prepare(db)
 }
 
@@ -614,3 +619,11 @@ func eventNIDsAsArray(eventNIDs []types.EventNID) string {
 	b, _ := json.Marshal(eventNIDs)
 	return string(b)
 }
+
+func (s *eventStatements) SelectEventRejected(
+	ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string,
+) (rejected bool, err error) {
+	stmt := sqlutil.TxStmt(txn, s.selectEventRejectedStmt)
+	err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected)
+	return
+}
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 0bc389b80..ed67c43d8 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -66,6 +66,7 @@ type Events interface {
 	BulkSelectUnsentEventNID(ctx context.Context, txn *sql.Tx, eventIDs []string) (map[string]types.EventNID, error)
 	SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error)
 	SelectRoomNIDsForEventNIDs(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (roomNIDs map[types.EventNID]types.RoomNID, err error)
+	SelectEventRejected(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string) (rejected bool, err error)
 }
 
 type Rooms interface {

From 7484689ad124c5759c8a10bebaabe9265602fd7d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 18 Aug 2022 12:14:42 +0200
Subject: [PATCH 282/391] Actually store EDUs once we retrieved from the
 database (#2651)

We now actually cache the EDUs once we got them from the database and ensures we only evict them if we successfully deleted them.
---
 federationapi/storage/shared/storage_edus.go | 21 ++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/federationapi/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go
index b62e5d9c5..ce9632ed3 100644
--- a/federationapi/storage/shared/storage_edus.go
+++ b/federationapi/storage/shared/storage_edus.go
@@ -110,6 +110,7 @@ func (d *Database) GetPendingEDUs(
 				return fmt.Errorf("json.Unmarshal: %w", err)
 			}
 			edus[&Receipt{nid}] = &event
+			d.Cache.StoreFederationQueuedEDU(nid, &event)
 		}
 
 		return nil
@@ -177,20 +178,18 @@ func (d *Database) GetPendingEDUServerNames(
 	return d.FederationQueueEDUs.SelectQueueEDUServerNames(ctx, nil)
 }
 
-// DeleteExpiredEDUs deletes expired EDUs
+// DeleteExpiredEDUs deletes expired EDUs and evicts them from the cache.
 func (d *Database) DeleteExpiredEDUs(ctx context.Context) error {
-	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+	var jsonNIDs []int64
+	err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) (err error) {
 		expiredBefore := gomatrixserverlib.AsTimestamp(time.Now())
-		jsonNIDs, err := d.FederationQueueEDUs.SelectExpiredEDUs(ctx, txn, expiredBefore)
+		jsonNIDs, err = d.FederationQueueEDUs.SelectExpiredEDUs(ctx, txn, expiredBefore)
 		if err != nil {
 			return err
 		}
 		if len(jsonNIDs) == 0 {
 			return nil
 		}
-		for i := range jsonNIDs {
-			d.Cache.EvictFederationQueuedEDU(jsonNIDs[i])
-		}
 
 		if err = d.FederationQueueJSON.DeleteQueueJSON(ctx, txn, jsonNIDs); err != nil {
 			return err
@@ -198,4 +197,14 @@ func (d *Database) DeleteExpiredEDUs(ctx context.Context) error {
 
 		return d.FederationQueueEDUs.DeleteExpiredEDUs(ctx, txn, expiredBefore)
 	})
+
+	if err != nil {
+		return err
+	}
+
+	for i := range jsonNIDs {
+		d.Cache.EvictFederationQueuedEDU(jsonNIDs[i])
+	}
+
+	return nil
 }

From 606cb6750639b1e680cbe1dd1b8026ac1536d642 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 18 Aug 2022 13:50:58 +0100
Subject: [PATCH 283/391] Enable `workflow_dispatch` in GHA

---
 .github/workflows/dendrite.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 3044b0b9d..6cd7a60e2 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -7,6 +7,7 @@ on:
   pull_request:
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

From 6b48ce0d757279965a6deb77a1ca8d72c15d4bff Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 18 Aug 2022 17:06:13 +0100
Subject: [PATCH 284/391] State handling tweaks (#2652)

This tweaks how rejected events are handled in room state and also to not apply checks we can't complete to outliers.
---
 roomserver/internal/helpers/auth.go           |  4 +-
 roomserver/internal/input/input_events.go     |  6 +-
 .../internal/perform/perform_backfill.go      |  4 +-
 roomserver/storage/interface.go               |  2 +-
 roomserver/storage/postgres/events_table.go   | 64 ++++++++++++-------
 roomserver/storage/shared/storage.go          |  4 +-
 roomserver/storage/sqlite3/events_table.go    | 57 +++++++++++------
 .../storage/tables/events_table_test.go       |  2 +-
 roomserver/storage/tables/interface.go        |  2 +-
 9 files changed, 92 insertions(+), 53 deletions(-)

diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go
index 648c50cf6..935a045df 100644
--- a/roomserver/internal/helpers/auth.go
+++ b/roomserver/internal/helpers/auth.go
@@ -39,7 +39,7 @@ func CheckForSoftFail(
 	var authStateEntries []types.StateEntry
 	var err error
 	if rewritesState {
-		authStateEntries, err = db.StateEntriesForEventIDs(ctx, stateEventIDs)
+		authStateEntries, err = db.StateEntriesForEventIDs(ctx, stateEventIDs, true)
 		if err != nil {
 			return true, fmt.Errorf("StateEntriesForEventIDs failed: %w", err)
 		}
@@ -97,7 +97,7 @@ func CheckAuthEvents(
 	authEventIDs []string,
 ) ([]types.EventNID, error) {
 	// Grab the numeric IDs for the supplied auth state events from the database.
-	authStateEntries, err := db.StateEntriesForEventIDs(ctx, authEventIDs)
+	authStateEntries, err := db.StateEntriesForEventIDs(ctx, authEventIDs, true)
 	if err != nil {
 		return nil, fmt.Errorf("db.StateEntriesForEventIDs: %w", err)
 	}
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 53ccd5973..0ece9d145 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -301,7 +301,7 @@ func (r *Inputer) processRoomEvent(
 	// bother doing this if the event was already rejected as it just ends up
 	// burning CPU time.
 	historyVisibility := gomatrixserverlib.HistoryVisibilityShared // Default to shared.
-	if rejectionErr == nil && !isRejected && !softfail {
+	if input.Kind != api.KindOutlier && rejectionErr == nil && !isRejected && !softfail {
 		var err error
 		historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev)
 		if err != nil {
@@ -356,6 +356,8 @@ func (r *Inputer) processRoomEvent(
 	// We stop here if the event is rejected: We've stored it but won't update forward extremities or notify anyone about it.
 	if isRejected || softfail {
 		logger.WithError(rejectionErr).WithFields(logrus.Fields{
+			"room_id":      event.RoomID(),
+			"event_id":     event.EventID(),
 			"soft_fail":    softfail,
 			"missing_prev": missingPrev,
 		}).Warn("Stored rejected event")
@@ -661,7 +663,7 @@ func (r *Inputer) calculateAndSetState(
 		// We've been told what the state at the event is so we don't need to calculate it.
 		// Check that those state events are in the database and store the state.
 		var entries []types.StateEntry
-		if entries, err = r.DB.StateEntriesForEventIDs(ctx, input.StateEventIDs); err != nil {
+		if entries, err = r.DB.StateEntriesForEventIDs(ctx, input.StateEventIDs, true); err != nil {
 			return fmt.Errorf("updater.StateEntriesForEventIDs: %w", err)
 		}
 		entries = types.DeduplicateStateEntries(entries)
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 298ba04f6..aecff8b88 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -140,11 +140,11 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform
 			continue
 		}
 		var entries []types.StateEntry
-		if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs); err != nil {
+		if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs, true); err != nil {
 			// attempt to fetch the missing events
 			r.fetchAndStoreMissingEvents(ctx, info.RoomVersion, requester, stateIDs)
 			// try again
-			entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs)
+			entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs, true)
 			if err != nil {
 				logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to get state entries for event")
 				return err
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 5c068873a..43e8da7bb 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -79,7 +79,7 @@ type Database interface {
 	// Look up the state entries for a list of string event IDs
 	// Returns an error if the there is an error talking to the database
 	// Returns a types.MissingEventError if the event IDs aren't in the database.
-	StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error)
+	StateEntriesForEventIDs(ctx context.Context, eventIDs []string, excludeRejected bool) ([]types.StateEntry, error)
 	// Look up the string event state keys for a list of numeric event state keys
 	// Returns an error if there was a problem talking to the database.
 	EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error)
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index c7748d2be..a310c3963 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -88,6 +88,14 @@ const bulkSelectStateEventByIDSQL = "" +
 	" WHERE event_id = ANY($1)" +
 	" ORDER BY event_type_nid, event_state_key_nid ASC"
 
+// Bulk lookup of events by string ID that aren't excluded.
+// Sort by the numeric IDs for event type and state key.
+// This means we can use binary search to lookup entries by type and state key.
+const bulkSelectStateEventByIDExcludingRejectedSQL = "" +
+	"SELECT event_type_nid, event_state_key_nid, event_nid FROM roomserver_events" +
+	" WHERE event_id = ANY($1) AND is_rejected = FALSE" +
+	" ORDER BY event_type_nid, event_state_key_nid ASC"
+
 // Bulk look up of events by event NID, optionally filtering by the event type
 // or event state key NIDs if provided. (The CARDINALITY check will return true
 // if the provided arrays are empty, ergo no filtering).
@@ -140,23 +148,24 @@ const selectEventRejectedSQL = "" +
 	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
 
 type eventStatements struct {
-	insertEventStmt                        *sql.Stmt
-	selectEventStmt                        *sql.Stmt
-	bulkSelectStateEventByIDStmt           *sql.Stmt
-	bulkSelectStateEventByNIDStmt          *sql.Stmt
-	bulkSelectStateAtEventByIDStmt         *sql.Stmt
-	updateEventStateStmt                   *sql.Stmt
-	selectEventSentToOutputStmt            *sql.Stmt
-	updateEventSentToOutputStmt            *sql.Stmt
-	selectEventIDStmt                      *sql.Stmt
-	bulkSelectStateAtEventAndReferenceStmt *sql.Stmt
-	bulkSelectEventReferenceStmt           *sql.Stmt
-	bulkSelectEventIDStmt                  *sql.Stmt
-	bulkSelectEventNIDStmt                 *sql.Stmt
-	bulkSelectUnsentEventNIDStmt           *sql.Stmt
-	selectMaxEventDepthStmt                *sql.Stmt
-	selectRoomNIDsForEventNIDsStmt         *sql.Stmt
-	selectEventRejectedStmt                *sql.Stmt
+	insertEventStmt                               *sql.Stmt
+	selectEventStmt                               *sql.Stmt
+	bulkSelectStateEventByIDStmt                  *sql.Stmt
+	bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt
+	bulkSelectStateEventByNIDStmt                 *sql.Stmt
+	bulkSelectStateAtEventByIDStmt                *sql.Stmt
+	updateEventStateStmt                          *sql.Stmt
+	selectEventSentToOutputStmt                   *sql.Stmt
+	updateEventSentToOutputStmt                   *sql.Stmt
+	selectEventIDStmt                             *sql.Stmt
+	bulkSelectStateAtEventAndReferenceStmt        *sql.Stmt
+	bulkSelectEventReferenceStmt                  *sql.Stmt
+	bulkSelectEventIDStmt                         *sql.Stmt
+	bulkSelectEventNIDStmt                        *sql.Stmt
+	bulkSelectUnsentEventNIDStmt                  *sql.Stmt
+	selectMaxEventDepthStmt                       *sql.Stmt
+	selectRoomNIDsForEventNIDsStmt                *sql.Stmt
+	selectEventRejectedStmt                       *sql.Stmt
 }
 
 func CreateEventsTable(db *sql.DB) error {
@@ -171,6 +180,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		{&s.insertEventStmt, insertEventSQL},
 		{&s.selectEventStmt, selectEventSQL},
 		{&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL},
+		{&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL},
 		{&s.bulkSelectStateEventByNIDStmt, bulkSelectStateEventByNIDSQL},
 		{&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL},
 		{&s.updateEventStateStmt, updateEventStateSQL},
@@ -221,11 +231,18 @@ func (s *eventStatements) SelectEvent(
 }
 
 // bulkSelectStateEventByID lookups a list of state events by event ID.
-// If any of the requested events are missing from the database it returns a types.MissingEventError
+// If not excluding rejected events, and any of the requested events are missing from
+// the database it returns a types.MissingEventError. If excluding rejected events,
+// the events will be silently omitted without error.
 func (s *eventStatements) BulkSelectStateEventByID(
-	ctx context.Context, txn *sql.Tx, eventIDs []string,
+	ctx context.Context, txn *sql.Tx, eventIDs []string, excludeRejected bool,
 ) ([]types.StateEntry, error) {
-	stmt := sqlutil.TxStmt(txn, s.bulkSelectStateEventByIDStmt)
+	var stmt *sql.Stmt
+	if excludeRejected {
+		stmt = sqlutil.TxStmt(txn, s.bulkSelectStateEventByIDExcludingRejectedStmt)
+	} else {
+		stmt = sqlutil.TxStmt(txn, s.bulkSelectStateEventByIDStmt)
+	}
 	rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs))
 	if err != nil {
 		return nil, err
@@ -235,10 +252,10 @@ func (s *eventStatements) BulkSelectStateEventByID(
 	// because of the unique constraint on event IDs.
 	// So we can allocate an array of the correct size now.
 	// We might get fewer results than IDs so we adjust the length of the slice before returning it.
-	results := make([]types.StateEntry, len(eventIDs))
+	results := make([]types.StateEntry, 0, len(eventIDs))
 	i := 0
 	for ; rows.Next(); i++ {
-		result := &results[i]
+		var result types.StateEntry
 		if err = rows.Scan(
 			&result.EventTypeNID,
 			&result.EventStateKeyNID,
@@ -246,11 +263,12 @@ func (s *eventStatements) BulkSelectStateEventByID(
 		); err != nil {
 			return nil, err
 		}
+		results = append(results, result)
 	}
 	if err = rows.Err(); err != nil {
 		return nil, err
 	}
-	if i != len(eventIDs) {
+	if !excludeRejected && i != len(eventIDs) {
 		// If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have.
 		// We don't know which ones were missing because we don't return the string IDs in the query.
 		// However it should be possible debug this by replaying queries or entries from the input kafka logs.
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 4f92adf1f..f35592a76 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -113,9 +113,9 @@ func (d *Database) eventStateKeyNIDs(
 }
 
 func (d *Database) StateEntriesForEventIDs(
-	ctx context.Context, eventIDs []string,
+	ctx context.Context, eventIDs []string, excludeRejected bool,
 ) ([]types.StateEntry, error) {
-	return d.EventsTable.BulkSelectStateEventByID(ctx, nil, eventIDs)
+	return d.EventsTable.BulkSelectStateEventByID(ctx, nil, eventIDs, excludeRejected)
 }
 
 func (d *Database) StateEntriesForTuples(
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index 174e3a9a7..943f256eb 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -65,6 +65,14 @@ const bulkSelectStateEventByIDSQL = "" +
 	" WHERE event_id IN ($1)" +
 	" ORDER BY event_type_nid, event_state_key_nid ASC"
 
+// Bulk lookup of events by string ID that aren't rejected.
+// Sort by the numeric IDs for event type and state key.
+// This means we can use binary search to lookup entries by type and state key.
+const bulkSelectStateEventByIDExcludingRejectedSQL = "" +
+	"SELECT event_type_nid, event_state_key_nid, event_nid FROM roomserver_events" +
+	" WHERE event_id IN ($1) AND is_rejected = 0" +
+	" ORDER BY event_type_nid, event_state_key_nid ASC"
+
 const bulkSelectStateEventByNIDSQL = "" +
 	"SELECT event_type_nid, event_state_key_nid, event_nid FROM roomserver_events" +
 	" WHERE event_nid IN ($1)"
@@ -113,19 +121,20 @@ const selectEventRejectedSQL = "" +
 	"SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2"
 
 type eventStatements struct {
-	db                                     *sql.DB
-	insertEventStmt                        *sql.Stmt
-	selectEventStmt                        *sql.Stmt
-	bulkSelectStateEventByIDStmt           *sql.Stmt
-	bulkSelectStateAtEventByIDStmt         *sql.Stmt
-	updateEventStateStmt                   *sql.Stmt
-	selectEventSentToOutputStmt            *sql.Stmt
-	updateEventSentToOutputStmt            *sql.Stmt
-	selectEventIDStmt                      *sql.Stmt
-	bulkSelectStateAtEventAndReferenceStmt *sql.Stmt
-	bulkSelectEventReferenceStmt           *sql.Stmt
-	bulkSelectEventIDStmt                  *sql.Stmt
-	selectEventRejectedStmt                *sql.Stmt
+	db                                            *sql.DB
+	insertEventStmt                               *sql.Stmt
+	selectEventStmt                               *sql.Stmt
+	bulkSelectStateEventByIDStmt                  *sql.Stmt
+	bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt
+	bulkSelectStateAtEventByIDStmt                *sql.Stmt
+	updateEventStateStmt                          *sql.Stmt
+	selectEventSentToOutputStmt                   *sql.Stmt
+	updateEventSentToOutputStmt                   *sql.Stmt
+	selectEventIDStmt                             *sql.Stmt
+	bulkSelectStateAtEventAndReferenceStmt        *sql.Stmt
+	bulkSelectEventReferenceStmt                  *sql.Stmt
+	bulkSelectEventIDStmt                         *sql.Stmt
+	selectEventRejectedStmt                       *sql.Stmt
 	//bulkSelectEventNIDStmt               *sql.Stmt
 	//bulkSelectUnsentEventNIDStmt         *sql.Stmt
 	//selectRoomNIDsForEventNIDsStmt       *sql.Stmt
@@ -145,6 +154,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
 		{&s.insertEventStmt, insertEventSQL},
 		{&s.selectEventStmt, selectEventSQL},
 		{&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL},
+		{&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL},
 		{&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL},
 		{&s.updateEventStateStmt, updateEventStateSQL},
 		{&s.updateEventSentToOutputStmt, updateEventSentToOutputSQL},
@@ -194,16 +204,24 @@ func (s *eventStatements) SelectEvent(
 }
 
 // bulkSelectStateEventByID lookups a list of state events by event ID.
-// If any of the requested events are missing from the database it returns a types.MissingEventError
+// If not excluding rejected events, and any of the requested events are missing from
+// the database it returns a types.MissingEventError. If excluding rejected events,
+// the events will be silently omitted without error.
 func (s *eventStatements) BulkSelectStateEventByID(
-	ctx context.Context, txn *sql.Tx, eventIDs []string,
+	ctx context.Context, txn *sql.Tx, eventIDs []string, excludeRejected bool,
 ) ([]types.StateEntry, error) {
 	///////////////
+	var sql string
+	if excludeRejected {
+		sql = bulkSelectStateEventByIDExcludingRejectedSQL
+	} else {
+		sql = bulkSelectStateEventByIDSQL
+	}
 	iEventIDs := make([]interface{}, len(eventIDs))
 	for k, v := range eventIDs {
 		iEventIDs[k] = v
 	}
-	selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1)
+	selectOrig := strings.Replace(sql, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1)
 	selectPrep, err := s.db.Prepare(selectOrig)
 	if err != nil {
 		return nil, err
@@ -221,10 +239,10 @@ func (s *eventStatements) BulkSelectStateEventByID(
 	// because of the unique constraint on event IDs.
 	// So we can allocate an array of the correct size now.
 	// We might get fewer results than IDs so we adjust the length of the slice before returning it.
-	results := make([]types.StateEntry, len(eventIDs))
+	results := make([]types.StateEntry, 0, len(eventIDs))
 	i := 0
 	for ; rows.Next(); i++ {
-		result := &results[i]
+		var result types.StateEntry
 		if err = rows.Scan(
 			&result.EventTypeNID,
 			&result.EventStateKeyNID,
@@ -232,8 +250,9 @@ func (s *eventStatements) BulkSelectStateEventByID(
 		); err != nil {
 			return nil, err
 		}
+		results = append(results, result)
 	}
-	if i != len(eventIDs) {
+	if !excludeRejected && i != len(eventIDs) {
 		// If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have.
 		// We don't know which ones were missing because we don't return the string IDs in the query.
 		// However it should be possible debug this by replaying queries or entries from the input kafka logs.
diff --git a/roomserver/storage/tables/events_table_test.go b/roomserver/storage/tables/events_table_test.go
index 74502a31f..107af4784 100644
--- a/roomserver/storage/tables/events_table_test.go
+++ b/roomserver/storage/tables/events_table_test.go
@@ -102,7 +102,7 @@ func Test_EventsTable(t *testing.T) {
 			})
 		}
 
-		stateEvents, err := tab.BulkSelectStateEventByID(ctx, nil, eventIDs)
+		stateEvents, err := tab.BulkSelectStateEventByID(ctx, nil, eventIDs, false)
 		assert.NoError(t, err)
 		assert.Equal(t, len(stateEvents), len(eventIDs))
 		nids := make([]types.EventNID, 0, len(stateEvents))
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index ed67c43d8..68d30f994 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -46,7 +46,7 @@ type Events interface {
 	SelectEvent(ctx context.Context, txn *sql.Tx, eventID string) (types.EventNID, types.StateSnapshotNID, error)
 	// bulkSelectStateEventByID lookups a list of state events by event ID.
 	// If any of the requested events are missing from the database it returns a types.MissingEventError
-	BulkSelectStateEventByID(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StateEntry, error)
+	BulkSelectStateEventByID(ctx context.Context, txn *sql.Tx, eventIDs []string, excludeRejected bool) ([]types.StateEntry, error)
 	BulkSelectStateEventByNID(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, stateKeyTuples []types.StateKeyTuple) ([]types.StateEntry, error)
 	// BulkSelectStateAtEventByID lookups the state at a list of events by event ID.
 	// If any of the requested events are missing from the database it returns a types.MissingEventError.

From a379d3e8141cfceb0ca49435ffb7a596d57619bb Mon Sep 17 00:00:00 2001
From: Winter <winter@winter.cafe>
Date: Fri, 19 Aug 2022 01:28:33 -0400
Subject: [PATCH 285/391] De-race `TestExpireEDUs` (#2654)

In some conditions (fast CPUs), this test would race the clock for EDU expiration when all we want to make sure of is that the expired EDUs are properly deleted. Given this, we set the expiry time to 0 so the specified EDUs are always deleted when DeleteExpiredEDUs is called.
Fixes #2650.

Signed-off-by: Winter <winter@winter.cafe>
---
 federationapi/storage/storage_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/federationapi/storage/storage_test.go b/federationapi/storage/storage_test.go
index 7eba2cbee..3b0268e55 100644
--- a/federationapi/storage/storage_test.go
+++ b/federationapi/storage/storage_test.go
@@ -31,7 +31,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Dat
 
 func TestExpireEDUs(t *testing.T) {
 	var expireEDUTypes = map[string]time.Duration{
-		gomatrixserverlib.MReceipt: time.Millisecond,
+		gomatrixserverlib.MReceipt: 0,
 	}
 
 	ctx := context.Background()

From 5cacca92d2b888d022f9fa346b8068ce13087b00 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 19 Aug 2022 11:03:55 +0200
Subject: [PATCH 286/391] Make SyncAPI unit tests more reliable (#2655)

This should hopefully make some SyncAPI tests more reliable
---
 syncapi/syncapi_test.go | 86 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 76 insertions(+), 10 deletions(-)

diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 089bdafaf..8b33c5e43 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -154,8 +154,12 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) {
 			wantJoinedRooms: []string{room.ID},
 		},
 	}
-	// TODO: find a better way
-	time.Sleep(500 * time.Millisecond)
+
+	syncUntil(t, base, alice.AccessToken, false, func(syncBody string) bool {
+		// wait for the last sent eventID to come down sync
+		path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID())
+		return gjson.Get(syncBody, path).Exists()
+	})
 
 	for _, tc := range testCases {
 		w := httptest.NewRecorder()
@@ -343,6 +347,13 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 
 	// create the users
 	alice := test.NewUser(t)
+	aliceDev := userapi.Device{
+		ID:          "ALICEID",
+		UserID:      alice.ID,
+		AccessToken: "ALICE_BEARER_TOKEN",
+		DisplayName: "ALICE",
+	}
+
 	bob := test.NewUser(t)
 
 	bobDev := userapi.Device{
@@ -409,7 +420,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 		rsAPI := roomserver.NewInternalAPI(base)
 		rsAPI.SetFederationAPI(nil, nil)
 
-		AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{bobDev}}, rsAPI, &syncKeyAPI{})
+		AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, &syncKeyAPI{})
 
 		for _, tc := range testCases {
 			testname := fmt.Sprintf("%s - %s", tc.historyVisibility, userType)
@@ -418,12 +429,18 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 				room := test.NewRoom(t, alice, test.RoomHistoryVisibility(tc.historyVisibility))
 
 				// send the events/messages to NATS to create the rooms
-				beforeJoinEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("Before invite in a %s room", tc.historyVisibility)})
+				beforeJoinBody := fmt.Sprintf("Before invite in a %s room", tc.historyVisibility)
+				beforeJoinEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": beforeJoinBody})
 				eventsToSend := append(room.Events(), beforeJoinEv)
 				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
 					t.Fatalf("failed to send events: %v", err)
 				}
-				time.Sleep(100 * time.Millisecond) // TODO: find a better way
+				syncUntil(t, base, aliceDev.AccessToken, false,
+					func(syncBody string) bool {
+						path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(content.body=="%s")`, room.ID, beforeJoinBody)
+						return gjson.Get(syncBody, path).Exists()
+					},
+				)
 
 				// There is only one event, we expect only to be able to see this, if the room is world_readable
 				w := httptest.NewRecorder()
@@ -449,14 +466,20 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) {
 				inviteEv := room.CreateAndInsert(t, alice, "m.room.member", map[string]interface{}{"membership": "invite"}, test.WithStateKey(bob.ID))
 				afterInviteEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("After invite in a %s room", tc.historyVisibility)})
 				joinEv := room.CreateAndInsert(t, bob, "m.room.member", map[string]interface{}{"membership": "join"}, test.WithStateKey(bob.ID))
-				msgEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("After join in a %s room", tc.historyVisibility)})
+				afterJoinBody := fmt.Sprintf("After join in a %s room", tc.historyVisibility)
+				msgEv := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": afterJoinBody})
 
 				eventsToSend = append([]*gomatrixserverlib.HeaderedEvent{}, inviteEv, afterInviteEv, joinEv, msgEv)
 
 				if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", nil, false); err != nil {
 					t.Fatalf("failed to send events: %v", err)
 				}
-				time.Sleep(100 * time.Millisecond) // TODO: find a better way
+				syncUntil(t, base, aliceDev.AccessToken, false,
+					func(syncBody string) bool {
+						path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(content.body=="%s")`, room.ID, afterJoinBody)
+						return gjson.Get(syncBody, path).Exists()
+					},
+				)
 
 				// Verify the messages after/before invite are visible or not
 				w = httptest.NewRecorder()
@@ -511,8 +534,8 @@ func testSendToDevice(t *testing.T, dbType test.DBType) {
 		AccountType: userapi.AccountTypeUser,
 	}
 
-	base, close := testrig.CreateBaseDendrite(t, dbType)
-	defer close()
+	base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+	defer baseClose()
 
 	jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 	defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
@@ -607,7 +630,14 @@ func testSendToDevice(t *testing.T, dbType test.DBType) {
 				t.Fatalf("unable to send to device message: %v", err)
 			}
 		}
-		time.Sleep((time.Millisecond * 15) * time.Duration(tc.sendMessagesCount)) // wait a bit, so the messages can be processed
+
+		syncUntil(t, base, alice.AccessToken,
+			len(tc.want) == 0,
+			func(body string) bool {
+				return gjson.Get(body, fmt.Sprintf(`to_device.events.#(content.dummy=="message %d")`, msgCounter)).Exists()
+			},
+		)
+
 		// Execute a /sync request, recording the response
 		w := httptest.NewRecorder()
 		base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
@@ -630,6 +660,42 @@ func testSendToDevice(t *testing.T, dbType test.DBType) {
 	}
 }
 
+func syncUntil(t *testing.T,
+	base *base.BaseDendrite, accessToken string,
+	skip bool,
+	checkFunc func(syncBody string) bool,
+) {
+	if checkFunc == nil {
+		t.Fatalf("No checkFunc defined")
+	}
+	if skip {
+		return
+	}
+	// loop on /sync until we receive the last send message or timeout after 5 seconds, since we don't know if the message made it
+	// to the syncAPI when hitting /sync
+	done := make(chan bool)
+	defer close(done)
+	go func() {
+		for {
+			w := httptest.NewRecorder()
+			base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{
+				"access_token": accessToken,
+				"timeout":      "1000",
+			})))
+			if checkFunc(w.Body.String()) {
+				done <- true
+				return
+			}
+		}
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatalf("Timed out waiting for messages")
+	}
+}
+
 func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input ...*gomatrixserverlib.HeaderedEvent) []*nats.Msg {
 	result := make([]*nats.Msg, len(input))
 	for i, ev := range input {

From 365da70a23cec9595a2854ed47f970f03cfde0a9 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 19 Aug 2022 11:04:26 +0200
Subject: [PATCH 287/391] Set historyVisibility for backfilled events over
 federation (#2656)

This should hopefully deflake Backfill works correctly with history visibility set to joined as we were using the default shared visibility, even if the events are set to joined (or something else)
---
 roomserver/api/perform.go                     |  6 +++--
 .../internal/perform/perform_backfill.go      | 24 ++++++++++++-------
 syncapi/routing/messages.go                   | 14 ++++++++---
 3 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index d9ea9dd1c..20931f807 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -5,9 +5,10 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 )
 
 type PerformErrorCode int
@@ -161,7 +162,8 @@ func (r *PerformBackfillRequest) PrevEventIDs() []string {
 // PerformBackfillResponse is a response to PerformBackfill.
 type PerformBackfillResponse struct {
 	// Missing events, arbritrary order.
-	Events []*gomatrixserverlib.HeaderedEvent `json:"events"`
+	Events            []*gomatrixserverlib.HeaderedEvent  `json:"events"`
+	HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"`
 }
 
 type PerformPublishRequest struct {
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index aecff8b88..de76b6412 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -164,6 +164,7 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform
 	// TODO: update backwards extremities, as that should be moved from syncapi to roomserver at some point.
 
 	res.Events = events
+	res.HistoryVisibility = requester.historyVisiblity
 	return nil
 }
 
@@ -248,6 +249,7 @@ type backfillRequester struct {
 	servers                 []gomatrixserverlib.ServerName
 	eventIDToBeforeStateIDs map[string][]string
 	eventIDMap              map[string]*gomatrixserverlib.Event
+	historyVisiblity        gomatrixserverlib.HistoryVisibility
 }
 
 func newBackfillRequester(
@@ -266,6 +268,7 @@ func newBackfillRequester(
 		eventIDMap:              make(map[string]*gomatrixserverlib.Event),
 		bwExtrems:               bwExtrems,
 		preferServer:            preferServer,
+		historyVisiblity:        gomatrixserverlib.HistoryVisibilityShared,
 	}
 }
 
@@ -447,7 +450,8 @@ FindSuccessor:
 	}
 
 	// possibly return all joined servers depending on history visiblity
-	memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries, b.thisServer)
+	memberEventsFromVis, visibility, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries, b.thisServer)
+	b.historyVisiblity = visibility
 	if err != nil {
 		logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules")
 		return nil
@@ -528,7 +532,7 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion,
 // pull all events and then filter by that table.
 func joinEventsFromHistoryVisibility(
 	ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry,
-	thisServer gomatrixserverlib.ServerName) ([]types.Event, error) {
+	thisServer gomatrixserverlib.ServerName) ([]types.Event, gomatrixserverlib.HistoryVisibility, error) {
 
 	var eventNIDs []types.EventNID
 	for _, entry := range stateEntries {
@@ -542,7 +546,9 @@ func joinEventsFromHistoryVisibility(
 	// Get all of the events in this state
 	stateEvents, err := db.Events(ctx, eventNIDs)
 	if err != nil {
-		return nil, err
+		// even though the default should be shared, restricting the visibility to joined
+		// feels more secure here.
+		return nil, gomatrixserverlib.HistoryVisibilityJoined, err
 	}
 	events := make([]*gomatrixserverlib.Event, len(stateEvents))
 	for i := range stateEvents {
@@ -551,20 +557,22 @@ func joinEventsFromHistoryVisibility(
 
 	// Can we see events in the room?
 	canSeeEvents := auth.IsServerAllowed(thisServer, true, events)
+	visibility := gomatrixserverlib.HistoryVisibility(auth.HistoryVisibilityForRoom(events))
 	if !canSeeEvents {
-		logrus.Infof("ServersAtEvent history not visible to us: %s", auth.HistoryVisibilityForRoom(events))
-		return nil, nil
+		logrus.Infof("ServersAtEvent history not visible to us: %s", visibility)
+		return nil, visibility, nil
 	}
 	// get joined members
 	info, err := db.RoomInfo(ctx, roomID)
 	if err != nil {
-		return nil, err
+		return nil, visibility, nil
 	}
 	joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false)
 	if err != nil {
-		return nil, err
+		return nil, visibility, err
 	}
-	return db.Events(ctx, joinEventNIDs)
+	evs, err := db.Events(ctx, joinEventNIDs)
+	return evs, visibility, err
 }
 
 func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) {
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 9db3d8e17..03614302c 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -350,8 +350,10 @@ func (r *messagesReq) retrieveEvents() (
 	startTime := time.Now()
 	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.db, r.rsAPI, events, nil, r.device.UserID, "messages")
 	logrus.WithFields(logrus.Fields{
-		"duration": time.Since(startTime),
-		"room_id":  r.roomID,
+		"duration":      time.Since(startTime),
+		"room_id":       r.roomID,
+		"events_before": len(events),
+		"events_after":  len(filteredEvents),
 	}).Debug("applied history visibility (messages)")
 	return gomatrixserverlib.HeaderedToClientEvents(filteredEvents, gomatrixserverlib.FormatAll), start, end, err
 }
@@ -513,6 +515,9 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 
 	// Store the events in the database, while marking them as unfit to show
 	// up in responses to sync requests.
+	if res.HistoryVisibility == "" {
+		res.HistoryVisibility = gomatrixserverlib.HistoryVisibilityShared
+	}
 	for i := range res.Events {
 		_, err = r.db.WriteEvent(
 			context.Background(),
@@ -521,7 +526,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 			[]string{},
 			[]string{},
 			nil, true,
-			gomatrixserverlib.HistoryVisibilityShared,
+			res.HistoryVisibility,
 		)
 		if err != nil {
 			return nil, err
@@ -534,6 +539,9 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 		// last `limit` events
 		events = events[len(events)-limit:]
 	}
+	for _, ev := range events {
+		ev.Visibility = res.HistoryVisibility
+	}
 
 	return events, nil
 }

From 5513f182ccd978866354eaa45effd293b0745207 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 19 Aug 2022 10:23:09 +0100
Subject: [PATCH 288/391] Enforce device list backoffs (#2653)

This ensures that if the device list updater is already backing off a node, we don't try to call processServer again anyway for server just because the server name arrived in the channel. Otherwise we can keep trying to hit a remote server that is offline or not behaving every second and that spams the logs too.
---
 keyserver/internal/device_list_update.go | 13 ++++++++++---
 sytest-whitelist                         |  2 --
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 80efbec51..304b67b23 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -335,8 +335,9 @@ func (u *DeviceListUpdater) worker(ch chan gomatrixserverlib.ServerName) {
 	retriesMu := &sync.Mutex{}
 	// restarter goroutine which will inject failed servers into ch when it is time
 	go func() {
+		var serversToRetry []gomatrixserverlib.ServerName
 		for {
-			var serversToRetry []gomatrixserverlib.ServerName
+			serversToRetry = serversToRetry[:0] // reuse memory
 			time.Sleep(time.Second)
 			retriesMu.Lock()
 			now := time.Now()
@@ -355,11 +356,17 @@ func (u *DeviceListUpdater) worker(ch chan gomatrixserverlib.ServerName) {
 		}
 	}()
 	for serverName := range ch {
+		retriesMu.Lock()
+		_, exists := retries[serverName]
+		retriesMu.Unlock()
+		if exists {
+			// Don't retry a server that we're already waiting for.
+			continue
+		}
 		waitTime, shouldRetry := u.processServer(serverName)
 		if shouldRetry {
 			retriesMu.Lock()
-			_, exists := retries[serverName]
-			if !exists {
+			if _, exists = retries[serverName]; !exists {
 				retries[serverName] = time.Now().Add(waitTime)
 			}
 			retriesMu.Unlock()
diff --git a/sytest-whitelist b/sytest-whitelist
index dcffeaffb..5c8896b99 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -144,7 +144,6 @@ Server correctly handles incoming m.device_list_update
 If remote user leaves room, changes device and rejoins we see update in sync
 If remote user leaves room, changes device and rejoins we see update in /keys/changes
 If remote user leaves room we no longer receive device updates
-If a device list update goes missing, the server resyncs on the next one
 Server correctly resyncs when client query keys and there is no remote cache
 Server correctly resyncs when server leaves and rejoins a room
 Device list doesn't change if remote server is down
@@ -633,7 +632,6 @@ Test that rejected pushers are removed.
 Trying to add push rule with no scope fails with 400
 Trying to add push rule with invalid scope fails with 400
 Forward extremities remain so even after the next events are populated as outliers
-If a device list update goes missing, the server resyncs on the next one
 uploading self-signing key notifies over federation
 uploading signed devices gets propagated over federation
 Device list doesn't change if remote server is down

From 56b55a28f591cd2e920c5f16bee6f7a97b0898c0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 19 Aug 2022 12:46:14 +0100
Subject: [PATCH 289/391] Version 0.9.4 (#2658)

### Fixes

* A bug in the roomserver around handling rejected outliers has been fixed
* Backfilled events will now use the correct history visibility where possible
* The device list updater backoff has been fixed, which should reduce the number of outbound HTTP requests and `Failed to query device keys for some users` log entries for dead servers
* The `/sync` endpoint will no longer incorrectly return room entries for retired invites which could cause some rooms to show up in the client "Historical" section
* The `/createRoom` endpoint will now correctly populate `is_direct` in invite membership events, which may help clients to classify direct messages correctly
* The `create-account` tool will now log an error if the shared secret is not set in the Dendrite config
* A couple of minor bugs have been fixed in the membership lazy-loading
* Queued EDUs in the federation API are now cached properly
---
 CHANGES.md          | 13 +++++++++++++
 internal/version.go |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 36a1a6311..aaf5836ba 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,18 @@
 # Changelog
 
+## Dendrite 0.9.4 (2022-08-19)
+
+### Fixes
+
+* A bug in the roomserver around handling rejected outliers has been fixed
+* Backfilled events will now use the correct history visibility where possible
+* The device list updater backoff has been fixed, which should reduce the number of outbound HTTP requests and `Failed to query device keys for some users` log entries for dead servers
+* The `/sync` endpoint will no longer incorrectly return room entries for retired invites which could cause some rooms to show up in the client "Historical" section
+* The `/createRoom` endpoint will now correctly populate `is_direct` in invite membership events, which may help clients to classify direct messages correctly
+* The `create-account` tool will now log an error if the shared secret is not set in the Dendrite config
+* A couple of minor bugs have been fixed in the membership lazy-loading
+* Queued EDUs in the federation API are now cached properly
+
 ## Dendrite 0.9.3 (2022-08-15)
 
 ### Important
diff --git a/internal/version.go b/internal/version.go
index 561e6c06f..384f091a0 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 3
+	VersionPatch = 4
 	VersionTag   = "" // example: "rc1"
 )
 

From 9dc57122d991d54ea6750448ba88c8763a569830 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 19 Aug 2022 15:32:24 +0200
Subject: [PATCH 290/391] Fetch more data for newly joined rooms in an
 incremental sync (#2657)

If we've joined a new room in an incremental sync, try fetching more data.
This deflakes the complement server notices test (at least in my tests).
---
 syncapi/internal/keychange.go      | 12 ++++++---
 syncapi/streams/stream_pdu.go      | 39 +++++++++++++++++++++++++++---
 syncapi/streams/stream_presence.go |  8 +++++-
 syncapi/syncapi_test.go            |  1 +
 4 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go
index 23824e366..c5180e338 100644
--- a/syncapi/internal/keychange.go
+++ b/syncapi/internal/keychange.go
@@ -18,14 +18,16 @@ import (
 	"context"
 	"strings"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
+
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	keytypes "github.com/matrix-org/dendrite/keyserver/types"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 // DeviceOTKCounts adds one-time key counts to the /sync response
@@ -277,6 +279,10 @@ func membershipEventPresent(events []gomatrixserverlib.ClientEvent, userID strin
 		// it's enough to know that we have our member event here, don't need to check membership content
 		// as it's implied by being in the respective section of the sync response.
 		if ev.Type == gomatrixserverlib.MRoomMember && ev.StateKey != nil && *ev.StateKey == userID {
+			// ignore e.g. join -> join changes
+			if gjson.GetBytes(ev.Unsigned, "prev_content.membership").Str == gjson.GetBytes(ev.Content, "membership").Str {
+				continue
+			}
 			return true
 		}
 	}
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 0e9dda577..2818aad87 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -178,24 +178,24 @@ func (p *PDUStreamProvider) IncrementalSync(
 
 	var err error
 	var stateDeltas []types.StateDelta
-	var joinedRooms []string
+	var syncJoinedRooms []string
 
 	stateFilter := req.Filter.Room.State
 	eventFilter := req.Filter.Room.Timeline
 
 	if req.WantFullState {
-		if stateDeltas, joinedRooms, err = p.DB.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
+		if stateDeltas, syncJoinedRooms, err = p.DB.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed")
 			return
 		}
 	} else {
-		if stateDeltas, joinedRooms, err = p.DB.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
+		if stateDeltas, syncJoinedRooms, err = p.DB.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltas failed")
 			return
 		}
 	}
 
-	for _, roomID := range joinedRooms {
+	for _, roomID := range syncJoinedRooms {
 		req.Rooms[roomID] = gomatrixserverlib.Join
 	}
 
@@ -222,6 +222,37 @@ func (p *PDUStreamProvider) IncrementalSync(
 		}
 	}
 
+	// If we joined a new room in this sync, make sure we add enough information about it.
+	// This does an "initial sync" for the newly joined rooms
+	newlyJoinedRooms := joinedRooms(req.Response, req.Device.UserID)
+	if len(newlyJoinedRooms) > 0 {
+		// remove already added rooms, as we're doing an "initial sync"
+		for _, x := range newlyJoinedRooms {
+			delete(req.Response.Rooms.Join, x)
+		}
+		r = types.Range{
+			From:      to,
+			To:        0,
+			Backwards: true,
+		}
+		// We only care about the newly joined rooms, so update the stateFilter to reflect that
+		stateFilter.Rooms = &newlyJoinedRooms
+		if stateDeltas, _, err = p.DB.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
+			req.Log.WithError(err).Error("p.DB.GetStateDeltas failed")
+			return newPos
+		}
+		for _, delta := range stateDeltas {
+			// Ignore deltas for rooms we didn't newly join
+			if _, ok := req.Response.Rooms.Join[delta.RoomID]; ok {
+				continue
+			}
+			if _, err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, &stateFilter, req.Response); err != nil {
+				req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
+				return newPos
+			}
+		}
+	}
+
 	return newPos
 }
 
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 877bcf141..15db4d30e 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -19,9 +19,11 @@ import (
 	"encoding/json"
 	"sync"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type PresenceStreamProvider struct {
@@ -175,6 +177,10 @@ func membershipEventPresent(events []gomatrixserverlib.ClientEvent, userID strin
 		// it's enough to know that we have our member event here, don't need to check membership content
 		// as it's implied by being in the respective section of the sync response.
 		if ev.Type == gomatrixserverlib.MRoomMember && ev.StateKey != nil && *ev.StateKey == userID {
+			// ignore e.g. join -> join changes
+			if gjson.GetBytes(ev.Unsigned, "prev_content.membership").Str == gjson.GetBytes(ev.Content, "membership").Str {
+				continue
+			}
 			return true
 		}
 	}
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 8b33c5e43..76d51c86b 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -195,6 +195,7 @@ func TestSyncAPICreateRoomSyncEarly(t *testing.T) {
 }
 
 func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
+	t.SkipNow() // Temporary?
 	user := test.NewUser(t)
 	room := test.NewRoom(t, user)
 	alice := userapi.Device{

From 2668050e535be7527d5dea2e97309027cbc12560 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 22 Aug 2022 10:30:35 +0100
Subject: [PATCH 291/391] Tweak soft-failure handling in roomserver

commit 1929b688e31987c46e0c8a546f0f9cb0a46bf9a3
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Mon Aug 22 10:09:44 2022 +0100

    Still process state-before for soft-failed events

commit e83c0b701d40d78b92072c4643f6bc6f71b72800
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Mon Aug 22 10:06:50 2022 +0100

    Improve logging

commit 29e26124bc27cb83d449de2a4214b253c594aa93
Author: Neil Alexander <neilalexander@users.noreply.github.com>
Date:   Mon Aug 22 09:58:13 2022 +0100

    Don't store soft-failed events as rejected
---
 roomserver/internal/input/input_events.go | 24 +++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 0ece9d145..29af649ad 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -301,7 +301,7 @@ func (r *Inputer) processRoomEvent(
 	// bother doing this if the event was already rejected as it just ends up
 	// burning CPU time.
 	historyVisibility := gomatrixserverlib.HistoryVisibilityShared // Default to shared.
-	if input.Kind != api.KindOutlier && rejectionErr == nil && !isRejected && !softfail {
+	if input.Kind != api.KindOutlier && rejectionErr == nil && !isRejected {
 		var err error
 		historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev)
 		if err != nil {
@@ -313,7 +313,7 @@ func (r *Inputer) processRoomEvent(
 	}
 
 	// Store the event.
-	_, _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected || softfail)
+	_, _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected)
 	if err != nil {
 		return fmt.Errorf("updater.StoreEvent: %w", err)
 	}
@@ -353,14 +353,18 @@ func (r *Inputer) processRoomEvent(
 		}
 	}
 
-	// We stop here if the event is rejected: We've stored it but won't update forward extremities or notify anyone about it.
-	if isRejected || softfail {
-		logger.WithError(rejectionErr).WithFields(logrus.Fields{
-			"room_id":      event.RoomID(),
-			"event_id":     event.EventID(),
-			"soft_fail":    softfail,
-			"missing_prev": missingPrev,
-		}).Warn("Stored rejected event")
+	// We stop here if the event is rejected: We've stored it but won't update
+	// forward extremities or notify downstream components about it.
+	switch {
+	case isRejected:
+		logger.WithError(rejectionErr).Warn("Stored rejected event")
+		if rejectionErr != nil {
+			return types.RejectedError(rejectionErr.Error())
+		}
+		return nil
+
+	case softfail:
+		logger.WithError(rejectionErr).Warn("Stored soft-failed event")
 		if rejectionErr != nil {
 			return types.RejectedError(rejectionErr.Error())
 		}

From 33129c02f79c951189bc4ab7018e855b1f563bf0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 23 Aug 2022 11:10:41 +0200
Subject: [PATCH 292/391] Add timeout parameter & trim URL (#2666)

A timeout of 10 seconds could cause issues with servers having a high `bcrypt_cost` configured in the config.
This adds a parameter to manually configure the timeout, defaults to 30 seconds.
---
 cmd/create-account/main.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index bd053f2f7..a9357f6db 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -66,10 +66,11 @@ var (
 	resetPassword      = flag.Bool("reset-password", false, "Deprecated")
 	serverURL          = flag.String("url", "https://localhost:8448", "The URL to connect to.")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
+	timeout            = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
 )
 
 var cl = http.Client{
-	Timeout:   time.Second * 10,
+	Timeout:   time.Second * 30,
 	Transport: http.DefaultTransport,
 }
 
@@ -108,6 +109,8 @@ func main() {
 		logrus.Fatalln(err)
 	}
 
+	cl.Timeout = *timeout
+
 	accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
 	if err != nil {
 		logrus.Fatalln("Failed to create the account:", err.Error())
@@ -124,8 +127,8 @@ type sharedSecretRegistrationRequest struct {
 	Admin    bool   `json:"admin"`
 }
 
-func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accesToken string, err error) {
-	registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", serverURL)
+func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accessToken string, err error) {
+	registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", strings.Trim(serverURL, "/"))
 	nonceReq, err := http.NewRequest(http.MethodGet, registerURL, nil)
 	if err != nil {
 		return "", fmt.Errorf("unable to create http request: %w", err)

From 95a509757a6ad160e8d480cf21acb999c3309099 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 23 Aug 2022 13:10:29 +0200
Subject: [PATCH 293/391] Complement QoL changes (#2663)

This PR does the following:
- adds a `keysize` parameter to `generate-keys`, so we can use lower sized keys when running in CI
- updates the Complement docker files to use BuildKit (requires Docker >18.09)
- uses `exec` when executing `dendrite-monotlith-server`, making it PID 1 inside docker, which results in Dendrite actually receiving the `SIGTERM` signal send by Docker. (Making it faster when running tests with Complement, as we don't take 10 seconds to timeout)
---
 .github/workflows/dendrite.yml              |  2 ++
 build/scripts/Complement.Dockerfile         | 20 +++++++++---------
 build/scripts/ComplementLocal.Dockerfile    | 23 ++++++++++-----------
 build/scripts/ComplementPostgres.Dockerfile | 20 +++++++++---------
 cmd/generate-keys/main.go                   |  5 +++--
 test/http.go                                |  2 +-
 test/keys.go                                | 12 +++++------
 7 files changed, 43 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 6cd7a60e2..be3c7c173 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -376,6 +376,8 @@ jobs:
       # Build initial Dendrite image
       - run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
         working-directory: dendrite
+        env:
+          DOCKER_BUILDKIT: 1
 
       # Run Complement
       - run: |
diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile
index 56877051b..14b28498b 100644
--- a/build/scripts/Complement.Dockerfile
+++ b/build/scripts/Complement.Dockerfile
@@ -1,3 +1,5 @@
+#syntax=docker/dockerfile:1.2
+
 FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
@@ -8,14 +10,12 @@ RUN mkdir /dendrite
 
 # Utilise Docker caching when downloading dependencies, this stops us needlessly
 # downloading dependencies every time.
-COPY go.mod .
-COPY go.sum .
-RUN go mod download
-
-COPY . .
-RUN go build -o /dendrite ./cmd/dendrite-monolith-server
-RUN go build -o /dendrite ./cmd/generate-keys
-RUN go build -o /dendrite ./cmd/generate-config
+RUN --mount=target=. \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o /dendrite ./cmd/generate-config && \
+    go build -o /dendrite ./cmd/generate-keys && \
+    go build -o /dendrite ./cmd/dendrite-monolith-server
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
@@ -26,7 +26,7 @@ EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
-CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
+CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile
index 704359a28..3a019fc20 100644
--- a/build/scripts/ComplementLocal.Dockerfile
+++ b/build/scripts/ComplementLocal.Dockerfile
@@ -1,3 +1,5 @@
+#syntax=docker/dockerfile:1.2
+
 # A local development Complement dockerfile, to be used with host mounts
 # /cache -> Contains the entire dendrite code at Dockerfile build time. Builds binaries but only keeps the generate-* ones. Pre-compilation saves time.
 # /dendrite -> Host-mounted sources
@@ -9,11 +11,10 @@
 FROM golang:1.18-stretch
 RUN apt-get update && apt-get install -y sqlite3
 
-WORKDIR /runtime
-
 ENV SERVER_NAME=localhost
 EXPOSE 8008 8448
 
+WORKDIR /runtime
 # This script compiles Dendrite for us.
 RUN echo '\
     #!/bin/bash -eux \n\
@@ -29,25 +30,23 @@ RUN echo '\
 RUN echo '\
     #!/bin/bash -eu \n\
     ./generate-keys --private-key matrix_key.pem \n\
-    ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
+    ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
-    ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
     ' > run.sh && chmod +x run.sh
 
 
 WORKDIR /cache
-# Pre-download deps; we don't need to do this if the GOPATH is mounted.
-COPY go.mod .
-COPY go.sum .
-RUN go mod download
-
 # Build the monolith in /cache - we won't actually use this but will rely on build artifacts to speed
 # up the real compilation. Build the generate-* binaries in the true /runtime locations.
 # If the generate-* source is changed, this dockerfile needs re-running.
-COPY . .
-RUN go build ./cmd/dendrite-monolith-server && go build -o /runtime ./cmd/generate-keys && go build -o /runtime ./cmd/generate-config
+RUN --mount=target=. \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o /runtime ./cmd/generate-config && \
+    go build -o /runtime ./cmd/generate-keys
 
 
 WORKDIR /runtime
-CMD /runtime/compile.sh && /runtime/run.sh
+CMD /runtime/compile.sh && exec /runtime/run.sh
diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile
index a8b4fbb1d..699540120 100644
--- a/build/scripts/ComplementPostgres.Dockerfile
+++ b/build/scripts/ComplementPostgres.Dockerfile
@@ -1,3 +1,5 @@
+#syntax=docker/dockerfile:1.2
+
 FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
@@ -26,14 +28,12 @@ RUN mkdir /dendrite
 
 # Utilise Docker caching when downloading dependencies, this stops us needlessly
 # downloading dependencies every time.
-COPY go.mod .
-COPY go.sum .
-RUN go mod download
-
-COPY . .
-RUN go build -o /dendrite ./cmd/dendrite-monolith-server
-RUN go build -o /dendrite ./cmd/generate-keys
-RUN go build -o /dendrite ./cmd/generate-config
+RUN --mount=target=. \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o /dendrite ./cmd/generate-config && \
+    go build -o /dendrite ./cmd/generate-keys && \
+    go build -o /dendrite ./cmd/dendrite-monolith-server
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
@@ -45,10 +45,10 @@ EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
-CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
+CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
     # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
     sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
     sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/cmd/generate-keys/main.go b/cmd/generate-keys/main.go
index 8acd28be0..d4c8cf78a 100644
--- a/cmd/generate-keys/main.go
+++ b/cmd/generate-keys/main.go
@@ -38,6 +38,7 @@ var (
 	authorityCertFile = flag.String("tls-authority-cert", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
 	authorityKeyFile  = flag.String("tls-authority-key", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
 	serverName        = flag.String("server", "", "Optional: Create TLS certificate/keys with this domain name set. Useful for integration testing.")
+	keySize           = flag.Int("keysize", 4096, "Optional: Create TLS RSA private key with the given key size")
 )
 
 func main() {
@@ -58,12 +59,12 @@ func main() {
 			log.Fatal("Zero or both of --tls-key and --tls-cert must be supplied")
 		}
 		if *authorityCertFile == "" && *authorityKeyFile == "" {
-			if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile); err != nil {
+			if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile, *keySize); err != nil {
 				panic(err)
 			}
 		} else {
 			// generate the TLS cert/key based on the authority given.
-			if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile); err != nil {
+			if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile, *keySize); err != nil {
 				panic(err)
 			}
 		}
diff --git a/test/http.go b/test/http.go
index 37b3648f8..8cd83d0a6 100644
--- a/test/http.go
+++ b/test/http.go
@@ -68,7 +68,7 @@ func ListenAndServe(t *testing.T, router http.Handler, withTLS bool) (apiURL str
 		if withTLS {
 			certFile := filepath.Join(t.TempDir(), "dendrite.cert")
 			keyFile := filepath.Join(t.TempDir(), "dendrite.key")
-			err = NewTLSKey(keyFile, certFile)
+			err = NewTLSKey(keyFile, certFile, 1024)
 			if err != nil {
 				t.Errorf("failed to make TLS key: %s", err)
 				return
diff --git a/test/keys.go b/test/keys.go
index 327c6ed7b..fb156ef27 100644
--- a/test/keys.go
+++ b/test/keys.go
@@ -69,8 +69,8 @@ func NewMatrixKey(matrixKeyPath string) (err error) {
 
 const certificateDuration = time.Hour * 24 * 365 * 10
 
-func generateTLSTemplate(dnsNames []string) (*rsa.PrivateKey, *x509.Certificate, error) {
-	priv, err := rsa.GenerateKey(rand.Reader, 4096)
+func generateTLSTemplate(dnsNames []string, bitSize int) (*rsa.PrivateKey, *x509.Certificate, error) {
+	priv, err := rsa.GenerateKey(rand.Reader, bitSize)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -118,8 +118,8 @@ func writePrivateKey(tlsKeyPath string, priv *rsa.PrivateKey) error {
 }
 
 // NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
-func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
-	priv, template, err := generateTLSTemplate(nil)
+func NewTLSKey(tlsKeyPath, tlsCertPath string, keySize int) error {
+	priv, template, err := generateTLSTemplate(nil, keySize)
 	if err != nil {
 		return err
 	}
@@ -136,8 +136,8 @@ func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
 	return writePrivateKey(tlsKeyPath, priv)
 }
 
-func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string) error {
-	priv, template, err := generateTLSTemplate([]string{serverName})
+func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string, keySize int) error {
+	priv, template, err := generateTLSTemplate([]string{serverName}, keySize)
 	if err != nil {
 		return err
 	}

From 14fea600bbeeb745c1ff1fc1519e4db968c17f86 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 23 Aug 2022 13:57:11 +0100
Subject: [PATCH 294/391] Detect `types.MissingStateError` in
 `CheckServerAllowedToSeeEvent` (#2667)

This will hopefully stop some 500 errors on `/event` where there is no state-before known.
---
 roomserver/internal/helpers/helpers.go      | 11 +++++++++--
 roomserver/storage/postgres/events_table.go |  2 +-
 roomserver/storage/sqlite3/events_table.go  |  2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index 6091f8ec2..cbd1561f7 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -254,8 +254,15 @@ func CheckServerAllowedToSeeEvent(
 			return false, err
 		}
 	default:
-		// Something else went wrong
-		return false, err
+		switch err.(type) {
+		case types.MissingStateError:
+			// If there's no state then we assume it's open visibility, as Synapse does:
+			// https://github.com/matrix-org/synapse/blob/aec87a0f9369a3015b2a53469f88d1de274e8b71/synapse/visibility.py#L654-L655
+			return true, nil
+		default:
+			// Something else went wrong
+			return false, err
+		}
 	}
 	return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil
 }
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index a310c3963..e758837a0 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -346,7 +346,7 @@ func (s *eventStatements) BulkSelectStateAtEventByID(
 		// Genuine create events are the only case where it's OK to have no previous state.
 		isCreate := result.EventTypeNID == types.MRoomCreateNID && result.EventStateKeyNID == 1
 		if result.BeforeStateSnapshotNID == 0 && !isCreate {
-			return nil, types.MissingEventError(
+			return nil, types.MissingStateError(
 				fmt.Sprintf("storage: missing state for event NID %d", result.EventNID),
 			)
 		}
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index 943f256eb..b5cc84bc5 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -362,7 +362,7 @@ func (s *eventStatements) BulkSelectStateAtEventByID(
 		// Genuine create events are the only case where it's OK to have no previous state.
 		isCreate := result.EventTypeNID == types.MRoomCreateNID && result.EventStateKeyNID == 1
 		if result.BeforeStateSnapshotNID == 0 && !isCreate {
-			return nil, types.MissingEventError(
+			return nil, types.MissingStateError(
 				fmt.Sprintf("storage: missing state for event NID %d", result.EventNID),
 			)
 		}

From 78e5d05efc95c959b380d6ee4230f75885a597ad Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 23 Aug 2022 16:54:42 +0200
Subject: [PATCH 295/391] Only set backOffStarted to false if until is not zero
 (#2669)

---
 federationapi/statistics/statistics.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go
index 8bac99cbc..b8e16a259 100644
--- a/federationapi/statistics/statistics.go
+++ b/federationapi/statistics/statistics.go
@@ -5,10 +5,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 	"go.uber.org/atomic"
+
+	"github.com/matrix-org/dendrite/federationapi/storage"
 )
 
 // Statistics contains information about all of the remote federated
@@ -126,13 +127,13 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 
 		go func() {
 			until, ok := s.backoffUntil.Load().(time.Time)
-			if ok {
+			if ok && !until.IsZero() {
 				select {
 				case <-time.After(time.Until(until)):
 				case <-s.interrupt:
 				}
+				s.backoffStarted.Store(false)
 			}
-			s.backoffStarted.Store(false)
 		}()
 	}
 

From 522bd2999f605258e95565c6d648d2f7ea001ea4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 24 Aug 2022 14:03:06 +0100
Subject: [PATCH 296/391] Allow un-rejecting events on reprocessing

---
 roomserver/api/wrapper.go                   | 8 ++++++++
 roomserver/storage/postgres/events_table.go | 2 +-
 roomserver/storage/sqlite3/events_table.go  | 2 +-
 syncapi/internal/keychange.go               | 2 +-
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go
index bc2f28176..8b031982c 100644
--- a/roomserver/api/wrapper.go
+++ b/roomserver/api/wrapper.go
@@ -19,6 +19,7 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // SendEvents to the roomserver The events are written with KindNew.
@@ -69,6 +70,13 @@ func SendEventWithState(
 		stateEventIDs[i] = stateEvents[i].EventID()
 	}
 
+	logrus.WithContext(ctx).WithFields(logrus.Fields{
+		"room_id":   event.RoomID(),
+		"event_id":  event.EventID(),
+		"outliers":  len(ires),
+		"state_ids": len(stateEventIDs),
+	}).Infof("Submitting %q event to roomserver with state snapshot", event.Type())
+
 	ires = append(ires, InputRoomEvent{
 		Kind:          kind,
 		Event:         event,
diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go
index e758837a0..1e7ca7669 100644
--- a/roomserver/storage/postgres/events_table.go
+++ b/roomserver/storage/postgres/events_table.go
@@ -74,7 +74,7 @@ const insertEventSQL = "" +
 	"INSERT INTO roomserver_events AS e (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids, depth, is_rejected)" +
 	" VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" +
 	" ON CONFLICT ON CONSTRAINT roomserver_event_id_unique DO UPDATE" +
-	" SET is_rejected = $8 WHERE e.event_id = $4 AND e.is_rejected = FALSE" +
+	" SET is_rejected = $8 WHERE e.event_id = $4 AND e.is_rejected = TRUE" +
 	" RETURNING event_nid, state_snapshot_nid"
 
 const selectEventSQL = "" +
diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go
index b5cc84bc5..950d03b03 100644
--- a/roomserver/storage/sqlite3/events_table.go
+++ b/roomserver/storage/sqlite3/events_table.go
@@ -50,7 +50,7 @@ const insertEventSQL = `
 	INSERT INTO roomserver_events (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids, depth, is_rejected)
 	  VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
 	  ON CONFLICT DO UPDATE
-	  SET is_rejected = $8 WHERE is_rejected = 0
+	  SET is_rejected = $8 WHERE is_rejected = 1
 	  RETURNING event_nid, state_snapshot_nid;
 `
 
diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go
index c5180e338..3d6b2a7f3 100644
--- a/syncapi/internal/keychange.go
+++ b/syncapi/internal/keychange.go
@@ -127,7 +127,7 @@ func DeviceListCatchup(
 		"from":            offset,
 		"to":              toOffset,
 		"response_offset": queryRes.Offset,
-	}).Debugf("QueryKeyChanges request result: %+v", res.DeviceLists)
+	}).Tracef("QueryKeyChanges request result: %+v", res.DeviceLists)
 
 	return types.StreamPosition(queryRes.Offset), hasNew, nil
 }

From 16156b0b0988e7b1746b2834e6357c3c90bc8465 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 25 Aug 2022 09:51:36 +0100
Subject: [PATCH 297/391] Fix 500s on `/state`, `/state_ids` when state not
 known (#2672)

This was due to bad error bubbling.
---
 federationapi/routing/join.go      |  6 ++++++
 federationapi/routing/state.go     |  6 ++++++
 roomserver/api/query.go            |  1 +
 roomserver/internal/query/query.go | 17 ++++++++++-------
 4 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index b48eaf78e..1a1219873 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -329,6 +329,12 @@ func SendJoin(
 			JSON: jsonerror.NotFound("Room does not exist"),
 		}
 	}
+	if !stateAndAuthChainResponse.StateKnown {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("State not known"),
+		}
+	}
 
 	// Check if the user is already in the room. If they're already in then
 	// there isn't much point in sending another join event into the room.
diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go
index 6fdce20ce..5377eb88f 100644
--- a/federationapi/routing/state.go
+++ b/federationapi/routing/state.go
@@ -135,6 +135,12 @@ func getState(
 		return nil, nil, &resErr
 	}
 
+	if !response.StateKnown {
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("State not known"),
+		}
+	}
 	if response.IsRejected {
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index c8e6f9dc6..32d63bb51 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -227,6 +227,7 @@ type QueryStateAndAuthChainResponse struct {
 	// Do all the previous events exist on this roomserver?
 	// If some of previous events do not exist this will be false and StateEvents will be empty.
 	PrevEventsExist bool `json:"prev_events_exist"`
+	StateKnown      bool `json:"state_known"`
 	// The state and auth chain events that were requested.
 	// The lists will be in an arbitrary order.
 	StateEvents     []*gomatrixserverlib.HeaderedEvent `json:"state_events"`
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index f5d8c2d49..6dce2bc3e 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -503,10 +503,11 @@ func (r *Queryer) QueryStateAndAuthChain(
 	}
 
 	var stateEvents []*gomatrixserverlib.Event
-	stateEvents, rejected, err := r.loadStateAtEventIDs(ctx, info, request.PrevEventIDs)
+	stateEvents, rejected, stateMissing, err := r.loadStateAtEventIDs(ctx, info, request.PrevEventIDs)
 	if err != nil {
 		return err
 	}
+	response.StateKnown = !stateMissing
 	response.IsRejected = rejected
 	response.PrevEventsExist = true
 
@@ -542,15 +543,18 @@ func (r *Queryer) QueryStateAndAuthChain(
 	return err
 }
 
-func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]*gomatrixserverlib.Event, bool, error) {
+// first bool: is rejected, second bool: state missing
+func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]*gomatrixserverlib.Event, bool, bool, error) {
 	roomState := state.NewStateResolution(r.DB, roomInfo)
 	prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs)
 	if err != nil {
 		switch err.(type) {
 		case types.MissingEventError:
-			return nil, false, nil
+			return nil, false, true, nil
+		case types.MissingStateError:
+			return nil, false, true, nil
 		default:
-			return nil, false, err
+			return nil, false, false, err
 		}
 	}
 	// Currently only used on /state and /state_ids
@@ -567,12 +571,11 @@ func (r *Queryer) loadStateAtEventIDs(ctx context.Context, roomInfo *types.RoomI
 		ctx, prevStates,
 	)
 	if err != nil {
-		return nil, rejected, err
+		return nil, rejected, false, err
 	}
 
 	events, err := helpers.LoadStateEvents(ctx, r.DB, stateEntries)
-
-	return events, rejected, err
+	return events, rejected, false, err
 }
 
 type eventsFromIDs func(context.Context, []string) ([]types.Event, error)

From cd7fa34595c0b4107677e32439b0d02599db8e51 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 25 Aug 2022 10:57:27 +0100
Subject: [PATCH 298/391] Tweak logging and Sentry reporting for roomserver
 input

---
 roomserver/internal/input/input.go | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 8d24f3c59..429cc4bd2 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -36,6 +36,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/internal/query"
 	"github.com/matrix-org/dendrite/roomserver/producers"
 	"github.com/matrix-org/dendrite/roomserver/storage"
+	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -247,14 +248,24 @@ func (w *worker) _next() {
 	// it was a synchronous request.
 	var errString string
 	if err = w.r.processRoomEvent(w.r.ProcessContext.Context(), &inputRoomEvent); err != nil {
-		if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
-			sentry.CaptureException(err)
+		switch err.(type) {
+		case types.RejectedError:
+			// Don't send events that were rejected to Sentry
+			logrus.WithError(err).WithFields(logrus.Fields{
+				"room_id":  w.roomID,
+				"event_id": inputRoomEvent.Event.EventID(),
+				"type":     inputRoomEvent.Event.Type(),
+			}).Warn("Roomserver rejected event")
+		default:
+			if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
+				sentry.CaptureException(err)
+			}
+			logrus.WithError(err).WithFields(logrus.Fields{
+				"room_id":  w.roomID,
+				"event_id": inputRoomEvent.Event.EventID(),
+				"type":     inputRoomEvent.Event.Type(),
+			}).Warn("Roomserver failed to process event")
 		}
-		logrus.WithError(err).WithFields(logrus.Fields{
-			"room_id":  w.roomID,
-			"event_id": inputRoomEvent.Event.EventID(),
-			"type":     inputRoomEvent.Event.Type(),
-		}).Warn("Roomserver failed to process async event")
 		_ = msg.Term()
 		errString = err.Error()
 	} else {

From 8ff3f1a7c9578a3a4f95755c4698da3219777097 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 25 Aug 2022 11:01:07 +0100
Subject: [PATCH 299/391] Remove a couple unnecessary Sentry captures from
 backfill

---
 roomserver/internal/perform/perform_backfill.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index de76b6412..51c66415a 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -320,7 +319,6 @@ FederationHit:
 		b.eventIDToBeforeStateIDs[targetEvent.EventID()] = res
 		return res, nil
 	}
-	sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
 	return nil, lastErr
 }
 
@@ -398,7 +396,6 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
 		}
 		return result, nil
 	}
-	sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
 	return nil, lastErr
 }
 

From 07dd9bd9954d0740664651bf9755a81c3ba2d011 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 25 Aug 2022 14:42:47 +0200
Subject: [PATCH 300/391] SyncAPI tweaks/fixes (#2671)

- Reverts 9dc57122d991d54ea6750448ba88c8763a569830 as it was causing issues https://github.com/matrix-org/dendrite/issues/2660
- Updates the GMSL `DefaultStateFilter` to use a limit of 20 events
- Uses the timeline events to determine the new position instead of the state events
---
 go.mod                               |  2 +-
 go.sum                               |  4 +-
 syncapi/storage/interface.go         |  3 +-
 syncapi/storage/shared/syncserver.go | 34 ++++++++---------
 syncapi/streams/stream_pdu.go        | 57 +++++++++++-----------------
 syncapi/sync/request.go              | 28 ++++++++------
 syncapi/sync/requestpool.go          |  2 +-
 syncapi/syncapi_test.go              |  2 +-
 syncapi/types/types.go               |  1 +
 9 files changed, 64 insertions(+), 69 deletions(-)

diff --git a/go.mod b/go.mod
index fe4604a03..0833f2908 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220824082345-662dca17bf94
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 7ac9fc6e2..fcbdb7e41 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c h1:GhKmb8s9iXA9qsFD1SbiRo6Ee7cnbfcgJQ/iy43wczM=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220815094957-74b7ff4ae09c/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220824082345-662dca17bf94 h1:zoTv/qxg7C/O995JBPvp+Z8KMR69HhB+M+P22A8Hmm0=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220824082345-662dca17bf94/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 43a75da95..0c8ba4e3d 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -19,10 +19,11 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index a46e55256..b06d2c6a9 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -20,15 +20,18 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/tidwall/gjson"
+
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 )
 
 // Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite
@@ -683,7 +686,7 @@ func (d *Database) GetStateDeltas(
 	ctx context.Context, device *userapi.Device,
 	r types.Range, userID string,
 	stateFilter *gomatrixserverlib.StateFilter,
-) ([]types.StateDelta, []string, error) {
+) (deltas []types.StateDelta, joinedRoomsIDs []string, err error) {
 	// Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821
 	// - Get membership list changes for this user in this sync response
 	// - For each room which has membership list changes:
@@ -718,8 +721,6 @@ func (d *Database) GetStateDeltas(
 		}
 	}
 
-	var deltas []types.StateDelta
-
 	// get all the state events ever (i.e. for all available rooms) between these two positions
 	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs)
 	if err != nil {
@@ -767,15 +768,11 @@ func (d *Database) GetStateDeltas(
 	}
 
 	// handle newly joined rooms and non-joined rooms
+	newlyJoinedRooms := make(map[string]bool, len(state))
 	for roomID, stateStreamEvents := range state {
 		for _, ev := range stateStreamEvents {
-			// TODO: Currently this will incorrectly add rooms which were ALREADY joined but they sent another no-op join event.
-			//       We should be checking if the user was already joined at fromPos and not proceed if so. As a result of this,
-			//       dupe join events will result in the entire room state coming down to the client again. This is added in
-			//       the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to
-			//       the timeline.
-			if membership := getMembershipFromEvent(ev.Event, userID); membership != "" {
-				if membership == gomatrixserverlib.Join {
+			if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" {
+				if membership == gomatrixserverlib.Join && prevMembership != membership {
 					// send full room state down instead of a delta
 					var s []types.StreamEvent
 					s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter)
@@ -786,6 +783,7 @@ func (d *Database) GetStateDeltas(
 						return nil, nil, err
 					}
 					state[roomID] = s
+					newlyJoinedRooms[roomID] = true
 					continue // we'll add this room in when we do joined rooms
 				}
 
@@ -806,6 +804,7 @@ func (d *Database) GetStateDeltas(
 			Membership:  gomatrixserverlib.Join,
 			StateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]),
 			RoomID:      joinedRoomID,
+			NewlyJoined: newlyJoinedRooms[joinedRoomID],
 		})
 	}
 
@@ -892,7 +891,7 @@ func (d *Database) GetStateDeltasForFullStateSync(
 
 	for roomID, stateStreamEvents := range state {
 		for _, ev := range stateStreamEvents {
-			if membership := getMembershipFromEvent(ev.Event, userID); membership != "" {
+			if membership, _ := getMembershipFromEvent(ev.Event, userID); membership != "" {
 				if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above.
 					deltas[roomID] = types.StateDelta{
 						Membership:    membership,
@@ -1003,15 +1002,16 @@ func (d *Database) CleanSendToDeviceUpdates(
 
 // getMembershipFromEvent returns the value of content.membership iff the event is a state event
 // with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned.
-func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string {
+func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) (string, string) {
 	if ev.Type() != "m.room.member" || !ev.StateKeyEquals(userID) {
-		return ""
+		return "", ""
 	}
 	membership, err := ev.Membership()
 	if err != nil {
-		return ""
+		return "", ""
 	}
-	return membership
+	prevMembership := gjson.GetBytes(ev.Unsigned(), "prev_content.membership").Str
+	return membership, prevMembership
 }
 
 // StoreReceipt stores user receipts
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 2818aad87..fa4c722ce 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -209,11 +209,27 @@ func (p *PDUStreamProvider) IncrementalSync(
 
 	newPos = from
 	for _, delta := range stateDeltas {
+		newRange := r
+		// If this room was joined in this sync, try to fetch
+		// as much timeline events as allowed by the filter.
+		if delta.NewlyJoined {
+			// Reverse the range, so we get the most recent first.
+			// This will be limited by the eventFilter.
+			newRange = types.Range{
+				From:      r.To,
+				To:        0,
+				Backwards: true,
+			}
+		}
 		var pos types.StreamPosition
-		if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, &stateFilter, req.Response); err != nil {
+		if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
 			return to
 		}
+		// Reset the position, as it is only for the special case of newly joined rooms
+		if delta.NewlyJoined {
+			pos = newRange.From
+		}
 		switch {
 		case r.Backwards && pos < newPos:
 			fallthrough
@@ -222,37 +238,6 @@ func (p *PDUStreamProvider) IncrementalSync(
 		}
 	}
 
-	// If we joined a new room in this sync, make sure we add enough information about it.
-	// This does an "initial sync" for the newly joined rooms
-	newlyJoinedRooms := joinedRooms(req.Response, req.Device.UserID)
-	if len(newlyJoinedRooms) > 0 {
-		// remove already added rooms, as we're doing an "initial sync"
-		for _, x := range newlyJoinedRooms {
-			delete(req.Response.Rooms.Join, x)
-		}
-		r = types.Range{
-			From:      to,
-			To:        0,
-			Backwards: true,
-		}
-		// We only care about the newly joined rooms, so update the stateFilter to reflect that
-		stateFilter.Rooms = &newlyJoinedRooms
-		if stateDeltas, _, err = p.DB.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
-			req.Log.WithError(err).Error("p.DB.GetStateDeltas failed")
-			return newPos
-		}
-		for _, delta := range stateDeltas {
-			// Ignore deltas for rooms we didn't newly join
-			if _, ok := req.Response.Rooms.Join[delta.RoomID]; ok {
-				continue
-			}
-			if _, err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, &stateFilter, req.Response); err != nil {
-				req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
-				return newPos
-			}
-		}
-	}
-
 	return newPos
 }
 
@@ -340,12 +325,12 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 	}
 
-	if len(events) > 0 {
-		updateLatestPosition(events[len(events)-1].EventID())
-	}
 	if len(delta.StateEvents) > 0 {
 		updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID())
 	}
+	if len(events) > 0 {
+		updateLatestPosition(events[len(events)-1].EventID())
+	}
 
 	switch delta.Membership {
 	case gomatrixserverlib.Join:
@@ -418,6 +403,8 @@ func applyHistoryVisibilityFilter(
 	logrus.WithFields(logrus.Fields{
 		"duration": time.Since(startTime),
 		"room_id":  roomID,
+		"before":   len(recentEvents),
+		"after":    len(events),
 	}).Debug("applied history visibility (sync)")
 	return events, nil
 }
diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go
index 9d4740e93..268ed70c6 100644
--- a/syncapi/sync/request.go
+++ b/syncapi/sync/request.go
@@ -23,12 +23,13 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/matrix-org/dendrite/syncapi/storage"
-	"github.com/matrix-org/dendrite/syncapi/types"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 const defaultSyncTimeout = time.Duration(0)
@@ -46,15 +47,9 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
 			return nil, err
 		}
 	}
-	// TODO: read from stored filters too
+
+	// Create a default filter and apply a stored filter on top of it (if specified)
 	filter := gomatrixserverlib.DefaultFilter()
-	if since.IsEmpty() {
-		// Send as much account data down for complete syncs as possible
-		// by default, otherwise clients do weird things while waiting
-		// for the rest of the data to trickle down.
-		filter.AccountData.Limit = math.MaxInt32
-		filter.Room.AccountData.Limit = math.MaxInt32
-	}
 	filterQuery := req.URL.Query().Get("filter")
 	if filterQuery != "" {
 		if filterQuery[0] == '{' {
@@ -76,6 +71,17 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
 		}
 	}
 
+	// A loaded filter might have overwritten these values,
+	// so set them after loading the filter.
+	if since.IsEmpty() {
+		// Send as much account data down for complete syncs as possible
+		// by default, otherwise clients do weird things while waiting
+		// for the rest of the data to trickle down.
+		filter.AccountData.Limit = math.MaxInt32
+		filter.Room.AccountData.Limit = math.MaxInt32
+		filter.Room.State.Limit = math.MaxInt32
+	}
+
 	logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{
 		"user_id":   device.UserID,
 		"device_id": device.ID,
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index d908a9629..c2c9616e8 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -298,8 +298,8 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 				return giveup()
 
 			case <-userStreamListener.GetNotifyChannel(syncReq.Since):
-				syncReq.Log.Debugln("Responding to sync after wake-up")
 				currentPos.ApplyUpdates(userStreamListener.GetSyncPosition())
+				syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync after wake-up")
 			}
 		} else {
 			syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index 76d51c86b..c81256aa7 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -195,7 +195,7 @@ func TestSyncAPICreateRoomSyncEarly(t *testing.T) {
 }
 
 func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) {
-	t.SkipNow() // Temporary?
+	t.Skip("Skipped, possibly fixed")
 	user := test.NewUser(t)
 	room := test.NewRoom(t, user)
 	alice := userapi.Device{
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index 39b085d9c..d75d53ca9 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -37,6 +37,7 @@ var (
 type StateDelta struct {
 	RoomID      string
 	StateEvents []*gomatrixserverlib.HeaderedEvent
+	NewlyJoined bool
 	Membership  string
 	// The PDU stream position of the latest membership event for this user, if applicable.
 	// Can be 0 if there is no membership event in this delta.

From ed79e8626aa3fc909b9ff2dbe4a1e16620ee0e37 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 25 Aug 2022 14:14:10 +0100
Subject: [PATCH 301/391] Version 0.9.5 (#2673)

Changelog and version bump.
---
 CHANGES.md          | 12 ++++++++++++
 internal/version.go |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index aaf5836ba..0f57bffcb 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,17 @@
 # Changelog
 
+## Dendrite 0.9.5 (2022-08-25)
+
+### Fixes
+
+* The roomserver will now correctly unreject previously rejected events if necessary when reprocessing
+* The handling of event soft-failure has been improved on the roomserver input by no longer applying rejection rules and still calculating state before the event if possible
+* The federation `/state` and `/state_ids` endpoints should now return the correct error code when the state isn't known instead of returning a HTTP 500
+* The federation `/event` should now return outlier events correctly instead of returning a HTTP 500
+* A bug in the federation backoff allowing zero intervals has been corrected
+* The `create-account` utility will no longer error if the homeserver URL ends in a trailing slash
+* A regression in `/sync` introduced in 0.9.4 should be fixed
+
 ## Dendrite 0.9.4 (2022-08-19)
 
 ### Fixes
diff --git a/internal/version.go b/internal/version.go
index 384f091a0..108b8ab0f 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 4
+	VersionPatch = 5
 	VersionTag   = "" // example: "rc1"
 )
 

From 38bed30b411d8e438d430eae2670482eb2778628 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 26 Aug 2022 09:56:41 +0100
Subject: [PATCH 302/391] Restrict username length properly (#2676)

This fixes #2674 by fixing the username check to comply with the [spec appendices](https://spec.matrix.org/v1.3/appendices/#user-identifiers):

> The length of a user ID, including the @ sigil and the domain, MUST NOT exceed 255 characters.
---
 clientapi/routing/register.go | 30 +++++++++++++++---------------
 clientapi/routing/routing.go  |  2 +-
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go
index af0329a48..0bda1e488 100644
--- a/clientapi/routing/register.go
+++ b/clientapi/routing/register.go
@@ -276,19 +276,19 @@ type recaptchaResponse struct {
 }
 
 // validateUsername returns an error response if the username is invalid
-func validateUsername(username string) *util.JSONResponse {
+func validateUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
 	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
-	if len(username) > maxUsernameLength {
+	if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("'username' >%d characters", maxUsernameLength)),
+			JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
 		}
-	} else if !validUsernameRegex.MatchString(username) {
+	} else if !validUsernameRegex.MatchString(localpart) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
 		}
-	} else if username[0] == '_' { // Regex checks its not a zero length string
+	} else if localpart[0] == '_' { // Regex checks its not a zero length string
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.InvalidUsername("Username cannot start with a '_'"),
@@ -298,13 +298,13 @@ func validateUsername(username string) *util.JSONResponse {
 }
 
 // validateApplicationServiceUsername returns an error response if the username is invalid for an application service
-func validateApplicationServiceUsername(username string) *util.JSONResponse {
-	if len(username) > maxUsernameLength {
+func validateApplicationServiceUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
+	if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("'username' >%d characters", maxUsernameLength)),
+			JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
 		}
-	} else if !validUsernameRegex.MatchString(username) {
+	} else if !validUsernameRegex.MatchString(localpart) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
@@ -523,7 +523,7 @@ func validateApplicationService(
 	}
 
 	// Check username application service is trying to register is valid
-	if err := validateApplicationServiceUsername(username); err != nil {
+	if err := validateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
 		return "", err
 	}
 
@@ -604,7 +604,7 @@ func Register(
 	case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil:
 		// Spec-compliant case (the access_token is specified and the login type
 		// is correctly set, so it's an appservice registration)
-		if resErr := validateApplicationServiceUsername(r.Username); resErr != nil {
+		if resErr := validateApplicationServiceUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
 			return *resErr
 		}
 	case accessTokenErr == nil:
@@ -617,7 +617,7 @@ func Register(
 	default:
 		// Spec-compliant case (neither the access_token nor the login type are
 		// specified, so it's a normal user registration)
-		if resErr := validateUsername(r.Username); resErr != nil {
+		if resErr := validateUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
 			return *resErr
 		}
 	}
@@ -1018,7 +1018,7 @@ func RegisterAvailable(
 	// Squash username to all lowercase letters
 	username = strings.ToLower(username)
 
-	if err := validateUsername(username); err != nil {
+	if err := validateUsername(username, cfg.Matrix.ServerName); err != nil {
 		return *err
 	}
 
@@ -1059,7 +1059,7 @@ func RegisterAvailable(
 	}
 }
 
-func handleSharedSecretRegistration(userAPI userapi.ClientUserAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
+func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.ClientUserAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
 	ssrr, err := NewSharedSecretRegistrationRequest(req.Body)
 	if err != nil {
 		return util.JSONResponse{
@@ -1080,7 +1080,7 @@ func handleSharedSecretRegistration(userAPI userapi.ClientUserAPI, sr *SharedSec
 	// downcase capitals
 	ssrr.User = strings.ToLower(ssrr.User)
 
-	if resErr := validateUsername(ssrr.User); resErr != nil {
+	if resErr := validateUsername(ssrr.User, cfg.Matrix.ServerName); resErr != nil {
 		return *resErr
 	}
 	if resErr := validatePassword(ssrr.Password); resErr != nil {
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 6904a2b34..d7a48d228 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -133,7 +133,7 @@ func Setup(
 					}
 				}
 				if req.Method == http.MethodPost {
-					return handleSharedSecretRegistration(userAPI, sr, req)
+					return handleSharedSecretRegistration(cfg, userAPI, sr, req)
 				}
 				return util.JSONResponse{
 					Code: http.StatusMethodNotAllowed,

From a3eb4e5e9806745f3e6fd36f43bfa07333650cb4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 28 Aug 2022 09:52:03 +0100
Subject: [PATCH 303/391] Enforce `type`, `sender`, `state_key` and `room_id`
 lengths using codepoints rather than bytes (update to
 matrix-org/gomatrixserverlib@5f66df0, matrix-org/matrix-spec#1001)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 0833f2908..1f55ee5b8 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220824082345-662dca17bf94
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220828085018-5f66df08dc66
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index fcbdb7e41..8e98e0bcf 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220824082345-662dca17bf94 h1:zoTv/qxg7C/O995JBPvp+Z8KMR69HhB+M+P22A8Hmm0=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220824082345-662dca17bf94/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220828085018-5f66df08dc66 h1:4+ycQUZ4Do5LeEuiesz6gD7LkjRNUtk8oETFqjalZxQ=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220828085018-5f66df08dc66/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From aba171d9bcfcfa31e8f1b16e2d91b256d0cfa07c Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 29 Aug 2022 07:23:46 +0200
Subject: [PATCH 304/391] Default to limit of 20 state events

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 1f55ee5b8..ee025c354 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220828085018-5f66df08dc66
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220829052120-b3746121e530
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 8e98e0bcf..d584a7333 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220828085018-5f66df08dc66 h1:4+ycQUZ4Do5LeEuiesz6gD7LkjRNUtk8oETFqjalZxQ=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220828085018-5f66df08dc66/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220829052120-b3746121e530 h1:G0xIy9wpOGpv/xkCneAM22bJ36cXsSpa7AThmtvClyE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220829052120-b3746121e530/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 704cc5c9f500bdd41a1c99ff2ae2fcef5eb74abb Mon Sep 17 00:00:00 2001
From: Brian Meek <brian@hntlabs.com>
Date: Mon, 29 Aug 2022 00:10:42 -0700
Subject: [PATCH 305/391] Race in keyserver intialization (#2619)

Signed-off-by: Brian Meek <brian@hntlabs.com>
---
 keyserver/keyserver.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go
index cd506f981..75cdaedb4 100644
--- a/keyserver/keyserver.go
+++ b/keyserver/keyserver.go
@@ -60,7 +60,7 @@ func NewInternalAPI(
 	updater := internal.NewDeviceListUpdater(db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable
 	ap.Updater = updater
 	go func() {
-		if err = updater.Start(); err != nil {
+		if err := updater.Start(); err != nil {
 			logrus.WithError(err).Panicf("failed to start device list updater")
 		}
 	}()
@@ -68,7 +68,7 @@ func NewInternalAPI(
 	dlConsumer := consumers.NewDeviceListUpdateConsumer(
 		base.ProcessContext, cfg, js, updater,
 	)
-	if err = dlConsumer.Start(); err != nil {
+	if err := dlConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start device list consumer")
 	}
 

From 21ef487ff8c36f7302e5fc47d3602ea0fc5ae3d3 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 29 Aug 2022 14:18:33 +0200
Subject: [PATCH 306/391] Log roomserver error

---
 appservice/consumers/roomserver.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index 37d4ef9c2..d567408be 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -18,14 +18,15 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+
 	"github.com/matrix-org/dendrite/appservice/storage"
 	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -103,6 +104,7 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg)
 			}
 			if len(eventsReq.EventIDs) > 0 {
 				if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+					log.WithError(err).Errorf("s.rsAPI.QueryEventsByID failed")
 					return false
 				}
 				events = append(events, eventsRes.Events...)

From bbb3ade4a2b49cfdaf7ec86ddf079ff7d48e0cf3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 30 Aug 2022 11:44:26 +0100
Subject: [PATCH 307/391] Fix the Cat Overflow bug (update to
 matrix-org/gomatrixserverlib@2020bdc)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index ee025c354..8c334f1e8 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220829052120-b3746121e530
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220830104348-2020bdc55859
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index d584a7333..aac86010b 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220829052120-b3746121e530 h1:G0xIy9wpOGpv/xkCneAM22bJ36cXsSpa7AThmtvClyE=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220829052120-b3746121e530/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220830104348-2020bdc55859 h1:14mOIE2sGS6YTG1+3GFbVWmAisW5pzyfn3SxJTjkR2I=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220830104348-2020bdc55859/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From b0e2ea0f374892a6f9c77623bc6c3fd9ea572b3b Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 30 Aug 2022 13:59:13 +0200
Subject: [PATCH 308/391] Fix race condition on startup (#2679)

`SetupAndServeHTTP` would race in `configureHTTPErrors` and while configuring routes.
---
 setup/base/base.go | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/setup/base/base.go b/setup/base/base.go
index b21eeba47..87f415764 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -25,21 +25,23 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 	"time"
 
 	"github.com/getsentry/sentry-go"
 	sentryhttp "github.com/getsentry/sentry-go/http"
-	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/internal/pushgateway"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/atomic"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/internal/pushgateway"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -47,6 +49,8 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/kardianos/minwinsvc"
 
+	"github.com/sirupsen/logrus"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	asinthttp "github.com/matrix-org/dendrite/appservice/inthttp"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
@@ -58,7 +62,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp"
-	"github.com/sirupsen/logrus"
 )
 
 // BaseDendrite is a base for creating new instances of dendrite. It parses
@@ -87,6 +90,7 @@ type BaseDendrite struct {
 	Database               *sql.DB
 	DatabaseWriter         sqlutil.Writer
 	EnableMetrics          bool
+	startupLock            sync.Mutex
 }
 
 const NoListener = ""
@@ -394,6 +398,9 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	internalHTTPAddr, externalHTTPAddr config.HTTPAddress,
 	certFile, keyFile *string,
 ) {
+	// Manually unlocked right before actually serving requests,
+	// as we don't return from this method (defer doesn't work).
+	b.startupLock.Lock()
 	internalAddr, _ := internalHTTPAddr.Address()
 	externalAddr, _ := externalHTTPAddr.Address()
 
@@ -472,6 +479,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 	externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux)
 	externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(b.PublicWellKnownAPIMux)
 
+	b.startupLock.Unlock()
 	if internalAddr != NoListener && internalAddr != externalAddr {
 		go func() {
 			var internalShutdown atomic.Bool // RegisterOnShutdown can be called more than once

From 7313f56f44f6820a5573aad79faaa6037703371f Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 30 Aug 2022 14:18:47 +0200
Subject: [PATCH 309/391] Use existing limit instead of default limit when lazy
 loading members (#2682)

This should fix an issue where we return less than the expected membership events, when doing an initial sync.
When doing an initial sync, the state limit is set to `math.MaxInt32`, while the default filter is set to 20.
---
 syncapi/streams/stream_pdu.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index fa4c722ce..ffcf64df6 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -303,7 +303,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 
 	if stateFilter.LazyLoadMembers {
 		delta.StateEvents, err = p.lazyLoadMembers(
-			ctx, delta.RoomID, true, limited, stateFilter.IncludeRedundantMembers,
+			ctx, delta.RoomID, true, limited, stateFilter,
 			device, recentEvents, delta.StateEvents,
 		)
 		if err != nil && err != sql.ErrNoRows {
@@ -532,7 +532,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 			return nil, err
 		}
 		stateEvents, err = p.lazyLoadMembers(ctx, roomID,
-			false, limited, stateFilter.IncludeRedundantMembers,
+			false, limited, stateFilter,
 			device, recentEvents, stateEvents,
 		)
 		if err != nil && err != sql.ErrNoRows {
@@ -551,7 +551,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 
 func (p *PDUStreamProvider) lazyLoadMembers(
 	ctx context.Context, roomID string,
-	incremental, limited, includeRedundant bool,
+	incremental, limited bool, stateFilter *gomatrixserverlib.StateFilter,
 	device *userapi.Device,
 	timelineEvents, stateEvents []*gomatrixserverlib.HeaderedEvent,
 ) ([]*gomatrixserverlib.HeaderedEvent, error) {
@@ -581,7 +581,7 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 			stateKey := *event.StateKey()
 			if _, ok := timelineUsers[stateKey]; ok || isGappedIncremental {
 				newStateEvents = append(newStateEvents, event)
-				if !includeRedundant {
+				if !stateFilter.IncludeRedundantMembers {
 					p.lazyLoadCache.StoreLazyLoadedUser(device, roomID, stateKey, event.EventID())
 				}
 				delete(timelineUsers, stateKey)
@@ -596,6 +596,7 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 	}
 	// Query missing membership events
 	filter := gomatrixserverlib.DefaultStateFilter()
+	filter.Limit = stateFilter.Limit
 	filter.Senders = &wantUsers
 	filter.Types = &[]string{gomatrixserverlib.MRoomMember}
 	memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &filter)

From 93a6e2f4d30d72c015c7086b97322f0b106ab061 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 30 Aug 2022 14:54:57 +0200
Subject: [PATCH 310/391] Set the notification count when sending push
 notifications (#2683)

---
 userapi/consumers/syncapi_streamevent.go | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/userapi/consumers/syncapi_streamevent.go b/userapi/consumers/syncapi_streamevent.go
index 7807c7637..ec351ef7e 100644
--- a/userapi/consumers/syncapi_streamevent.go
+++ b/userapi/consumers/syncapi_streamevent.go
@@ -7,6 +7,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/pushgateway"
 	"github.com/matrix-org/dendrite/internal/pushrules"
@@ -20,9 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 	"github.com/matrix-org/dendrite/userapi/util"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	log "github.com/sirupsen/logrus"
 )
 
 type OutputStreamEventConsumer struct {
@@ -529,7 +530,9 @@ func (s *OutputStreamEventConsumer) notifyHTTP(ctx context.Context, event *gomat
 	case "event_id_only":
 		req = pushgateway.NotifyRequest{
 			Notification: pushgateway.Notification{
-				Counts:  &pushgateway.Counts{},
+				Counts: &pushgateway.Counts{
+					Unread: userNumUnreadNotifs,
+				},
 				Devices: devices,
 				EventID: event.EventID(),
 				RoomID:  event.RoomID(),

From 2be43560ca7d81601e6763c9fcb8b72ecd74009b Mon Sep 17 00:00:00 2001
From: PiotrKozimor <37144818+PiotrKozimor@users.noreply.github.com>
Date: Tue, 30 Aug 2022 15:47:54 +0200
Subject: [PATCH 311/391] Index on `syncapi_send_to_device` table (#2684)

Introduced index improves select query performance. Example execution time of `selectSendToDeviceMessagesSQL` query dropped from 80 ms to 15 ms. No sytest modifications are required.

### Pull Request Checklist

* [x] I have added added tests for PR _or_ I have justified why this PR doesn't need tests.
* [x] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)

Signed-off-by: `Piotr Kozimor <p1996k@gmail.com>`
---
 syncapi/storage/postgres/send_to_device_table.go | 2 ++
 syncapi/storage/sqlite3/send_to_device_table.go  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go
index fd0c1c56b..6ab1f0f48 100644
--- a/syncapi/storage/postgres/send_to_device_table.go
+++ b/syncapi/storage/postgres/send_to_device_table.go
@@ -41,6 +41,8 @@ CREATE TABLE IF NOT EXISTS syncapi_send_to_device (
 	-- The event content JSON.
 	content TEXT NOT NULL
 );
+
+CREATE INDEX IF NOT EXISTS syncapi_send_to_device_user_id_device_id_idx ON syncapi_send_to_device(user_id, device_id);
 `
 
 const insertSendToDeviceMessageSQL = `
diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go
index e3aa1b7a1..0da06506c 100644
--- a/syncapi/storage/sqlite3/send_to_device_table.go
+++ b/syncapi/storage/sqlite3/send_to_device_table.go
@@ -39,6 +39,8 @@ CREATE TABLE IF NOT EXISTS syncapi_send_to_device (
 	-- The event content JSON.
 	content TEXT NOT NULL
 );
+
+CREATE INDEX IF NOT EXISTS syncapi_send_to_device_user_id_device_id_idx ON syncapi_send_to_device(user_id, device_id);
 `
 
 const insertSendToDeviceMessageSQL = `

From 02ec00b1bbeb43f68a4f9b1f19ee6282e4a34194 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 30 Aug 2022 17:40:46 +0100
Subject: [PATCH 312/391] Check for `"creator"` field in `m.room.create` during
 event auth (update to matrix-org/gomatrixserverlib@c71e518)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 8c334f1e8..8bf8f454d 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220830104348-2020bdc55859
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2
 	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index aac86010b..8b8baabcf 100644
--- a/go.sum
+++ b/go.sum
@@ -343,8 +343,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220830104348-2020bdc55859 h1:14mOIE2sGS6YTG1+3GFbVWmAisW5pzyfn3SxJTjkR2I=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220830104348-2020bdc55859/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 h1:esbNn9hg//tAStA6TogatAJAursw23A+yfVRQsdiv70=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
 github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From ba0b3adab4de7865afd467b61638437b1af39fce Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 31 Aug 2022 10:41:32 +0100
Subject: [PATCH 313/391] Pinecone standalone refactoring (#2685)

This refactors the `dendrite-demo-pinecone` executable so that it:

1. Converts the old `.key` file into a standard `.pem` file
2. Allows passing in the `--config` option to supply a normal Dendrite configuration file, so that you can configure PostgreSQL instead of SQLite, appservices and all the other usual stuff
---
 cmd/dendrite-demo-pinecone/main.go | 110 +++++++++++++++++++----------
 setup/config/config.go             |  15 ++--
 test/keys.go                       |   7 +-
 3 files changed, 86 insertions(+), 46 deletions(-)

diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 75f29fe27..b16cfec6a 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -42,6 +43,7 @@ import (
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -70,31 +72,84 @@ func main() {
 	var pk ed25519.PublicKey
 	var sk ed25519.PrivateKey
 
-	keyfile := *instanceName + ".key"
-	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
-		if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
-			panic(err)
+	// iterate through the cli args and check if the config flag was set
+	configFlagSet := false
+	for _, arg := range os.Args {
+		if arg == "--config" || arg == "-config" {
+			configFlagSet = true
+			break
 		}
-		if err = os.WriteFile(keyfile, sk, 0644); err != nil {
-			panic(err)
-		}
-	} else if err == nil {
-		if sk, err = os.ReadFile(keyfile); err != nil {
-			panic(err)
-		}
-		if len(sk) != ed25519.PrivateKeySize {
-			panic("the private key is not long enough")
-		}
-		pk = sk.Public().(ed25519.PublicKey)
 	}
 
+	cfg := &config.Dendrite{}
+
+	// use custom config if config flag is set
+	if configFlagSet {
+		cfg = setup.ParseFlags(true)
+		sk = cfg.Global.PrivateKey
+	} else {
+		keyfile := *instanceName + ".pem"
+		if _, err := os.Stat(keyfile); os.IsNotExist(err) {
+			oldkeyfile := *instanceName + ".key"
+			if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
+				if err = test.NewMatrixKey(keyfile); err != nil {
+					panic("failed to generate a new PEM key: " + err.Error())
+				}
+				if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+					panic("failed to load PEM key: " + err.Error())
+				}
+			} else {
+				if sk, err = os.ReadFile(oldkeyfile); err != nil {
+					panic("failed to read the old private key: " + err.Error())
+				}
+				if len(sk) != ed25519.PrivateKeySize {
+					panic("the private key is not long enough")
+				}
+				if err := test.SaveMatrixKey(keyfile, sk); err != nil {
+					panic("failed to convert the private key to PEM format: " + err.Error())
+				}
+			}
+		} else {
+			var err error
+			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+				panic("failed to load PEM key: " + err.Error())
+			}
+		}
+		cfg.Defaults(true)
+		cfg.Global.PrivateKey = sk
+		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
+		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
+		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
+		cfg.ClientAPI.RegistrationDisabled = false
+		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+		if err := cfg.Derive(); err != nil {
+			panic(err)
+		}
+	}
+
+	pk = sk.Public().(ed25519.PublicKey)
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+
+	base := base.NewBaseDendrite(cfg, "Monolith")
+	defer base.Close() // nolint: errcheck
+
 	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
 	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
 	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
 	pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
 	pMulticast.Start()
 	if instancePeer != nil && *instancePeer != "" {
-		pManager.AddPeer(*instancePeer)
+		for _, peer := range strings.Split(*instancePeer, ",") {
+			pManager.AddPeer(strings.Trim(peer, " \t\r\n"))
+		}
 	}
 
 	go func() {
@@ -125,29 +180,6 @@ func main() {
 		}
 	}()
 
-	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
-	cfg.Global.PrivateKey = sk
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
-	cfg.ClientAPI.RegistrationDisabled = false
-	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-	if err := cfg.Derive(); err != nil {
-		panic(err)
-	}
-
-	base := base.NewBaseDendrite(cfg, "Monolith")
-	defer base.Close() // nolint: errcheck
-
 	federation := conn.CreateFederationClient(base, pQUIC)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
diff --git a/setup/config/config.go b/setup/config/config.go
index 924b51f22..cc9c04470 100644
--- a/setup/config/config.go
+++ b/setup/config/config.go
@@ -224,12 +224,7 @@ func loadConfig(
 	}
 
 	privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath)
-	privateKeyData, err := readFile(privateKeyPath)
-	if err != nil {
-		return nil, err
-	}
-
-	if c.Global.KeyID, c.Global.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData, true); err != nil {
+	if c.Global.KeyID, c.Global.PrivateKey, err = LoadMatrixKey(privateKeyPath, readFile); err != nil {
 		return nil, err
 	}
 
@@ -265,6 +260,14 @@ func loadConfig(
 	return &c, nil
 }
 
+func LoadMatrixKey(privateKeyPath string, readFile func(string) ([]byte, error)) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) {
+	privateKeyData, err := readFile(privateKeyPath)
+	if err != nil {
+		return "", nil, err
+	}
+	return readKeyPEM(privateKeyPath, privateKeyData, true)
+}
+
 // Derive generates data that is derived from various values provided in
 // the config file.
 func (config *Dendrite) Derive() error {
diff --git a/test/keys.go b/test/keys.go
index fb156ef27..05f7317cf 100644
--- a/test/keys.go
+++ b/test/keys.go
@@ -15,6 +15,7 @@
 package test
 
 import (
+	"crypto/ed25519"
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
@@ -44,6 +45,10 @@ func NewMatrixKey(matrixKeyPath string) (err error) {
 	if err != nil {
 		return err
 	}
+	return SaveMatrixKey(matrixKeyPath, data[3:])
+}
+
+func SaveMatrixKey(matrixKeyPath string, data ed25519.PrivateKey) error {
 	keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return err
@@ -62,7 +67,7 @@ func NewMatrixKey(matrixKeyPath string) (err error) {
 		Headers: map[string]string{
 			"Key-ID": fmt.Sprintf("ed25519:%s", keyID[:6]),
 		},
-		Bytes: data[3:],
+		Bytes: data,
 	})
 	return err
 }

From 175f65407a7f684753334022e66b8209f3db7396 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 31 Aug 2022 12:21:56 +0100
Subject: [PATCH 314/391] Allow batching in `JetStreamConsumer` (#2686)

This allows us to receive more than one message from NATS at a time if we want.
---
 appservice/consumers/roomserver.go       |  7 ++++---
 federationapi/consumers/keychange.go     |  7 ++++---
 federationapi/consumers/presence.go      |  5 +++--
 federationapi/consumers/receipts.go      |  5 +++--
 federationapi/consumers/roomserver.go    |  7 ++++---
 federationapi/consumers/sendtodevice.go  |  7 ++++---
 federationapi/consumers/typing.go        |  5 +++--
 keyserver/consumers/devicelistupdate.go  |  7 ++++---
 setup/jetstream/helpers.go               | 25 +++++++++++++++++++-----
 syncapi/consumers/clientapi.go           |  7 ++++---
 syncapi/consumers/keychange.go           |  7 ++++---
 syncapi/consumers/presence.go            |  5 +++--
 syncapi/consumers/receipts.go            |  7 ++++---
 syncapi/consumers/roomserver.go          |  7 ++++---
 syncapi/consumers/sendtodevice.go        |  7 ++++---
 syncapi/consumers/typing.go              |  7 ++++---
 syncapi/consumers/userapi.go             |  7 ++++---
 userapi/consumers/syncapi_readupdate.go  |  7 ++++---
 userapi/consumers/syncapi_streamevent.go |  7 ++++---
 19 files changed, 88 insertions(+), 55 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index d567408be..21b52bc3c 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -68,14 +68,15 @@ func NewOutputRoomEventConsumer(
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
 // onMessage is called when the appservice component receives a new event from
 // the room server output log.
-func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// Parse out the event JSON
 	var output api.OutputEvent
 	if err := json.Unmarshal(msg.Data, &output); err != nil {
diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go
index 6d3cf0e46..f3314bc98 100644
--- a/federationapi/consumers/keychange.go
+++ b/federationapi/consumers/keychange.go
@@ -67,14 +67,15 @@ func NewKeyChangeConsumer(
 // Start consuming from key servers
 func (t *KeyChangeConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		t.ctx, t.jetstream, t.topic, t.durable, 1,
+		t.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
 // onMessage is called in response to a message received on the
 // key change events topic from the key server.
-func (t *KeyChangeConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (t *KeyChangeConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	var m api.DeviceMessage
 	if err := json.Unmarshal(msg.Data, &m); err != nil {
 		logrus.WithError(err).Errorf("failed to read device message from key change topic")
diff --git a/federationapi/consumers/presence.go b/federationapi/consumers/presence.go
index a65d2aa04..e76103cd3 100644
--- a/federationapi/consumers/presence.go
+++ b/federationapi/consumers/presence.go
@@ -69,14 +69,15 @@ func (t *OutputPresenceConsumer) Start() error {
 		return nil
 	}
 	return jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		t.ctx, t.jetstream, t.topic, t.durable, 1, t.onMessage,
 		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
 	)
 }
 
 // onMessage is called in response to a message received on the presence
 // events topic from the client api.
-func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// only send presence events which originated from us
 	userID := msg.Header.Get(jetstream.UserID)
 	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go
index 2c9d79bcb..366cb264e 100644
--- a/federationapi/consumers/receipts.go
+++ b/federationapi/consumers/receipts.go
@@ -65,14 +65,15 @@ func NewOutputReceiptConsumer(
 // Start consuming from the clientapi
 func (t *OutputReceiptConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		t.ctx, t.jetstream, t.topic, t.durable, 1, t.onMessage,
 		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
 	)
 }
 
 // onMessage is called in response to a message received on the receipt
 // events topic from the client api.
-func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	receipt := syncTypes.OutputReceiptEvent{
 		UserID:  msg.Header.Get(jetstream.UserID),
 		RoomID:  msg.Header.Get(jetstream.RoomID),
diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index 2622ecb3f..349b50b05 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -68,8 +68,8 @@ func NewOutputRoomEventConsumer(
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
@@ -77,7 +77,8 @@ func (s *OutputRoomEventConsumer) Start() error {
 // It is unsafe to call this with messages for the same room in multiple gorountines
 // because updates it will likely fail with a types.EventIDMismatchError when it
 // realises that it cannot update the room state using the deltas.
-func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// Parse out the event JSON
 	var output api.OutputEvent
 	if err := json.Unmarshal(msg.Data, &output); err != nil {
diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go
index f99a895e0..e44bad723 100644
--- a/federationapi/consumers/sendtodevice.go
+++ b/federationapi/consumers/sendtodevice.go
@@ -63,14 +63,15 @@ func NewOutputSendToDeviceConsumer(
 // Start consuming from the client api
 func (t *OutputSendToDeviceConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		t.ctx, t.jetstream, t.topic, t.durable, 1,
+		t.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
 // onMessage is called in response to a message received on the
 // send-to-device events topic from the client api.
-func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// only send send-to-device events which originated from us
 	sender := msg.Header.Get("sender")
 	_, originServerName, err := gomatrixserverlib.SplitID('@', sender)
diff --git a/federationapi/consumers/typing.go b/federationapi/consumers/typing.go
index 428e1a867..9c7379136 100644
--- a/federationapi/consumers/typing.go
+++ b/federationapi/consumers/typing.go
@@ -62,14 +62,15 @@ func NewOutputTypingConsumer(
 // Start consuming from the clientapi
 func (t *OutputTypingConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		t.ctx, t.jetstream, t.topic, t.durable, 1, t.onMessage,
 		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
 	)
 }
 
 // onMessage is called in response to a message received on the typing
 // events topic from the client api.
-func (t *OutputTypingConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (t *OutputTypingConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// Extract the typing event from msg.
 	roomID := msg.Header.Get(jetstream.RoomID)
 	userID := msg.Header.Get(jetstream.UserID)
diff --git a/keyserver/consumers/devicelistupdate.go b/keyserver/consumers/devicelistupdate.go
index f4f246280..d15f94267 100644
--- a/keyserver/consumers/devicelistupdate.go
+++ b/keyserver/consumers/devicelistupdate.go
@@ -55,14 +55,15 @@ func NewDeviceListUpdateConsumer(
 // Start consuming from key servers
 func (t *DeviceListUpdateConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		t.ctx, t.jetstream, t.topic, t.durable, 1,
+		t.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
 // onMessage is called in response to a message received on the
 // key change events topic from the key server.
-func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	var m gomatrixserverlib.DeviceListUpdateEvent
 	if err := json.Unmarshal(msg.Data, &m); err != nil {
 		logrus.WithError(err).Errorf("Failed to read from device list update input topic")
diff --git a/setup/jetstream/helpers.go b/setup/jetstream/helpers.go
index 1c07583e9..f47637c69 100644
--- a/setup/jetstream/helpers.go
+++ b/setup/jetstream/helpers.go
@@ -9,9 +9,16 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// JetStreamConsumer starts a durable consumer on the given subject with the
+// given durable name. The function will be called when one or more messages
+// is available, up to the maximum batch size specified. If the batch is set to
+// 1 then messages will be delivered one at a time. If the function is called,
+// the messages array is guaranteed to be at least 1 in size. Any provided NATS
+// options will be passed through to the pull subscriber creation. The consumer
+// will continue to run until the context expires, at which point it will stop.
 func JetStreamConsumer(
-	ctx context.Context, js nats.JetStreamContext, subj, durable string,
-	f func(ctx context.Context, msg *nats.Msg) bool,
+	ctx context.Context, js nats.JetStreamContext, subj, durable string, batch int,
+	f func(ctx context.Context, msgs []*nats.Msg) bool,
 	opts ...nats.SubOpt,
 ) error {
 	defer func() {
@@ -27,6 +34,14 @@ func JetStreamConsumer(
 		}
 	}()
 
+	// If the batch size is greater than 1, we will want to acknowledge all
+	// received messages in the batch. Below we will send an acknowledgement
+	// for the most recent message in the batch and AckAll will ensure that
+	// all messages that came before it are also acknowledged implicitly.
+	if batch > 1 {
+		opts = append(opts, nats.AckAll())
+	}
+
 	name := durable + "Pull"
 	sub, err := js.PullSubscribe(subj, name, opts...)
 	if err != nil {
@@ -50,7 +65,7 @@ func JetStreamConsumer(
 			// enforce its own deadline (roughly 5 seconds by default). Therefore
 			// it is our responsibility to check whether our context expired or
 			// not when a context error is returned. Footguns. Footguns everywhere.
-			msgs, err := sub.Fetch(1, nats.Context(ctx))
+			msgs, err := sub.Fetch(batch, nats.Context(ctx))
 			if err != nil {
 				if err == context.Canceled || err == context.DeadlineExceeded {
 					// Work out whether it was the JetStream context that expired
@@ -74,13 +89,13 @@ func JetStreamConsumer(
 			if len(msgs) < 1 {
 				continue
 			}
-			msg := msgs[0]
+			msg := msgs[len(msgs)-1] // most recent message, in case of AckAll
 			if err = msg.InProgress(nats.Context(ctx)); err != nil {
 				logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err))
 				sentry.CaptureException(err)
 				continue
 			}
-			if f(ctx, msg) {
+			if f(ctx, msgs) {
 				if err = msg.AckSync(nats.Context(ctx)); err != nil {
 					logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err))
 					sentry.CaptureException(err)
diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go
index 02633b567..f0588cab8 100644
--- a/syncapi/consumers/clientapi.go
+++ b/syncapi/consumers/clientapi.go
@@ -75,15 +75,16 @@ func NewOutputClientDataConsumer(
 // Start consuming from room servers
 func (s *OutputClientDataConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
 // onMessage is called when the sync server receives a new event from the client API server output log.
 // It is not safe for this function to be called from multiple goroutines, or else the
 // sync stream position may race and be incorrectly calculated.
-func (s *OutputClientDataConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputClientDataConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// Parse out the event JSON
 	userID := msg.Header.Get(jetstream.UserID)
 	var output eventutil.AccountData
diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go
index c8d88ddac..c42e71971 100644
--- a/syncapi/consumers/keychange.go
+++ b/syncapi/consumers/keychange.go
@@ -75,12 +75,13 @@ func NewOutputKeyChangeEventConsumer(
 // Start consuming from the key server
 func (s *OutputKeyChangeEventConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
-func (s *OutputKeyChangeEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputKeyChangeEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	var m api.DeviceMessage
 	if err := json.Unmarshal(msg.Data, &m); err != nil {
 		logrus.WithError(err).Errorf("failed to read device message from key change topic")
diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go
index db7d67fa6..61bdc13de 100644
--- a/syncapi/consumers/presence.go
+++ b/syncapi/consumers/presence.go
@@ -128,12 +128,13 @@ func (s *PresenceConsumer) Start() error {
 		return nil
 	}
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.presenceTopic, s.durable, s.onMessage,
+		s.ctx, s.jetstream, s.presenceTopic, s.durable, 1, s.onMessage,
 		nats.DeliverAll(), nats.ManualAck(), nats.HeadersOnly(),
 	)
 }
 
-func (s *PresenceConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *PresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	userID := msg.Header.Get(jetstream.UserID)
 	presence := msg.Header.Get("presence")
 	timestamp := msg.Header.Get("last_active_ts")
diff --git a/syncapi/consumers/receipts.go b/syncapi/consumers/receipts.go
index 83156cf93..a18244c44 100644
--- a/syncapi/consumers/receipts.go
+++ b/syncapi/consumers/receipts.go
@@ -74,12 +74,13 @@ func NewOutputReceiptEventConsumer(
 // Start consuming receipts events.
 func (s *OutputReceiptEventConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
-func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	output := types.OutputReceiptEvent{
 		UserID:  msg.Header.Get(jetstream.UserID),
 		RoomID:  msg.Header.Get(jetstream.RoomID),
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index f77b1673b..6979eb484 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -79,15 +79,16 @@ func NewOutputRoomEventConsumer(
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
 // onMessage is called when the sync server receives a new event from the room server output log.
 // It is not safe for this function to be called from multiple goroutines, or else the
 // sync stream position may race and be incorrectly calculated.
-func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	// Parse out the event JSON
 	var err error
 	var output api.OutputEvent
diff --git a/syncapi/consumers/sendtodevice.go b/syncapi/consumers/sendtodevice.go
index 0b9153fcd..89b01d7e5 100644
--- a/syncapi/consumers/sendtodevice.go
+++ b/syncapi/consumers/sendtodevice.go
@@ -68,12 +68,13 @@ func NewOutputSendToDeviceEventConsumer(
 // Start consuming send-to-device events.
 func (s *OutputSendToDeviceEventConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
-func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	userID := msg.Header.Get(jetstream.UserID)
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
diff --git a/syncapi/consumers/typing.go b/syncapi/consumers/typing.go
index 48e484ec5..88db80f8c 100644
--- a/syncapi/consumers/typing.go
+++ b/syncapi/consumers/typing.go
@@ -64,12 +64,13 @@ func NewOutputTypingEventConsumer(
 // Start consuming typing events.
 func (s *OutputTypingEventConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
-func (s *OutputTypingEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputTypingEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	roomID := msg.Header.Get(jetstream.RoomID)
 	userID := msg.Header.Get(jetstream.UserID)
 	typing, err := strconv.ParseBool(msg.Header.Get("typing"))
diff --git a/syncapi/consumers/userapi.go b/syncapi/consumers/userapi.go
index 010fa7c8e..227823522 100644
--- a/syncapi/consumers/userapi.go
+++ b/syncapi/consumers/userapi.go
@@ -67,8 +67,8 @@ func NewOutputNotificationDataConsumer(
 // Start starts consumption.
 func (s *OutputNotificationDataConsumer) Start() error {
 	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	)
 }
 
@@ -76,7 +76,8 @@ func (s *OutputNotificationDataConsumer) Start() error {
 // the push server. It is not safe for this function to be called from
 // multiple goroutines, or else the sync stream position may race and
 // be incorrectly calculated.
-func (s *OutputNotificationDataConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputNotificationDataConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	userID := string(msg.Header.Get(jetstream.UserID))
 
 	// Parse out the event JSON
diff --git a/userapi/consumers/syncapi_readupdate.go b/userapi/consumers/syncapi_readupdate.go
index 067f93330..54654f757 100644
--- a/userapi/consumers/syncapi_readupdate.go
+++ b/userapi/consumers/syncapi_readupdate.go
@@ -56,15 +56,16 @@ func NewOutputReadUpdateConsumer(
 
 func (s *OutputReadUpdateConsumer) Start() error {
 	if err := jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (s *OutputReadUpdateConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputReadUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	var read types.ReadUpdate
 	if err := json.Unmarshal(msg.Data, &read); err != nil {
 		log.WithError(err).Error("userapi clientapi consumer: message parse failure")
diff --git a/userapi/consumers/syncapi_streamevent.go b/userapi/consumers/syncapi_streamevent.go
index ec351ef7e..3ac6f58d0 100644
--- a/userapi/consumers/syncapi_streamevent.go
+++ b/userapi/consumers/syncapi_streamevent.go
@@ -65,15 +65,16 @@ func NewOutputStreamEventConsumer(
 
 func (s *OutputStreamEventConsumer) Start() error {
 	if err := jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
-		nats.DeliverAll(), nats.ManualAck(),
+		s.ctx, s.jetstream, s.topic, s.durable, 1,
+		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
 	); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (s *OutputStreamEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+func (s *OutputStreamEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	var output types.StreamedEvent
 	output.Event = &gomatrixserverlib.HeaderedEvent{}
 	if err := json.Unmarshal(msg.Data, &output); err != nil {

From ad6b902b8462adb568d799c69a74b60d69574d0c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 1 Sep 2022 09:20:40 +0100
Subject: [PATCH 315/391] Refactor appservices component (#2687)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR refactors the app services component. It makes the following changes:

* Each appservice now gets its own NATS JetStream consumer
* The appservice database is now removed entirely, since we just use JetStream as a data source instead
* The entire component is now much simpler and we deleted lots of lines of code 💅

The result is that it should be much lighter and hopefully much more performant.
---
 appservice/README.md                          |  10 -
 appservice/appservice.go                      |  55 +--
 appservice/consumers/roomserver.go            | 330 +++++++++++-------
 appservice/query/query.go                     |   2 +-
 appservice/storage/interface.go               |  30 --
 .../postgres/appservice_events_table.go       | 256 --------------
 appservice/storage/postgres/storage.go        | 115 ------
 .../storage/postgres/txn_id_counter_table.go  |  53 ---
 .../sqlite3/appservice_events_table.go        | 267 --------------
 appservice/storage/sqlite3/storage.go         | 114 ------
 .../storage/sqlite3/txn_id_counter_table.go   |  82 -----
 appservice/storage/storage.go                 |  40 ---
 appservice/storage/storage_wasm.go            |  34 --
 appservice/types/types.go                     |  64 ----
 appservice/workers/transaction_scheduler.go   | 236 -------------
 build/gobind-pinecone/monolith.go             |   1 -
 build/gobind-yggdrasil/monolith.go            |   1 -
 cmd/dendrite-demo-pinecone/main.go            |   1 -
 cmd/dendrite-demo-yggdrasil/main.go           |   1 -
 cmd/generate-config/main.go                   |   1 -
 dendrite-sample.polylith.yaml                 |   7 -
 setup/config/config_appservice.go             |   9 -
 setup/jetstream/helpers.go                    |  35 +-
 setup/jetstream/nats.go                       |   1 +
 test/testrig/base.go                          |   1 -
 userapi/internal/api.go                       |   3 +-
 26 files changed, 231 insertions(+), 1518 deletions(-)
 delete mode 100644 appservice/README.md
 delete mode 100644 appservice/storage/interface.go
 delete mode 100644 appservice/storage/postgres/appservice_events_table.go
 delete mode 100644 appservice/storage/postgres/storage.go
 delete mode 100644 appservice/storage/postgres/txn_id_counter_table.go
 delete mode 100644 appservice/storage/sqlite3/appservice_events_table.go
 delete mode 100644 appservice/storage/sqlite3/storage.go
 delete mode 100644 appservice/storage/sqlite3/txn_id_counter_table.go
 delete mode 100644 appservice/storage/storage.go
 delete mode 100644 appservice/storage/storage_wasm.go
 delete mode 100644 appservice/types/types.go
 delete mode 100644 appservice/workers/transaction_scheduler.go

diff --git a/appservice/README.md b/appservice/README.md
deleted file mode 100644
index d75557448..000000000
--- a/appservice/README.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Application Service
-
-This component interfaces with external [Application
-Services](https://matrix.org/docs/spec/application_service/unstable.html).
-This includes any HTTP endpoints that application services call, as well as talking
-to any HTTP endpoints that application services provide themselves.
-
-## Consumers
-
-This component consumes and filters events from the Roomserver Kafka stream, passing on any necessary events to subscribing application services.
\ No newline at end of file
diff --git a/appservice/appservice.go b/appservice/appservice.go
index 8fe1b2fc4..9000adb1d 100644
--- a/appservice/appservice.go
+++ b/appservice/appservice.go
@@ -18,7 +18,6 @@ import (
 	"context"
 	"crypto/tls"
 	"net/http"
-	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -28,9 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/inthttp"
 	"github.com/matrix-org/dendrite/appservice/query"
-	"github.com/matrix-org/dendrite/appservice/storage"
-	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/appservice/workers"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -59,57 +55,40 @@ func NewInternalAPI(
 			Proxy: http.ProxyFromEnvironment,
 		},
 	}
-	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	// Create appserivce query API with an HTTP client that will be used for all
+	// outbound and inbound requests (inbound only for the internal API)
+	appserviceQueryAPI := &query.AppServiceQueryAPI{
+		HTTPClient: client,
+		Cfg:        &base.Cfg.AppServiceAPI,
+	}
 
-	// Create a connection to the appservice postgres DB
-	appserviceDB, err := storage.NewDatabase(base, &base.Cfg.AppServiceAPI.Database)
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to appservice db")
+	if len(base.Cfg.Derived.ApplicationServices) == 0 {
+		return appserviceQueryAPI
 	}
 
 	// Wrap application services in a type that relates the application service and
 	// a sync.Cond object that can be used to notify workers when there are new
 	// events to be sent out.
-	workerStates := make([]types.ApplicationServiceWorkerState, len(base.Cfg.Derived.ApplicationServices))
-	for i, appservice := range base.Cfg.Derived.ApplicationServices {
-		m := sync.Mutex{}
-		ws := types.ApplicationServiceWorkerState{
-			AppService: appservice,
-			Cond:       sync.NewCond(&m),
-		}
-		workerStates[i] = ws
-
+	for _, appservice := range base.Cfg.Derived.ApplicationServices {
 		// Create bot account for this AS if it doesn't already exist
-		if err = generateAppServiceAccount(userAPI, appservice); err != nil {
+		if err := generateAppServiceAccount(userAPI, appservice); err != nil {
 			logrus.WithFields(logrus.Fields{
 				"appservice": appservice.ID,
 			}).WithError(err).Panicf("failed to generate bot account for appservice")
 		}
 	}
 
-	// Create appserivce query API with an HTTP client that will be used for all
-	// outbound and inbound requests (inbound only for the internal API)
-	appserviceQueryAPI := &query.AppServiceQueryAPI{
-		HTTPClient: client,
-		Cfg:        base.Cfg,
-	}
-
 	// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
 	// We can't add ASes at runtime so this is safe to do.
-	if len(workerStates) > 0 {
-		consumer := consumers.NewOutputRoomEventConsumer(
-			base.ProcessContext, base.Cfg, js, appserviceDB,
-			rsAPI, workerStates,
-		)
-		if err := consumer.Start(); err != nil {
-			logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
-		}
+	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	consumer := consumers.NewOutputRoomEventConsumer(
+		base.ProcessContext, &base.Cfg.AppServiceAPI,
+		client, js, rsAPI,
+	)
+	if err := consumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
 	}
 
-	// Create application service transaction workers
-	if err := workers.SetupTransactionWorkers(client, appserviceDB, workerStates); err != nil {
-		logrus.WithError(err).Panicf("failed to start app service transaction workers")
-	}
 	return appserviceQueryAPI
 }
 
diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index 21b52bc3c..a30944e75 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -15,14 +15,18 @@
 package consumers
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"net/url"
+	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 
-	"github.com/matrix-org/dendrite/appservice/storage"
-	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -33,178 +37,192 @@ import (
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	ctx          context.Context
-	jetstream    nats.JetStreamContext
-	durable      string
-	topic        string
-	asDB         storage.Database
-	rsAPI        api.AppserviceRoomserverAPI
-	serverName   string
-	workerStates []types.ApplicationServiceWorkerState
+	ctx       context.Context
+	cfg       *config.AppServiceAPI
+	client    *http.Client
+	jetstream nats.JetStreamContext
+	topic     string
+	rsAPI     api.AppserviceRoomserverAPI
+}
+
+type appserviceState struct {
+	*config.ApplicationService
+	backoff int
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call
 // Start() to begin consuming from room servers.
 func NewOutputRoomEventConsumer(
 	process *process.ProcessContext,
-	cfg *config.Dendrite,
+	cfg *config.AppServiceAPI,
+	client *http.Client,
 	js nats.JetStreamContext,
-	appserviceDB storage.Database,
 	rsAPI api.AppserviceRoomserverAPI,
-	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
-		ctx:          process.Context(),
-		jetstream:    js,
-		durable:      cfg.Global.JetStream.Durable("AppserviceRoomserverConsumer"),
-		topic:        cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
-		asDB:         appserviceDB,
-		rsAPI:        rsAPI,
-		serverName:   string(cfg.Global.ServerName),
-		workerStates: workerStates,
+		ctx:       process.Context(),
+		cfg:       cfg,
+		client:    client,
+		jetstream: js,
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
+		rsAPI:     rsAPI,
 	}
 }
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	return jetstream.JetStreamConsumer(
-		s.ctx, s.jetstream, s.topic, s.durable, 1,
-		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
-	)
+	for _, as := range s.cfg.Derived.ApplicationServices {
+		appsvc := as
+		state := &appserviceState{
+			ApplicationService: &appsvc,
+		}
+		token := jetstream.Tokenise(as.ID)
+		if err := jetstream.JetStreamConsumer(
+			s.ctx, s.jetstream, s.topic,
+			s.cfg.Matrix.JetStream.Durable("Appservice_"+token),
+			50, // maximum number of events to send in a single transaction
+			func(ctx context.Context, msgs []*nats.Msg) bool {
+				return s.onMessage(ctx, state, msgs)
+			},
+			nats.DeliverNew(), nats.ManualAck(),
+		); err != nil {
+			return fmt.Errorf("failed to create %q consumer: %w", token, err)
+		}
+	}
+	return nil
 }
 
 // onMessage is called when the appservice component receives a new event from
 // the room server output log.
-func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
-	msg := msgs[0] // Guaranteed to exist if onMessage is called
-	// Parse out the event JSON
-	var output api.OutputEvent
-	if err := json.Unmarshal(msg.Data, &output); err != nil {
-		// If the message was invalid, log it and move on to the next message in the stream
-		log.WithError(err).Errorf("roomserver output log: message parse failure")
-		return true
-	}
-
-	log.WithFields(log.Fields{
-		"type": output.Type,
-	}).Debug("Got a message in OutputRoomEventConsumer")
-
-	events := []*gomatrixserverlib.HeaderedEvent{}
-	if output.Type == api.OutputTypeNewRoomEvent && output.NewRoomEvent != nil {
-		newEventID := output.NewRoomEvent.Event.EventID()
-		events = append(events, output.NewRoomEvent.Event)
-		if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
-			eventsReq := &api.QueryEventsByIDRequest{
-				EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
-			}
-			eventsRes := &api.QueryEventsByIDResponse{}
-			for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
-				if eventID != newEventID {
-					eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
-				}
-			}
-			if len(eventsReq.EventIDs) > 0 {
-				if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
-					log.WithError(err).Errorf("s.rsAPI.QueryEventsByID failed")
-					return false
-				}
-				events = append(events, eventsRes.Events...)
-			}
+func (s *OutputRoomEventConsumer) onMessage(
+	ctx context.Context, state *appserviceState, msgs []*nats.Msg,
+) bool {
+	log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs))
+	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs))
+	for _, msg := range msgs {
+		// Parse out the event JSON
+		var output api.OutputEvent
+		if err := json.Unmarshal(msg.Data, &output); err != nil {
+			// If the message was invalid, log it and move on to the next message in the stream
+			log.WithField("appservice", state.ID).WithError(err).Errorf("Appservice failed to parse message, ignoring")
+			continue
 		}
-	} else if output.Type == api.OutputTypeNewInviteEvent && output.NewInviteEvent != nil {
-		events = append(events, output.NewInviteEvent.Event)
-	} else {
-		log.WithFields(log.Fields{
-			"type": output.Type,
-		}).Debug("appservice OutputRoomEventConsumer ignoring event", string(msg.Data))
+		switch output.Type {
+		case api.OutputTypeNewRoomEvent:
+			if output.NewRoomEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewRoomEvent.Event, state.ApplicationService) {
+				continue
+			}
+			events = append(events, output.NewRoomEvent.Event)
+			if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
+				newEventID := output.NewRoomEvent.Event.EventID()
+				eventsReq := &api.QueryEventsByIDRequest{
+					EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
+				}
+				eventsRes := &api.QueryEventsByIDResponse{}
+				for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
+					if eventID != newEventID {
+						eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
+					}
+				}
+				if len(eventsReq.EventIDs) > 0 {
+					if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
+						log.WithError(err).Errorf("s.rsAPI.QueryEventsByID failed")
+						return false
+					}
+					events = append(events, eventsRes.Events...)
+				}
+			}
+
+		case api.OutputTypeNewInviteEvent:
+			if output.NewInviteEvent == nil {
+				continue
+			}
+			events = append(events, output.NewInviteEvent.Event)
+
+		default:
+			continue
+		}
+	}
+
+	// If there are no events selected for sending then we should
+	// ack the messages so that we don't get sent them again in the
+	// future.
+	if len(events) == 0 {
 		return true
 	}
 
-	// Send event to any relevant application services
-	if err := s.filterRoomserverEvents(context.TODO(), events); err != nil {
-		log.WithError(err).Errorf("roomserver output log: filter error")
-		return true
-	}
-
-	return true
+	// Send event to any relevant application services. If we hit
+	// an error here, return false, so that we negatively ack.
+	log.WithField("appservice", state.ID).Debugf("Appservice worker sending %d events(s) from roomserver", len(events))
+	return s.sendEvents(ctx, state, events) == nil
 }
 
-// filterRoomserverEvents takes in events and decides whether any of them need
-// to be passed on to an external application service. It does this by checking
-// each namespace of each registered application service, and if there is a
-// match, adds the event to the queue for events to be sent to a particular
-// application service.
-func (s *OutputRoomEventConsumer) filterRoomserverEvents(
-	ctx context.Context,
+// sendEvents passes events to the appservice by using the transactions
+// endpoint. It will block for the backoff period if necessary.
+func (s *OutputRoomEventConsumer) sendEvents(
+	ctx context.Context, state *appserviceState,
 	events []*gomatrixserverlib.HeaderedEvent,
 ) error {
-	for _, ws := range s.workerStates {
-		for _, event := range events {
-			// Check if this event is interesting to this application service
-			if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) {
-				// Queue this event to be sent off to the application service
-				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, event); err != nil {
-					log.WithError(err).Warn("failed to insert incoming event into appservices database")
-					return err
-				} else {
-					// Tell our worker to send out new messages by updating remaining message
-					// count and waking them up with a broadcast
-					ws.NotifyNewEvents()
-				}
-			}
-		}
+	// Create the transaction body.
+	transaction, err := json.Marshal(
+		gomatrixserverlib.ApplicationServiceTransaction{
+			Events: gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll),
+		},
+	)
+	if err != nil {
+		return err
 	}
 
+	// TODO: We should probably be more intelligent and pick something not
+	// in the control of the event. A NATS timestamp header or something maybe.
+	txnID := events[0].Event.OriginServerTS()
+
+	// Send the transaction to the appservice.
+	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
+	address := fmt.Sprintf("%s/transactions/%d?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken))
+	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := s.client.Do(req)
+	if err != nil {
+		return state.backoffAndPause(err)
+	}
+
+	// If the response was fine then we can clear any backoffs in place and
+	// report that everything was OK. Otherwise, back off for a while.
+	switch resp.StatusCode {
+	case http.StatusOK:
+		state.backoff = 0
+	default:
+		return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice", resp.StatusCode))
+	}
 	return nil
 }
 
-// appserviceJoinedAtEvent returns a boolean depending on whether a given
-// appservice has membership at the time a given event was created.
-func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
-	// TODO: This is only checking the current room state, not the state at
-	// the event in question. Pretty sure this is what Synapse does too, but
-	// until we have a lighter way of checking the state before the event that
-	// doesn't involve state res, then this is probably OK.
-	membershipReq := &api.QueryMembershipsForRoomRequest{
-		RoomID:     event.RoomID(),
-		JoinedOnly: true,
+// backoff pauses the calling goroutine for a 2^some backoff exponent seconds
+func (s *appserviceState) backoffAndPause(err error) error {
+	if s.backoff < 6 {
+		s.backoff++
 	}
-	membershipRes := &api.QueryMembershipsForRoomResponse{}
-
-	// XXX: This could potentially race if the state for the event is not known yet
-	// e.g. the event came over federation but we do not have the full state persisted.
-	if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil {
-		for _, ev := range membershipRes.JoinEvents {
-			var membership gomatrixserverlib.MemberContent
-			if err = json.Unmarshal(ev.Content, &membership); err != nil || ev.StateKey == nil {
-				continue
-			}
-			if appservice.IsInterestedInUserID(*ev.StateKey) {
-				return true
-			}
-		}
-	} else {
-		log.WithFields(log.Fields{
-			"room_id": event.RoomID(),
-		}).WithError(err).Errorf("Unable to get membership for room")
-	}
-	return false
+	duration := time.Second * time.Duration(math.Pow(2, float64(s.backoff)))
+	log.WithField("appservice", s.ID).WithError(err).Errorf("Unable to send transaction to appservice, backing off for %s", duration.String())
+	time.Sleep(duration)
+	return err
 }
 
 // appserviceIsInterestedInEvent returns a boolean depending on whether a given
 // event falls within one of a given application service's namespaces.
 //
 // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
-func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
-	// No reason to queue events if they'll never be sent to the application
-	// service
-	if appservice.URL == "" {
+func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
+	switch {
+	case appservice.URL == "":
 		return false
-	}
-
-	// Check Room ID and Sender of the event
-	if appservice.IsInterestedInUserID(event.Sender()) ||
-		appservice.IsInterestedInRoomID(event.RoomID()) {
+	case appservice.IsInterestedInUserID(event.Sender()):
+		return true
+	case appservice.IsInterestedInRoomID(event.RoomID()):
 		return true
 	}
 
@@ -225,10 +243,52 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
 		}
 	} else {
 		log.WithFields(log.Fields{
-			"room_id": event.RoomID(),
+			"appservice": appservice.ID,
+			"room_id":    event.RoomID(),
 		}).WithError(err).Errorf("Unable to get aliases for room")
 	}
 
 	// Check if any of the members in the room match the appservice
 	return s.appserviceJoinedAtEvent(ctx, event, appservice)
 }
+
+// appserviceJoinedAtEvent returns a boolean depending on whether a given
+// appservice has membership at the time a given event was created.
+func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
+	// TODO: This is only checking the current room state, not the state at
+	// the event in question. Pretty sure this is what Synapse does too, but
+	// until we have a lighter way of checking the state before the event that
+	// doesn't involve state res, then this is probably OK.
+	membershipReq := &api.QueryMembershipsForRoomRequest{
+		RoomID:     event.RoomID(),
+		JoinedOnly: true,
+	}
+	membershipRes := &api.QueryMembershipsForRoomResponse{}
+
+	// XXX: This could potentially race if the state for the event is not known yet
+	// e.g. the event came over federation but we do not have the full state persisted.
+	if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil {
+		for _, ev := range membershipRes.JoinEvents {
+			switch {
+			case ev.StateKey == nil:
+				continue
+			case ev.Type != gomatrixserverlib.MRoomMember:
+				continue
+			}
+			var membership gomatrixserverlib.MemberContent
+			err = json.Unmarshal(ev.Content, &membership)
+			switch {
+			case err != nil:
+				continue
+			case membership.Membership == gomatrixserverlib.Join:
+				return true
+			}
+		}
+	} else {
+		log.WithFields(log.Fields{
+			"appservice": appservice.ID,
+			"room_id":    event.RoomID(),
+		}).WithError(err).Errorf("Unable to get membership for room")
+	}
+	return false
+}
diff --git a/appservice/query/query.go b/appservice/query/query.go
index dacd3caa8..53b34cb18 100644
--- a/appservice/query/query.go
+++ b/appservice/query/query.go
@@ -33,7 +33,7 @@ const userIDExistsPath = "/users/"
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
 	HTTPClient *http.Client
-	Cfg        *config.Dendrite
+	Cfg        *config.AppServiceAPI
 }
 
 // RoomAliasExists performs a request to '/room/{roomAlias}' on all known
diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go
deleted file mode 100644
index 25d35af6c..000000000
--- a/appservice/storage/interface.go
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package storage
-
-import (
-	"context"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-type Database interface {
-	StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error
-	GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error)
-	CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error)
-	UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error
-	RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error
-	GetLatestTxnID(ctx context.Context) (int, error)
-}
diff --git a/appservice/storage/postgres/appservice_events_table.go b/appservice/storage/postgres/appservice_events_table.go
deleted file mode 100644
index a95be6b8a..000000000
--- a/appservice/storage/postgres/appservice_events_table.go
+++ /dev/null
@@ -1,256 +0,0 @@
-// Copyright 2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"time"
-
-	"github.com/matrix-org/gomatrixserverlib"
-	log "github.com/sirupsen/logrus"
-)
-
-const appserviceEventsSchema = `
--- Stores events to be sent to application services
-CREATE TABLE IF NOT EXISTS appservice_events (
-	-- An auto-incrementing id unique to each event in the table
-	id BIGSERIAL NOT NULL PRIMARY KEY,
-	-- The ID of the application service the event will be sent to
-	as_id TEXT NOT NULL,
-	-- JSON representation of the event
-	headered_event_json TEXT NOT NULL,
-	-- The ID of the transaction that this event is a part of
-	txn_id BIGINT NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id);
-`
-
-const selectEventsByApplicationServiceIDSQL = "" +
-	"SELECT id, headered_event_json, txn_id " +
-	"FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC"
-
-const countEventsByApplicationServiceIDSQL = "" +
-	"SELECT COUNT(id) FROM appservice_events WHERE as_id = $1"
-
-const insertEventSQL = "" +
-	"INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " +
-	"VALUES ($1, $2, $3)"
-
-const updateTxnIDForEventsSQL = "" +
-	"UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3"
-
-const deleteEventsBeforeAndIncludingIDSQL = "" +
-	"DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2"
-
-const (
-	// A transaction ID number that no transaction should ever have. Used for
-	// checking again the default value.
-	invalidTxnID = -2
-)
-
-type eventsStatements struct {
-	selectEventsByApplicationServiceIDStmt *sql.Stmt
-	countEventsByApplicationServiceIDStmt  *sql.Stmt
-	insertEventStmt                        *sql.Stmt
-	updateTxnIDForEventsStmt               *sql.Stmt
-	deleteEventsBeforeAndIncludingIDStmt   *sql.Stmt
-}
-
-func (s *eventsStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(appserviceEventsSchema)
-	if err != nil {
-		return
-	}
-
-	if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil {
-		return
-	}
-	if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil {
-		return
-	}
-	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
-		return
-	}
-	if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil {
-		return
-	}
-	if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil {
-		return
-	}
-
-	return
-}
-
-// selectEventsByApplicationServiceID takes in an application service ID and
-// returns a slice of events that need to be sent to that application service,
-// as well as an int later used to remove these same events from the database
-// once successfully sent to an application service.
-func (s *eventsStatements) selectEventsByApplicationServiceID(
-	ctx context.Context,
-	applicationServiceID string,
-	limit int,
-) (
-	txnID, maxID int,
-	events []gomatrixserverlib.HeaderedEvent,
-	eventsRemaining bool,
-	err error,
-) {
-	defer func() {
-		if err != nil {
-			log.WithFields(log.Fields{
-				"appservice": applicationServiceID,
-			}).WithError(err).Fatalf("appservice unable to select new events to send")
-		}
-	}()
-	// Retrieve events from the database. Unsuccessfully sent events first
-	eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID)
-	if err != nil {
-		return
-	}
-	defer checkNamedErr(eventRows.Close, &err)
-	events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil
-func checkNamedErr(fn func() error, err *error) {
-	if e := fn(); e != nil && *err == nil {
-		*err = e
-	}
-}
-
-func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) {
-	// Get current time for use in calculating event age
-	nowMilli := time.Now().UnixNano() / int64(time.Millisecond)
-
-	// Iterate through each row and store event contents
-	// If txn_id changes dramatically, we've switched from collecting old events to
-	// new ones. Send back those events first.
-	lastTxnID := invalidTxnID
-	for eventsProcessed := 0; eventRows.Next(); {
-		var event gomatrixserverlib.HeaderedEvent
-		var eventJSON []byte
-		var id int
-		err = eventRows.Scan(
-			&id,
-			&eventJSON,
-			&txnID,
-		)
-		if err != nil {
-			return nil, 0, 0, false, err
-		}
-
-		// Unmarshal eventJSON
-		if err = json.Unmarshal(eventJSON, &event); err != nil {
-			return nil, 0, 0, false, err
-		}
-
-		// If txnID has changed on this event from the previous event, then we've
-		// reached the end of a transaction's events. Return only those events.
-		if lastTxnID > invalidTxnID && lastTxnID != txnID {
-			return events, maxID, lastTxnID, true, nil
-		}
-		lastTxnID = txnID
-
-		// Limit events that aren't part of an old transaction
-		if txnID == -1 {
-			// Return if we've hit the limit
-			if eventsProcessed++; eventsProcessed > limit {
-				return events, maxID, lastTxnID, true, nil
-			}
-		}
-
-		if id > maxID {
-			maxID = id
-		}
-
-		// Portion of the event that is unsigned due to rapid change
-		// TODO: Consider removing age as not many app services use it
-		if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil {
-			return nil, 0, 0, false, err
-		}
-
-		events = append(events, event)
-	}
-
-	return
-}
-
-// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service
-// IDs into the db.
-func (s *eventsStatements) countEventsByApplicationServiceID(
-	ctx context.Context,
-	appServiceID string,
-) (int, error) {
-	var count int
-	err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count)
-	if err != nil && err != sql.ErrNoRows {
-		return 0, err
-	}
-
-	return count, nil
-}
-
-// insertEvent inserts an event mapped to its corresponding application service
-// IDs into the db.
-func (s *eventsStatements) insertEvent(
-	ctx context.Context,
-	appServiceID string,
-	event *gomatrixserverlib.HeaderedEvent,
-) (err error) {
-	// Convert event to JSON before inserting
-	eventJSON, err := json.Marshal(event)
-	if err != nil {
-		return err
-	}
-
-	_, err = s.insertEventStmt.ExecContext(
-		ctx,
-		appServiceID,
-		eventJSON,
-		-1, // No transaction ID yet
-	)
-	return
-}
-
-// updateTxnIDForEvents sets the transactionID for a collection of events. Done
-// before sending them to an AppService. Referenced before sending to make sure
-// we aren't constructing multiple transactions with the same events.
-func (s *eventsStatements) updateTxnIDForEvents(
-	ctx context.Context,
-	appserviceID string,
-	maxID, txnID int,
-) (err error) {
-	_, err = s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID)
-	return
-}
-
-// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database.
-func (s *eventsStatements) deleteEventsBeforeAndIncludingID(
-	ctx context.Context,
-	appserviceID string,
-	eventTableID int,
-) (err error) {
-	_, err = s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID)
-	return
-}
diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go
deleted file mode 100644
index a4c04b2cc..000000000
--- a/appservice/storage/postgres/storage.go
+++ /dev/null
@@ -1,115 +0,0 @@
-// Copyright 2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-
-	// Import postgres database driver
-	_ "github.com/lib/pq"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// Database stores events intended to be later sent to application services
-type Database struct {
-	events eventsStatements
-	txnID  txnStatements
-	db     *sql.DB
-	writer sqlutil.Writer
-}
-
-// NewDatabase opens a new database
-func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*Database, error) {
-	var result Database
-	var err error
-	if result.db, result.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
-		return nil, err
-	}
-	if err = result.prepare(); err != nil {
-		return nil, err
-	}
-	return &result, nil
-}
-
-func (d *Database) prepare() error {
-	if err := d.events.prepare(d.db); err != nil {
-		return err
-	}
-
-	return d.txnID.prepare(d.db)
-}
-
-// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database
-// for a transaction worker to pull and later send to an application service.
-func (d *Database) StoreEvent(
-	ctx context.Context,
-	appServiceID string,
-	event *gomatrixserverlib.HeaderedEvent,
-) error {
-	return d.events.insertEvent(ctx, appServiceID, event)
-}
-
-// GetEventsWithAppServiceID returns a slice of events and their IDs intended to
-// be sent to an application service given its ID.
-func (d *Database) GetEventsWithAppServiceID(
-	ctx context.Context,
-	appServiceID string,
-	limit int,
-) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) {
-	return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit)
-}
-
-// CountEventsWithAppServiceID returns the number of events destined for an
-// application service given its ID.
-func (d *Database) CountEventsWithAppServiceID(
-	ctx context.Context,
-	appServiceID string,
-) (int, error) {
-	return d.events.countEventsByApplicationServiceID(ctx, appServiceID)
-}
-
-// UpdateTxnIDForEvents takes in an application service ID and a
-// and stores them in the DB, unless the pair already exists, in
-// which case it updates them.
-func (d *Database) UpdateTxnIDForEvents(
-	ctx context.Context,
-	appserviceID string,
-	maxID, txnID int,
-) error {
-	return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID)
-}
-
-// RemoveEventsBeforeAndIncludingID removes all events from the database that
-// are less than or equal to a given maximum ID. IDs here are implemented as a
-// serial, thus this should always delete events in chronological order.
-func (d *Database) RemoveEventsBeforeAndIncludingID(
-	ctx context.Context,
-	appserviceID string,
-	eventTableID int,
-) error {
-	return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID)
-}
-
-// GetLatestTxnID returns the latest available transaction id
-func (d *Database) GetLatestTxnID(
-	ctx context.Context,
-) (int, error) {
-	return d.txnID.selectTxnID(ctx)
-}
diff --git a/appservice/storage/postgres/txn_id_counter_table.go b/appservice/storage/postgres/txn_id_counter_table.go
deleted file mode 100644
index a96a0e360..000000000
--- a/appservice/storage/postgres/txn_id_counter_table.go
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright 2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-)
-
-const txnIDSchema = `
--- Keeps a count of the current transaction ID
-CREATE SEQUENCE IF NOT EXISTS txn_id_counter START 1;
-`
-
-const selectTxnIDSQL = "SELECT nextval('txn_id_counter')"
-
-type txnStatements struct {
-	selectTxnIDStmt *sql.Stmt
-}
-
-func (s *txnStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(txnIDSchema)
-	if err != nil {
-		return
-	}
-
-	if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil {
-		return
-	}
-
-	return
-}
-
-// selectTxnID selects the latest ascending transaction ID
-func (s *txnStatements) selectTxnID(
-	ctx context.Context,
-) (txnID int, err error) {
-	err = s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID)
-	return
-}
diff --git a/appservice/storage/sqlite3/appservice_events_table.go b/appservice/storage/sqlite3/appservice_events_table.go
deleted file mode 100644
index 34b4859ea..000000000
--- a/appservice/storage/sqlite3/appservice_events_table.go
+++ /dev/null
@@ -1,267 +0,0 @@
-// Copyright 2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"time"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/gomatrixserverlib"
-	log "github.com/sirupsen/logrus"
-)
-
-const appserviceEventsSchema = `
--- Stores events to be sent to application services
-CREATE TABLE IF NOT EXISTS appservice_events (
-	-- An auto-incrementing id unique to each event in the table
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	-- The ID of the application service the event will be sent to
-	as_id TEXT NOT NULL,
-	-- JSON representation of the event
-	headered_event_json TEXT NOT NULL,
-	-- The ID of the transaction that this event is a part of
-	txn_id INTEGER NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id);
-`
-
-const selectEventsByApplicationServiceIDSQL = "" +
-	"SELECT id, headered_event_json, txn_id " +
-	"FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC"
-
-const countEventsByApplicationServiceIDSQL = "" +
-	"SELECT COUNT(id) FROM appservice_events WHERE as_id = $1"
-
-const insertEventSQL = "" +
-	"INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " +
-	"VALUES ($1, $2, $3)"
-
-const updateTxnIDForEventsSQL = "" +
-	"UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3"
-
-const deleteEventsBeforeAndIncludingIDSQL = "" +
-	"DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2"
-
-const (
-	// A transaction ID number that no transaction should ever have. Used for
-	// checking again the default value.
-	invalidTxnID = -2
-)
-
-type eventsStatements struct {
-	db                                     *sql.DB
-	writer                                 sqlutil.Writer
-	selectEventsByApplicationServiceIDStmt *sql.Stmt
-	countEventsByApplicationServiceIDStmt  *sql.Stmt
-	insertEventStmt                        *sql.Stmt
-	updateTxnIDForEventsStmt               *sql.Stmt
-	deleteEventsBeforeAndIncludingIDStmt   *sql.Stmt
-}
-
-func (s *eventsStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	s.db = db
-	s.writer = writer
-	_, err = db.Exec(appserviceEventsSchema)
-	if err != nil {
-		return
-	}
-
-	if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil {
-		return
-	}
-	if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil {
-		return
-	}
-	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
-		return
-	}
-	if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil {
-		return
-	}
-	if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil {
-		return
-	}
-
-	return
-}
-
-// selectEventsByApplicationServiceID takes in an application service ID and
-// returns a slice of events that need to be sent to that application service,
-// as well as an int later used to remove these same events from the database
-// once successfully sent to an application service.
-func (s *eventsStatements) selectEventsByApplicationServiceID(
-	ctx context.Context,
-	applicationServiceID string,
-	limit int,
-) (
-	txnID, maxID int,
-	events []gomatrixserverlib.HeaderedEvent,
-	eventsRemaining bool,
-	err error,
-) {
-	defer func() {
-		if err != nil {
-			log.WithFields(log.Fields{
-				"appservice": applicationServiceID,
-			}).WithError(err).Fatalf("appservice unable to select new events to send")
-		}
-	}()
-	// Retrieve events from the database. Unsuccessfully sent events first
-	eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID)
-	if err != nil {
-		return
-	}
-	defer checkNamedErr(eventRows.Close, &err)
-	events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil
-func checkNamedErr(fn func() error, err *error) {
-	if e := fn(); e != nil && *err == nil {
-		*err = e
-	}
-}
-
-func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) {
-	// Get current time for use in calculating event age
-	nowMilli := time.Now().UnixNano() / int64(time.Millisecond)
-
-	// Iterate through each row and store event contents
-	// If txn_id changes dramatically, we've switched from collecting old events to
-	// new ones. Send back those events first.
-	lastTxnID := invalidTxnID
-	for eventsProcessed := 0; eventRows.Next(); {
-		var event gomatrixserverlib.HeaderedEvent
-		var eventJSON []byte
-		var id int
-		err = eventRows.Scan(
-			&id,
-			&eventJSON,
-			&txnID,
-		)
-		if err != nil {
-			return nil, 0, 0, false, err
-		}
-
-		// Unmarshal eventJSON
-		if err = json.Unmarshal(eventJSON, &event); err != nil {
-			return nil, 0, 0, false, err
-		}
-
-		// If txnID has changed on this event from the previous event, then we've
-		// reached the end of a transaction's events. Return only those events.
-		if lastTxnID > invalidTxnID && lastTxnID != txnID {
-			return events, maxID, lastTxnID, true, nil
-		}
-		lastTxnID = txnID
-
-		// Limit events that aren't part of an old transaction
-		if txnID == -1 {
-			// Return if we've hit the limit
-			if eventsProcessed++; eventsProcessed > limit {
-				return events, maxID, lastTxnID, true, nil
-			}
-		}
-
-		if id > maxID {
-			maxID = id
-		}
-
-		// Portion of the event that is unsigned due to rapid change
-		// TODO: Consider removing age as not many app services use it
-		if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil {
-			return nil, 0, 0, false, err
-		}
-
-		events = append(events, event)
-	}
-
-	return
-}
-
-// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service
-// IDs into the db.
-func (s *eventsStatements) countEventsByApplicationServiceID(
-	ctx context.Context,
-	appServiceID string,
-) (int, error) {
-	var count int
-	err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count)
-	if err != nil && err != sql.ErrNoRows {
-		return 0, err
-	}
-
-	return count, nil
-}
-
-// insertEvent inserts an event mapped to its corresponding application service
-// IDs into the db.
-func (s *eventsStatements) insertEvent(
-	ctx context.Context,
-	appServiceID string,
-	event *gomatrixserverlib.HeaderedEvent,
-) (err error) {
-	// Convert event to JSON before inserting
-	eventJSON, err := json.Marshal(event)
-	if err != nil {
-		return err
-	}
-
-	return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		_, err := s.insertEventStmt.ExecContext(
-			ctx,
-			appServiceID,
-			eventJSON,
-			-1, // No transaction ID yet
-		)
-		return err
-	})
-}
-
-// updateTxnIDForEvents sets the transactionID for a collection of events. Done
-// before sending them to an AppService. Referenced before sending to make sure
-// we aren't constructing multiple transactions with the same events.
-func (s *eventsStatements) updateTxnIDForEvents(
-	ctx context.Context,
-	appserviceID string,
-	maxID, txnID int,
-) (err error) {
-	return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		_, err := s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID)
-		return err
-	})
-}
-
-// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database.
-func (s *eventsStatements) deleteEventsBeforeAndIncludingID(
-	ctx context.Context,
-	appserviceID string,
-	eventTableID int,
-) (err error) {
-	return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		_, err := s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID)
-		return err
-	})
-}
diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go
deleted file mode 100644
index ad62b3628..000000000
--- a/appservice/storage/sqlite3/storage.go
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-
-	// Import SQLite database driver
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// Database stores events intended to be later sent to application services
-type Database struct {
-	events eventsStatements
-	txnID  txnStatements
-	db     *sql.DB
-	writer sqlutil.Writer
-}
-
-// NewDatabase opens a new database
-func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*Database, error) {
-	var result Database
-	var err error
-	if result.db, result.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
-		return nil, err
-	}
-	if err = result.prepare(); err != nil {
-		return nil, err
-	}
-	return &result, nil
-}
-
-func (d *Database) prepare() error {
-	if err := d.events.prepare(d.db, d.writer); err != nil {
-		return err
-	}
-
-	return d.txnID.prepare(d.db, d.writer)
-}
-
-// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database
-// for a transaction worker to pull and later send to an application service.
-func (d *Database) StoreEvent(
-	ctx context.Context,
-	appServiceID string,
-	event *gomatrixserverlib.HeaderedEvent,
-) error {
-	return d.events.insertEvent(ctx, appServiceID, event)
-}
-
-// GetEventsWithAppServiceID returns a slice of events and their IDs intended to
-// be sent to an application service given its ID.
-func (d *Database) GetEventsWithAppServiceID(
-	ctx context.Context,
-	appServiceID string,
-	limit int,
-) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) {
-	return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit)
-}
-
-// CountEventsWithAppServiceID returns the number of events destined for an
-// application service given its ID.
-func (d *Database) CountEventsWithAppServiceID(
-	ctx context.Context,
-	appServiceID string,
-) (int, error) {
-	return d.events.countEventsByApplicationServiceID(ctx, appServiceID)
-}
-
-// UpdateTxnIDForEvents takes in an application service ID and a
-// and stores them in the DB, unless the pair already exists, in
-// which case it updates them.
-func (d *Database) UpdateTxnIDForEvents(
-	ctx context.Context,
-	appserviceID string,
-	maxID, txnID int,
-) error {
-	return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID)
-}
-
-// RemoveEventsBeforeAndIncludingID removes all events from the database that
-// are less than or equal to a given maximum ID. IDs here are implemented as a
-// serial, thus this should always delete events in chronological order.
-func (d *Database) RemoveEventsBeforeAndIncludingID(
-	ctx context.Context,
-	appserviceID string,
-	eventTableID int,
-) error {
-	return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID)
-}
-
-// GetLatestTxnID returns the latest available transaction id
-func (d *Database) GetLatestTxnID(
-	ctx context.Context,
-) (int, error) {
-	return d.txnID.selectTxnID(ctx)
-}
diff --git a/appservice/storage/sqlite3/txn_id_counter_table.go b/appservice/storage/sqlite3/txn_id_counter_table.go
deleted file mode 100644
index f2e902f98..000000000
--- a/appservice/storage/sqlite3/txn_id_counter_table.go
+++ /dev/null
@@ -1,82 +0,0 @@
-// Copyright 2018 New Vector Ltd
-// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-)
-
-const txnIDSchema = `
--- Keeps a count of the current transaction ID
-CREATE TABLE IF NOT EXISTS appservice_counters (
-  name TEXT PRIMARY KEY NOT NULL,
-  last_id INTEGER DEFAULT 1
-);
-INSERT OR IGNORE INTO appservice_counters (name, last_id) VALUES('txn_id', 1);
-`
-
-const selectTxnIDSQL = `
-  SELECT last_id FROM appservice_counters WHERE name='txn_id'
-`
-
-const updateTxnIDSQL = `
-  UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id'
-`
-
-type txnStatements struct {
-	db              *sql.DB
-	writer          sqlutil.Writer
-	selectTxnIDStmt *sql.Stmt
-	updateTxnIDStmt *sql.Stmt
-}
-
-func (s *txnStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
-	s.db = db
-	s.writer = writer
-	_, err = db.Exec(txnIDSchema)
-	if err != nil {
-		return
-	}
-
-	if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil {
-		return
-	}
-
-	if s.updateTxnIDStmt, err = db.Prepare(updateTxnIDSQL); err != nil {
-		return
-	}
-
-	return
-}
-
-// selectTxnID selects the latest ascending transaction ID
-func (s *txnStatements) selectTxnID(
-	ctx context.Context,
-) (txnID int, err error) {
-	err = s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
-		err := s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID)
-		if err != nil {
-			return err
-		}
-
-		_, err = s.updateTxnIDStmt.ExecContext(ctx)
-		return err
-	})
-	return
-}
diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go
deleted file mode 100644
index 89d5e0cc2..000000000
--- a/appservice/storage/storage.go
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build !wasm
-// +build !wasm
-
-package storage
-
-import (
-	"fmt"
-
-	"github.com/matrix-org/dendrite/appservice/storage/postgres"
-	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
-// and sets DB connection parameters
-func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
-	switch {
-	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(base, dbProperties)
-	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(base, dbProperties)
-	default:
-		return nil, fmt.Errorf("unexpected database type")
-	}
-}
diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go
deleted file mode 100644
index 230254598..000000000
--- a/appservice/storage/storage_wasm.go
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package storage
-
-import (
-	"fmt"
-
-	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
-	switch {
-	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(base, dbProperties)
-	case dbProperties.ConnectionString.IsPostgres():
-		return nil, fmt.Errorf("can't use Postgres implementation")
-	default:
-		return nil, fmt.Errorf("unexpected database type")
-	}
-}
diff --git a/appservice/types/types.go b/appservice/types/types.go
deleted file mode 100644
index 098face62..000000000
--- a/appservice/types/types.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package types
-
-import (
-	"sync"
-
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-const (
-	// AppServiceDeviceID is the AS dummy device ID
-	AppServiceDeviceID = "AS_Device"
-)
-
-// ApplicationServiceWorkerState is a type that couples an application service,
-// a lockable condition as well as some other state variables, allowing the
-// roomserver to notify appservice workers when there are events ready to send
-// externally to application services.
-type ApplicationServiceWorkerState struct {
-	AppService config.ApplicationService
-	Cond       *sync.Cond
-	// Events ready to be sent
-	EventsReady bool
-	// Backoff exponent (2^x secs). Max 6, aka 64s.
-	Backoff int
-}
-
-// NotifyNewEvents wakes up all waiting goroutines, notifying that events remain
-// in the event queue for this application service worker.
-func (a *ApplicationServiceWorkerState) NotifyNewEvents() {
-	a.Cond.L.Lock()
-	a.EventsReady = true
-	a.Cond.Broadcast()
-	a.Cond.L.Unlock()
-}
-
-// FinishEventProcessing marks all events of this worker as being sent to the
-// application service.
-func (a *ApplicationServiceWorkerState) FinishEventProcessing() {
-	a.Cond.L.Lock()
-	a.EventsReady = false
-	a.Cond.L.Unlock()
-}
-
-// WaitForNewEvents causes the calling goroutine to wait on the worker state's
-// condition for a broadcast or similar wakeup, if there are no events ready.
-func (a *ApplicationServiceWorkerState) WaitForNewEvents() {
-	a.Cond.L.Lock()
-	if !a.EventsReady {
-		a.Cond.Wait()
-	}
-	a.Cond.L.Unlock()
-}
diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go
deleted file mode 100644
index 4dab00bd7..000000000
--- a/appservice/workers/transaction_scheduler.go
+++ /dev/null
@@ -1,236 +0,0 @@
-// Copyright 2018 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package workers
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"math"
-	"net/http"
-	"net/url"
-	"time"
-
-	"github.com/matrix-org/dendrite/appservice/storage"
-	"github.com/matrix-org/dendrite/appservice/types"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	log "github.com/sirupsen/logrus"
-)
-
-var (
-	// Maximum size of events sent in each transaction.
-	transactionBatchSize = 50
-)
-
-// SetupTransactionWorkers spawns a separate goroutine for each application
-// service. Each of these "workers" handle taking all events intended for their
-// app service, batch them up into a single transaction (up to a max transaction
-// size), then send that off to the AS's /transactions/{txnID} endpoint. It also
-// handles exponentially backing off in case the AS isn't currently available.
-func SetupTransactionWorkers(
-	client *http.Client,
-	appserviceDB storage.Database,
-	workerStates []types.ApplicationServiceWorkerState,
-) error {
-	// Create a worker that handles transmitting events to a single homeserver
-	for _, workerState := range workerStates {
-		// Don't create a worker if this AS doesn't want to receive events
-		if workerState.AppService.URL != "" {
-			go worker(client, appserviceDB, workerState)
-		}
-	}
-	return nil
-}
-
-// worker is a goroutine that sends any queued events to the application service
-// it is given.
-func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
-	log.WithFields(log.Fields{
-		"appservice": ws.AppService.ID,
-	}).Info("Starting application service")
-	ctx := context.Background()
-
-	// Initial check for any leftover events to send from last time
-	eventCount, err := db.CountEventsWithAppServiceID(ctx, ws.AppService.ID)
-	if err != nil {
-		log.WithFields(log.Fields{
-			"appservice": ws.AppService.ID,
-		}).WithError(err).Fatal("appservice worker unable to read queued events from DB")
-		return
-	}
-	if eventCount > 0 {
-		ws.NotifyNewEvents()
-	}
-
-	// Loop forever and keep waiting for more events to send
-	for {
-		// Wait for more events if we've sent all the events in the database
-		ws.WaitForNewEvents()
-
-		// Batch events up into a transaction
-		transactionJSON, txnID, maxEventID, eventsRemaining, err := createTransaction(ctx, db, ws.AppService.ID)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"appservice": ws.AppService.ID,
-			}).WithError(err).Fatal("appservice worker unable to create transaction")
-
-			return
-		}
-
-		// Send the events off to the application service
-		// Backoff if the application service does not respond
-		err = send(client, ws.AppService, txnID, transactionJSON)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"appservice": ws.AppService.ID,
-			}).WithError(err).Error("unable to send event")
-			// Backoff
-			backoff(&ws, err)
-			continue
-		}
-
-		// We sent successfully, hooray!
-		ws.Backoff = 0
-
-		// Transactions have a maximum event size, so there may still be some events
-		// left over to send. Keep sending until none are left
-		if !eventsRemaining {
-			ws.FinishEventProcessing()
-		}
-
-		// Remove sent events from the DB
-		err = db.RemoveEventsBeforeAndIncludingID(ctx, ws.AppService.ID, maxEventID)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"appservice": ws.AppService.ID,
-			}).WithError(err).Fatal("unable to remove appservice events from the database")
-			return
-		}
-	}
-}
-
-// backoff pauses the calling goroutine for a 2^some backoff exponent seconds
-func backoff(ws *types.ApplicationServiceWorkerState, err error) {
-	// Calculate how long to backoff for
-	backoffDuration := time.Duration(math.Pow(2, float64(ws.Backoff)))
-	backoffSeconds := time.Second * backoffDuration
-
-	log.WithFields(log.Fields{
-		"appservice": ws.AppService.ID,
-	}).WithError(err).Warnf("unable to send transactions successfully, backing off for %ds",
-		backoffDuration)
-
-	ws.Backoff++
-	if ws.Backoff > 6 {
-		ws.Backoff = 6
-	}
-
-	// Backoff
-	time.Sleep(backoffSeconds)
-}
-
-// createTransaction takes in a slice of AS events, stores them in an AS
-// transaction, and JSON-encodes the results.
-func createTransaction(
-	ctx context.Context,
-	db storage.Database,
-	appserviceID string,
-) (
-	transactionJSON []byte,
-	txnID, maxID int,
-	eventsRemaining bool,
-	err error,
-) {
-	// Retrieve the latest events from the DB (will return old events if they weren't successfully sent)
-	txnID, maxID, events, eventsRemaining, err := db.GetEventsWithAppServiceID(ctx, appserviceID, transactionBatchSize)
-	if err != nil {
-		log.WithFields(log.Fields{
-			"appservice": appserviceID,
-		}).WithError(err).Fatalf("appservice worker unable to read queued events from DB")
-
-		return
-	}
-
-	// Check if these events do not already have a transaction ID
-	if txnID == -1 {
-		// If not, grab next available ID from the DB
-		txnID, err = db.GetLatestTxnID(ctx)
-		if err != nil {
-			return nil, 0, 0, false, err
-		}
-
-		// Mark new events with current transactionID
-		if err = db.UpdateTxnIDForEvents(ctx, appserviceID, maxID, txnID); err != nil {
-			return nil, 0, 0, false, err
-		}
-	}
-
-	var ev []*gomatrixserverlib.HeaderedEvent
-	for i := range events {
-		ev = append(ev, &events[i])
-	}
-
-	// Create a transaction and store the events inside
-	transaction := gomatrixserverlib.ApplicationServiceTransaction{
-		Events: gomatrixserverlib.HeaderedToClientEvents(ev, gomatrixserverlib.FormatAll),
-	}
-
-	transactionJSON, err = json.Marshal(transaction)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-// send sends events to an application service. Returns an error if an OK was not
-// received back from the application service or the request timed out.
-func send(
-	client *http.Client,
-	appservice config.ApplicationService,
-	txnID int,
-	transaction []byte,
-) (err error) {
-	// PUT a transaction to our AS
-	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	address := fmt.Sprintf("%s/transactions/%d?access_token=%s", appservice.URL, txnID, url.QueryEscape(appservice.HSToken))
-	req, err := http.NewRequest("PUT", address, bytes.NewBuffer(transaction))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer checkNamedErr(resp.Body.Close, &err)
-
-	// Check the AS received the events correctly
-	if resp.StatusCode != http.StatusOK {
-		// TODO: Handle non-200 error codes from application services
-		return fmt.Errorf("non-OK status code %d returned from AS", resp.StatusCode)
-	}
-
-	return nil
-}
-
-// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil
-func checkNamedErr(fn func() error, err *error) {
-	if e := fn(); e != nil && *err == nil {
-		*err = e
-	}
-}
diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index f3895ae23..60e817212 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -255,7 +255,6 @@ func (m *DendriteMonolith) Start() {
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix))
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix))
 	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix))
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix))
 	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
 	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
 	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 99b180c81..4432a3a45 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -94,7 +94,6 @@ func (m *DendriteMonolith) Start() {
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory))
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory))
 	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory))
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory))
 	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
 	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
 	cfg.ClientAPI.RegistrationDisabled = false
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index b16cfec6a..52c9dc8eb 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -124,7 +124,6 @@ func main() {
 		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
 		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
 		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
 		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
 		cfg.ClientAPI.RegistrationDisabled = false
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 619720d6c..086baa264 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -86,7 +86,6 @@ func main() {
 		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
 		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
 		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
 		cfg.MSCs.MSCs = []string{"msc2836"}
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
 		cfg.ClientAPI.RegistrationDisabled = false
diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go
index 1c585d916..6ae3ff9c9 100644
--- a/cmd/generate-config/main.go
+++ b/cmd/generate-config/main.go
@@ -24,7 +24,6 @@ func main() {
 		cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
 	}
 	if *dbURI != "" {
-		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(*dbURI)
 		cfg.FederationAPI.Database.ConnectionString = config.DataSource(*dbURI)
 		cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI)
 		cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI)
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 856b4ab22..3caf91434 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -132,13 +132,6 @@ app_service_api:
     listen: http://[::]:7777 # The listen address for incoming API requests
     connect: http://app_service_api:7777 # The connect address for other components to use
 
-  # Database configuration for this component.
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_appservice?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
   # Disable the validation of TLS certificates of appservices. This is
   # not recommended in production since it may allow appservice traffic
   # to be sent to an insecure endpoint.
diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go
index b8f99a612..9c3771272 100644
--- a/setup/config/config_appservice.go
+++ b/setup/config/config_appservice.go
@@ -31,8 +31,6 @@ type AppServiceAPI struct {
 
 	InternalAPI InternalAPIOptions `yaml:"internal_api"`
 
-	Database DatabaseOptions `yaml:"database"`
-
 	// DisableTLSValidation disables the validation of X.509 TLS certs
 	// on appservice endpoints. This is not recommended in production!
 	DisableTLSValidation bool `yaml:"disable_tls_validation"`
@@ -43,16 +41,9 @@ type AppServiceAPI struct {
 func (c *AppServiceAPI) Defaults(generate bool) {
 	c.InternalAPI.Listen = "http://localhost:7777"
 	c.InternalAPI.Connect = "http://localhost:7777"
-	c.Database.Defaults(5)
-	if generate {
-		c.Database.ConnectionString = "file:appservice.db"
-	}
 }
 
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	if c.Matrix.DatabaseOptions.ConnectionString == "" {
-		checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
-	}
 	if isMonolith { // polylith required configs below
 		return
 	}
diff --git a/setup/jetstream/helpers.go b/setup/jetstream/helpers.go
index f47637c69..1ec860b04 100644
--- a/setup/jetstream/helpers.go
+++ b/setup/jetstream/helpers.go
@@ -34,14 +34,6 @@ func JetStreamConsumer(
 		}
 	}()
 
-	// If the batch size is greater than 1, we will want to acknowledge all
-	// received messages in the batch. Below we will send an acknowledgement
-	// for the most recent message in the batch and AckAll will ensure that
-	// all messages that came before it are also acknowledged implicitly.
-	if batch > 1 {
-		opts = append(opts, nats.AckAll())
-	}
-
 	name := durable + "Pull"
 	sub, err := js.PullSubscribe(subj, name, opts...)
 	if err != nil {
@@ -89,21 +81,26 @@ func JetStreamConsumer(
 			if len(msgs) < 1 {
 				continue
 			}
-			msg := msgs[len(msgs)-1] // most recent message, in case of AckAll
-			if err = msg.InProgress(nats.Context(ctx)); err != nil {
-				logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err))
-				sentry.CaptureException(err)
-				continue
+			for _, msg := range msgs {
+				if err = msg.InProgress(nats.Context(ctx)); err != nil {
+					logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err))
+					sentry.CaptureException(err)
+					continue
+				}
 			}
 			if f(ctx, msgs) {
-				if err = msg.AckSync(nats.Context(ctx)); err != nil {
-					logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err))
-					sentry.CaptureException(err)
+				for _, msg := range msgs {
+					if err = msg.AckSync(nats.Context(ctx)); err != nil {
+						logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err))
+						sentry.CaptureException(err)
+					}
 				}
 			} else {
-				if err = msg.Nak(nats.Context(ctx)); err != nil {
-					logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err))
-					sentry.CaptureException(err)
+				for _, msg := range msgs {
+					if err = msg.Nak(nats.Context(ctx)); err != nil {
+						logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err))
+						sentry.CaptureException(err)
+					}
 				}
 			}
 		}
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 051d55a35..3660e91e3 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -183,6 +183,7 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
 		OutputReceiptEvent:      {"SyncAPIEDUServerReceiptConsumer", "FederationAPIEDUServerConsumer"},
 		OutputSendToDeviceEvent: {"SyncAPIEDUServerSendToDeviceConsumer", "FederationAPIEDUServerConsumer"},
 		OutputTypingEvent:       {"SyncAPIEDUServerTypingConsumer", "FederationAPIEDUServerConsumer"},
+		OutputRoomEvent:         {"AppserviceRoomserverConsumer"},
 	} {
 		streamName := cfg.Matrix.JetStream.Prefixed(stream)
 		for _, consumer := range consumers {
diff --git a/test/testrig/base.go b/test/testrig/base.go
index d13c43129..33230921c 100644
--- a/test/testrig/base.go
+++ b/test/testrig/base.go
@@ -57,7 +57,6 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 		return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), func() {
 			// cleanup db files. This risks getting out of sync as we add more database strings :(
 			dbFiles := []config.DataSource{
-				cfg.AppServiceAPI.Database.ConnectionString,
 				cfg.FederationAPI.Database.ConnectionString,
 				cfg.KeyServer.Database.ConnectionString,
 				cfg.MSCs.Database.ConnectionString,
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 6ba469327..2f28ee906 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -28,7 +28,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/bcrypt"
 
-	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/pushrules"
@@ -454,7 +453,7 @@ func (a *UserInternalAPI) queryAppServiceToken(ctx context.Context, token, appSe
 	// Create a dummy device for AS user
 	dev := api.Device{
 		// Use AS dummy device ID
-		ID: types.AppServiceDeviceID,
+		ID: "AS_Device",
 		// AS dummy device has AS's token.
 		AccessToken:  token,
 		AppserviceID: appService.ID,

From 51d229b02505df770ed891ec8ab20843bc2b1d1d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 1 Sep 2022 14:15:41 +0100
Subject: [PATCH 316/391] Configuration tweaks (#2567)

This makes the following changes:

* The various `Defaults` functions are now responsible for setting sane defaults if `generate` is specified, rather than hiding them in `generate-config`
* Some configuration options have been marked as `omitempty` so that they don't appear in generated configs unnecessarily (monolith-specific vs. polylith-specific options)
* A new option `-polylith` has been added to `generate-config` to create a config that makes sense for polylith deployments (i.e. including the internal/external API listeners and per-component database sections)
* A new option `-normalise` has been added to `generate-config` to take an existing file and add any missing options and/or defaults
---
 build/gobind-pinecone/monolith.go           |   5 +-
 build/gobind-yggdrasil/monolith.go          |   5 +-
 build/scripts/ComplementPostgres.Dockerfile |   7 +-
 clientapi/routing/register_test.go          |   5 +-
 cmd/dendrite-demo-pinecone/main.go          |   5 +-
 cmd/dendrite-demo-yggdrasil/main.go         |   5 +-
 cmd/generate-config/main.go                 | 150 +++++++++-----------
 federationapi/federationapi_keys_test.go    |   5 +-
 federationapi/federationapi_test.go         |   5 +-
 setup/config/config.go                      |  33 +++--
 setup/config/config_appservice.go           |  10 +-
 setup/config/config_clientapi.go            |  16 ++-
 setup/config/config_federationapi.go        |  45 ++++--
 setup/config/config_global.go               |  32 +++--
 setup/config/config_jetstream.go            |   4 +-
 setup/config/config_keyserver.go            |  26 ++--
 setup/config/config_mediaapi.go             |  47 ++++--
 setup/config/config_mscs.go                 |  17 ++-
 setup/config/config_roomserver.go           |  26 ++--
 setup/config/config_syncapi.go              |  30 ++--
 setup/config/config_userapi.go              |  26 ++--
 setup/mscs/msc2836/msc2836_test.go          |   5 +-
 test/testrig/base.go                        |  25 +++-
 23 files changed, 322 insertions(+), 212 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 60e817212..44eab4cd7 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -243,7 +243,10 @@ func (m *DendriteMonolith) Start() {
 
 	prefix := hex.EncodeToString(pk)
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 4432a3a45..896f3d14f 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -82,7 +82,10 @@ func (m *DendriteMonolith) Start() {
 	m.YggdrasilNode = ygg
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
 	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile
index 699540120..785090b0b 100644
--- a/build/scripts/ComplementPostgres.Dockerfile
+++ b/build/scripts/ComplementPostgres.Dockerfile
@@ -46,9 +46,8 @@ EXPOSE 8008 8448
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
 CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
-    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
-    # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
-    sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
-    sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
+    ./generate-config -server $SERVER_NAME --ci --db postgresql://postgres@localhost/postgres?sslmode=disable > dendrite.yaml && \
+    # Bump max_open_conns up here in the global database config
+    sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
     exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go
index 0507116f1..85846c7d6 100644
--- a/clientapi/routing/register_test.go
+++ b/clientapi/routing/register_test.go
@@ -181,7 +181,10 @@ func TestValidationOfApplicationServices(t *testing.T) {
 
 	// Set up a config
 	fakeConfig := &config.Dendrite{}
-	fakeConfig.Defaults(true)
+	fakeConfig.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	fakeConfig.Global.ServerName = "localhost"
 	fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
 
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 52c9dc8eb..83b52132a 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -115,7 +115,10 @@ func main() {
 				panic("failed to load PEM key: " + err.Error())
 			}
 		}
-		cfg.Defaults(true)
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: true,
+		})
 		cfg.Global.PrivateKey = sk
 		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 086baa264..d2fddf8bc 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -78,7 +78,10 @@ func main() {
 	if configFlagSet {
 		cfg = setup.ParseFlags(true)
 	} else {
-		cfg.Defaults(true)
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: true,
+		})
 		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
 		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go
index 6ae3ff9c9..c24e8153e 100644
--- a/cmd/generate-config/main.go
+++ b/cmd/generate-config/main.go
@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"path/filepath"
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -11,90 +12,81 @@ import (
 )
 
 func main() {
-	defaultsForCI := flag.Bool("ci", false, "sane defaults for CI testing")
+	defaultsForCI := flag.Bool("ci", false, "Populate the configuration with sane defaults for use in CI")
 	serverName := flag.String("server", "", "The domain name of the server if not 'localhost'")
-	dbURI := flag.String("db", "", "The DB URI to use for all components if not SQLite files")
+	dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)")
+	dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)")
+	normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults")
+	polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments")
 	flag.Parse()
 
-	cfg := &config.Dendrite{
-		Version: config.Version,
-	}
-	cfg.Defaults(true)
-	if *serverName != "" {
-		cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
-	}
-	if *dbURI != "" {
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.MediaAPI.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI)
-	}
-	cfg.Global.TrustedIDServers = []string{
-		"matrix.org",
-		"vector.im",
-	}
-	cfg.Logging = []config.LogrusHook{
-		{
-			Type:  "file",
-			Level: "info",
-			Params: map[string]interface{}{
-				"path": "/var/log/dendrite",
-			},
-		},
-	}
-	cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{
-		{
-			ServerName: "matrix.org",
-			Keys: []config.KeyPerspectiveTrustKey{
-				{
-					KeyID:     "ed25519:auto",
-					PublicKey: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw",
-				},
-				{
-					KeyID:     "ed25519:a_RXGa",
-					PublicKey: "l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ",
+	var cfg *config.Dendrite
+	if *normalise == "" {
+		cfg = &config.Dendrite{
+			Version: config.Version,
+		}
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: !*polylith,
+		})
+		if *serverName != "" {
+			cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
+		}
+		uri := config.DataSource(*dbURI)
+		if *polylith || uri.IsSQLite() || uri == "" {
+			for name, db := range map[string]*config.DatabaseOptions{
+				"federationapi": &cfg.FederationAPI.Database,
+				"keyserver":     &cfg.KeyServer.Database,
+				"mscs":          &cfg.MSCs.Database,
+				"mediaapi":      &cfg.MediaAPI.Database,
+				"roomserver":    &cfg.RoomServer.Database,
+				"syncapi":       &cfg.SyncAPI.Database,
+				"userapi":       &cfg.UserAPI.AccountDatabase,
+			} {
+				if uri == "" {
+					path := filepath.Join(*dirPath, fmt.Sprintf("dendrite_%s.db", name))
+					db.ConnectionString = config.DataSource(fmt.Sprintf("file:%s", path))
+				} else {
+					db.ConnectionString = uri
+				}
+			}
+		} else {
+			cfg.Global.DatabaseOptions.ConnectionString = uri
+		}
+		cfg.Logging = []config.LogrusHook{
+			{
+				Type:  "file",
+				Level: "info",
+				Params: map[string]interface{}{
+					"path": filepath.Join(*dirPath, "log"),
 				},
 			},
-		},
-	}
-	cfg.MediaAPI.ThumbnailSizes = []config.ThumbnailSize{
-		{
-			Width:        32,
-			Height:       32,
-			ResizeMethod: "crop",
-		},
-		{
-			Width:        96,
-			Height:       96,
-			ResizeMethod: "crop",
-		},
-		{
-			Width:        640,
-			Height:       480,
-			ResizeMethod: "scale",
-		},
-	}
-
-	if *defaultsForCI {
-		cfg.AppServiceAPI.DisableTLSValidation = true
-		cfg.ClientAPI.RateLimiting.Enabled = false
-		cfg.FederationAPI.DisableTLSValidation = false
-		// don't hit matrix.org when running tests!!!
-		cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
-		cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"}
-		cfg.Logging[0].Level = "trace"
-		cfg.Logging[0].Type = "std"
-		cfg.UserAPI.BCryptCost = bcrypt.MinCost
-		cfg.Global.JetStream.InMemory = true
-		cfg.ClientAPI.RegistrationDisabled = false
-		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-		cfg.ClientAPI.RegistrationSharedSecret = "complement"
-		cfg.Global.Presence = config.PresenceOptions{
-			EnableInbound:  true,
-			EnableOutbound: true,
+		}
+		if *defaultsForCI {
+			cfg.AppServiceAPI.DisableTLSValidation = true
+			cfg.ClientAPI.RateLimiting.Enabled = false
+			cfg.FederationAPI.DisableTLSValidation = false
+			// don't hit matrix.org when running tests!!!
+			cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
+			cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
+			cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"}
+			cfg.Logging[0].Level = "trace"
+			cfg.Logging[0].Type = "std"
+			cfg.UserAPI.BCryptCost = bcrypt.MinCost
+			cfg.Global.JetStream.InMemory = true
+			cfg.Global.JetStream.StoragePath = config.Path(*dirPath)
+			cfg.ClientAPI.RegistrationDisabled = false
+			cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+			cfg.ClientAPI.RegistrationSharedSecret = "complement"
+			cfg.Global.Presence = config.PresenceOptions{
+				EnableInbound:  true,
+				EnableOutbound: true,
+			}
+		}
+	} else {
+		var err error
+		if cfg, err = config.Load(*normalise, !*polylith); err != nil {
+			panic(err)
 		}
 	}
 
diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go
index 9c3446222..4469da35b 100644
--- a/federationapi/federationapi_keys_test.go
+++ b/federationapi/federationapi_keys_test.go
@@ -75,7 +75,10 @@ func TestMain(m *testing.M) {
 			// Draw up just enough Dendrite config for the server key
 			// API to work.
 			cfg := &config.Dendrite{}
-			cfg.Defaults(true)
+			cfg.Defaults(config.DefaultOpts{
+				Generate:   true,
+				Monolithic: true,
+			})
 			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
 			cfg.Global.PrivateKey = testPriv
 			cfg.Global.JetStream.InMemory = true
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index bdcb9f57c..15f7a6840 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -263,7 +263,10 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	_, privKey, _ := ed25519.GenerateKey(nil)
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
 	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
 	cfg.Global.PrivateKey = privKey
diff --git a/setup/config/config.go b/setup/config/config.go
index cc9c04470..5a618d671 100644
--- a/setup/config/config.go
+++ b/setup/config/config.go
@@ -211,7 +211,10 @@ func loadConfig(
 	monolithic bool,
 ) (*Dendrite, error) {
 	var c Dendrite
-	c.Defaults(false)
+	c.Defaults(DefaultOpts{
+		Generate:   false,
+		Monolithic: monolithic,
+	})
 	c.IsMonolith = monolithic
 
 	var err error
@@ -295,21 +298,25 @@ func (config *Dendrite) Derive() error {
 	return nil
 }
 
+type DefaultOpts struct {
+	Generate   bool
+	Monolithic bool
+}
+
 // SetDefaults sets default config values if they are not explicitly set.
-func (c *Dendrite) Defaults(generate bool) {
+func (c *Dendrite) Defaults(opts DefaultOpts) {
 	c.Version = Version
 
-	c.Global.Defaults(generate)
-	c.ClientAPI.Defaults(generate)
-	c.FederationAPI.Defaults(generate)
-	c.KeyServer.Defaults(generate)
-	c.MediaAPI.Defaults(generate)
-	c.RoomServer.Defaults(generate)
-	c.SyncAPI.Defaults(generate)
-	c.UserAPI.Defaults(generate)
-	c.AppServiceAPI.Defaults(generate)
-	c.MSCs.Defaults(generate)
-
+	c.Global.Defaults(opts)
+	c.ClientAPI.Defaults(opts)
+	c.FederationAPI.Defaults(opts)
+	c.KeyServer.Defaults(opts)
+	c.MediaAPI.Defaults(opts)
+	c.RoomServer.Defaults(opts)
+	c.SyncAPI.Defaults(opts)
+	c.UserAPI.Defaults(opts)
+	c.AppServiceAPI.Defaults(opts)
+	c.MSCs.Defaults(opts)
 	c.Wiring()
 }
 
diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go
index 9c3771272..bd21826fe 100644
--- a/setup/config/config_appservice.go
+++ b/setup/config/config_appservice.go
@@ -29,7 +29,7 @@ type AppServiceAPI struct {
 	Matrix  *Global  `yaml:"-"`
 	Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
 
 	// DisableTLSValidation disables the validation of X.509 TLS certs
 	// on appservice endpoints. This is not recommended in production!
@@ -38,9 +38,11 @@ type AppServiceAPI struct {
 	ConfigFiles []string `yaml:"config_files"`
 }
 
-func (c *AppServiceAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7777"
-	c.InternalAPI.Connect = "http://localhost:7777"
+func (c *AppServiceAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7777"
+		c.InternalAPI.Connect = "http://localhost:7777"
+	}
 }
 
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go
index ecf8f6bd5..56f4b3f92 100644
--- a/setup/config/config_clientapi.go
+++ b/setup/config/config_clientapi.go
@@ -9,8 +9,8 @@ type ClientAPI struct {
 	Matrix  *Global  `yaml:"-"`
 	Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
-	ExternalAPI ExternalAPIOptions `yaml:"external_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
+	ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"`
 
 	// If set disables new users from registering (except via shared
 	// secrets)
@@ -48,13 +48,15 @@ type ClientAPI struct {
 	// Rate-limiting options
 	RateLimiting RateLimiting `yaml:"rate_limiting"`
 
-	MSCs *MSCs `yaml:"mscs"`
+	MSCs *MSCs `yaml:"-"`
 }
 
-func (c *ClientAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7771"
-	c.InternalAPI.Connect = "http://localhost:7771"
-	c.ExternalAPI.Listen = "http://[::]:8071"
+func (c *ClientAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7771"
+		c.InternalAPI.Connect = "http://localhost:7771"
+		c.ExternalAPI.Listen = "http://[::]:8071"
+	}
 	c.RegistrationSharedSecret = ""
 	c.RecaptchaPublicKey = ""
 	c.RecaptchaPrivateKey = ""
diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go
index a7a515fda..d3a4b2b67 100644
--- a/setup/config/config_federationapi.go
+++ b/setup/config/config_federationapi.go
@@ -5,12 +5,12 @@ import "github.com/matrix-org/gomatrixserverlib"
 type FederationAPI struct {
 	Matrix *Global `yaml:"-"`
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
-	ExternalAPI ExternalAPIOptions `yaml:"external_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
+	ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"`
 
 	// The database stores information used by the federation destination queues to
 	// send transactions to remote servers.
-	Database DatabaseOptions `yaml:"database"`
+	Database DatabaseOptions `yaml:"database,omitempty"`
 
 	// Federation failure threshold. How many consecutive failures that we should
 	// tolerate when sending federation requests to a specific server. The backoff
@@ -30,25 +30,44 @@ type FederationAPI struct {
 	PreferDirectFetch bool `yaml:"prefer_direct_fetch"`
 }
 
-func (c *FederationAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7772"
-	c.InternalAPI.Connect = "http://localhost:7772"
-	c.ExternalAPI.Listen = "http://[::]:8072"
+func (c *FederationAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7772"
+		c.InternalAPI.Connect = "http://localhost:7772"
+		c.ExternalAPI.Listen = "http://[::]:8072"
+		c.Database.Defaults(10)
+	}
 	c.FederationMaxRetries = 16
 	c.DisableTLSValidation = false
-	c.Database.Defaults(10)
-	if generate {
-		c.Database.ConnectionString = "file:federationapi.db"
+	if opts.Generate {
+		c.KeyPerspectives = KeyPerspectives{
+			{
+				ServerName: "matrix.org",
+				Keys: []KeyPerspectiveTrustKey{
+					{
+						KeyID:     "ed25519:auto",
+						PublicKey: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw",
+					},
+					{
+						KeyID:     "ed25519:a_RXGa",
+						PublicKey: "l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ",
+					},
+				},
+			},
+		}
+		if !opts.Monolithic {
+			c.Database.ConnectionString = "file:federationapi.db"
+		}
 	}
 }
 
 func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	if c.Matrix.DatabaseOptions.ConnectionString == "" {
-		checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
-	}
 	if isMonolith { // polylith required configs below
 		return
 	}
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString))
+	}
 	checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
 	checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index d4e54e203..acc608dd7 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -41,7 +41,7 @@ type Global struct {
 	// connections will be used instead. This way we don't have to manage connection
 	// counts on a per-component basis, but can instead do it for the entire monolith.
 	// In a polylith deployment, this will be ignored.
-	DatabaseOptions DatabaseOptions `yaml:"database"`
+	DatabaseOptions DatabaseOptions `yaml:"database,omitempty"`
 
 	// The server name to delegate server-server communications to, with optional port
 	WellKnownServerName string `yaml:"well_known_server_name"`
@@ -83,22 +83,28 @@ type Global struct {
 	Cache Cache `yaml:"cache"`
 }
 
-func (c *Global) Defaults(generate bool) {
-	if generate {
+func (c *Global) Defaults(opts DefaultOpts) {
+	if opts.Generate {
 		c.ServerName = "localhost"
 		c.PrivateKeyPath = "matrix_key.pem"
 		_, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0)))
 		c.KeyID = "ed25519:auto"
+		c.TrustedIDServers = []string{
+			"matrix.org",
+			"vector.im",
+		}
 	}
 	c.KeyValidityPeriod = time.Hour * 24 * 7
-
-	c.JetStream.Defaults(generate)
-	c.Metrics.Defaults(generate)
+	if opts.Monolithic {
+		c.DatabaseOptions.Defaults(90)
+	}
+	c.JetStream.Defaults(opts)
+	c.Metrics.Defaults(opts)
 	c.DNSCache.Defaults()
 	c.Sentry.Defaults()
-	c.ServerNotices.Defaults(generate)
+	c.ServerNotices.Defaults(opts)
 	c.ReportStats.Defaults()
-	c.Cache.Defaults(generate)
+	c.Cache.Defaults()
 }
 
 func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
@@ -142,9 +148,9 @@ type Metrics struct {
 	} `yaml:"basic_auth"`
 }
 
-func (c *Metrics) Defaults(generate bool) {
+func (c *Metrics) Defaults(opts DefaultOpts) {
 	c.Enabled = false
-	if generate {
+	if opts.Generate {
 		c.BasicAuth.Username = "metrics"
 		c.BasicAuth.Password = "metrics"
 	}
@@ -166,8 +172,8 @@ type ServerNotices struct {
 	RoomName string `yaml:"room_name"`
 }
 
-func (c *ServerNotices) Defaults(generate bool) {
-	if generate {
+func (c *ServerNotices) Defaults(opts DefaultOpts) {
+	if opts.Generate {
 		c.Enabled = true
 		c.LocalPart = "_server"
 		c.DisplayName = "Server Alert"
@@ -183,7 +189,7 @@ type Cache struct {
 	MaxAge           time.Duration `yaml:"max_age"`
 }
 
-func (c *Cache) Defaults(generate bool) {
+func (c *Cache) Defaults() {
 	c.EstimatedMaxSize = 1024 * 1024 * 1024 // 1GB
 	c.MaxAge = time.Hour
 }
diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go
index a7827597e..ef8bf014b 100644
--- a/setup/config/config_jetstream.go
+++ b/setup/config/config_jetstream.go
@@ -31,10 +31,10 @@ func (c *JetStream) Durable(name string) string {
 	return c.Prefixed(name)
 }
 
-func (c *JetStream) Defaults(generate bool) {
+func (c *JetStream) Defaults(opts DefaultOpts) {
 	c.Addresses = []string{}
 	c.TopicPrefix = "Dendrite"
-	if generate {
+	if opts.Generate {
 		c.StoragePath = Path("./")
 		c.NoLog = true
 		c.DisableTLSValidation = true
diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go
index 5f2f22c8a..dca9ca9f5 100644
--- a/setup/config/config_keyserver.go
+++ b/setup/config/config_keyserver.go
@@ -3,27 +3,31 @@ package config
 type KeyServer struct {
 	Matrix *Global `yaml:"-"`
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
 
-	Database DatabaseOptions `yaml:"database"`
+	Database DatabaseOptions `yaml:"database,omitempty"`
 }
 
-func (c *KeyServer) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7779"
-	c.InternalAPI.Connect = "http://localhost:7779"
-	c.Database.Defaults(10)
-	if generate {
-		c.Database.ConnectionString = "file:keyserver.db"
+func (c *KeyServer) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7779"
+		c.InternalAPI.Connect = "http://localhost:7779"
+		c.Database.Defaults(10)
+	}
+	if opts.Generate {
+		if !opts.Monolithic {
+			c.Database.ConnectionString = "file:keyserver.db"
+		}
 	}
 }
 
 func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	if c.Matrix.DatabaseOptions.ConnectionString == "" {
-		checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
-	}
 	if isMonolith { // polylith required configs below
 		return
 	}
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
+	}
 	checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect))
 }
diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go
index 9717aa59e..53a8219eb 100644
--- a/setup/config/config_mediaapi.go
+++ b/setup/config/config_mediaapi.go
@@ -7,12 +7,12 @@ import (
 type MediaAPI struct {
 	Matrix *Global `yaml:"-"`
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
-	ExternalAPI ExternalAPIOptions `yaml:"external_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
+	ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"`
 
 	// The MediaAPI database stores information about files uploaded and downloaded
 	// by local users. It is only accessed by the MediaAPI.
-	Database DatabaseOptions `yaml:"database"`
+	Database DatabaseOptions `yaml:"database,omitempty"`
 
 	// The base path to where the media files will be stored. May be relative or absolute.
 	BasePath Path `yaml:"base_path"`
@@ -38,23 +38,41 @@ type MediaAPI struct {
 // DefaultMaxFileSizeBytes defines the default file size allowed in transfers
 var DefaultMaxFileSizeBytes = FileSizeBytes(10485760)
 
-func (c *MediaAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7774"
-	c.InternalAPI.Connect = "http://localhost:7774"
-	c.ExternalAPI.Listen = "http://[::]:8074"
+func (c *MediaAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7774"
+		c.InternalAPI.Connect = "http://localhost:7774"
+		c.ExternalAPI.Listen = "http://[::]:8074"
+		c.Database.Defaults(5)
+	}
 	c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
 	c.MaxThumbnailGenerators = 10
-	c.Database.Defaults(5)
-	if generate {
-		c.Database.ConnectionString = "file:mediaapi.db"
+	if opts.Generate {
+		c.ThumbnailSizes = []ThumbnailSize{
+			{
+				Width:        32,
+				Height:       32,
+				ResizeMethod: "crop",
+			},
+			{
+				Width:        96,
+				Height:       96,
+				ResizeMethod: "crop",
+			},
+			{
+				Width:        640,
+				Height:       480,
+				ResizeMethod: "scale",
+			},
+		}
+		if !opts.Monolithic {
+			c.Database.ConnectionString = "file:mediaapi.db"
+		}
 		c.BasePath = "./media_store"
 	}
 }
 
 func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	if c.Matrix.DatabaseOptions.ConnectionString == "" {
-		checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
-	}
 	checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
 	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes))
 	checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
@@ -66,6 +84,9 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	if isMonolith { // polylith required configs below
 		return
 	}
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
+	}
 	checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
 	checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go
index b992f7152..6d5ff39a5 100644
--- a/setup/config/config_mscs.go
+++ b/setup/config/config_mscs.go
@@ -10,13 +10,17 @@ type MSCs struct {
 	// 'msc2946': Spaces Summary - https://github.com/matrix-org/matrix-doc/pull/2946
 	MSCs []string `yaml:"mscs"`
 
-	Database DatabaseOptions `yaml:"database"`
+	Database DatabaseOptions `yaml:"database,omitempty"`
 }
 
-func (c *MSCs) Defaults(generate bool) {
-	c.Database.Defaults(5)
-	if generate {
-		c.Database.ConnectionString = "file:mscs.db"
+func (c *MSCs) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.Database.Defaults(5)
+	}
+	if opts.Generate {
+		if !opts.Monolithic {
+			c.Database.ConnectionString = "file:mscs.db"
+		}
 	}
 }
 
@@ -31,6 +35,9 @@ func (c *MSCs) Enabled(msc string) bool {
 }
 
 func (c *MSCs) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	if isMonolith { // polylith required configs below
+		return
+	}
 	if c.Matrix.DatabaseOptions.ConnectionString == "" {
 		checkNotEmpty(configErrs, "mscs.database.connection_string", string(c.Database.ConnectionString))
 	}
diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go
index bd6aa1167..5e3b7f2ec 100644
--- a/setup/config/config_roomserver.go
+++ b/setup/config/config_roomserver.go
@@ -3,27 +3,31 @@ package config
 type RoomServer struct {
 	Matrix *Global `yaml:"-"`
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
 
-	Database DatabaseOptions `yaml:"database"`
+	Database DatabaseOptions `yaml:"database,omitempty"`
 }
 
-func (c *RoomServer) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7770"
-	c.InternalAPI.Connect = "http://localhost:7770"
-	c.Database.Defaults(10)
-	if generate {
-		c.Database.ConnectionString = "file:roomserver.db"
+func (c *RoomServer) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7770"
+		c.InternalAPI.Connect = "http://localhost:7770"
+		c.Database.Defaults(20)
+	}
+	if opts.Generate {
+		if !opts.Monolithic {
+			c.Database.ConnectionString = "file:roomserver.db"
+		}
 	}
 }
 
 func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	if c.Matrix.DatabaseOptions.ConnectionString == "" {
-		checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
-	}
 	if isMonolith { // polylith required configs below
 		return
 	}
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
+	}
 	checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect))
 }
diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go
index 7d5e3808a..4e302ae09 100644
--- a/setup/config/config_syncapi.go
+++ b/setup/config/config_syncapi.go
@@ -3,31 +3,35 @@ package config
 type SyncAPI struct {
 	Matrix *Global `yaml:"-"`
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
-	ExternalAPI ExternalAPIOptions `yaml:"external_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
+	ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"`
 
-	Database DatabaseOptions `yaml:"database"`
+	Database DatabaseOptions `yaml:"database,omitempty"`
 
 	RealIPHeader string `yaml:"real_ip_header"`
 }
 
-func (c *SyncAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7773"
-	c.InternalAPI.Connect = "http://localhost:7773"
-	c.ExternalAPI.Listen = "http://localhost:8073"
-	c.Database.Defaults(10)
-	if generate {
-		c.Database.ConnectionString = "file:syncapi.db"
+func (c *SyncAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7773"
+		c.InternalAPI.Connect = "http://localhost:7773"
+		c.ExternalAPI.Listen = "http://localhost:8073"
+		c.Database.Defaults(20)
+	}
+	if opts.Generate {
+		if !opts.Monolithic {
+			c.Database.ConnectionString = "file:syncapi.db"
+		}
 	}
 }
 
 func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
-	if c.Matrix.DatabaseOptions.ConnectionString == "" {
-		checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
-	}
 	if isMonolith { // polylith required configs below
 		return
 	}
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
+	}
 	checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
 	checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go
index d1e2b7fe1..97a6d738b 100644
--- a/setup/config/config_userapi.go
+++ b/setup/config/config_userapi.go
@@ -5,7 +5,7 @@ import "golang.org/x/crypto/bcrypt"
 type UserAPI struct {
 	Matrix *Global `yaml:"-"`
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
 
 	// The cost when hashing passwords.
 	BCryptCost int `yaml:"bcrypt_cost"`
@@ -18,30 +18,34 @@ type UserAPI struct {
 
 	// The Account database stores the login details and account information
 	// for local users. It is accessed by the UserAPI.
-	AccountDatabase DatabaseOptions `yaml:"account_database"`
+	AccountDatabase DatabaseOptions `yaml:"account_database,omitempty"`
 }
 
 const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
 
-func (c *UserAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7781"
-	c.InternalAPI.Connect = "http://localhost:7781"
+func (c *UserAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7781"
+		c.InternalAPI.Connect = "http://localhost:7781"
+		c.AccountDatabase.Defaults(10)
+	}
 	c.BCryptCost = bcrypt.DefaultCost
 	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
-	c.AccountDatabase.Defaults(10)
-	if generate {
-		c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
+	if opts.Generate {
+		if !opts.Monolithic {
+			c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
+		}
 	}
 }
 
 func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
-	if c.Matrix.DatabaseOptions.ConnectionString == "" {
-		checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
-	}
 	if isMonolith { // polylith required configs below
 		return
 	}
+	if c.Matrix.DatabaseOptions.ConnectionString == "" {
+		checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
+	}
 	checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
 	checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
 }
diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go
index 3e9d90a1f..0388fcc53 100644
--- a/setup/mscs/msc2836/msc2836_test.go
+++ b/setup/mscs/msc2836/msc2836_test.go
@@ -547,7 +547,10 @@ func (r *testRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *roo
 func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router {
 	t.Helper()
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	cfg.Global.ServerName = "localhost"
 	cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db"
 	cfg.MSCs.MSCs = []string{"msc2836"}
diff --git a/test/testrig/base.go b/test/testrig/base.go
index 33230921c..9747ea600 100644
--- a/test/testrig/base.go
+++ b/test/testrig/base.go
@@ -30,12 +30,21 @@ import (
 
 func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) {
 	var cfg config.Dendrite
-	cfg.Defaults(false)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   false,
+		Monolithic: true,
+	})
 	cfg.Global.JetStream.InMemory = true
 	switch dbType {
 	case test.DBTypePostgres:
-		cfg.Global.Defaults(true)   // autogen a signing key
-		cfg.MediaAPI.Defaults(true) // autogen a media path
+		cfg.Global.Defaults(config.DefaultOpts{ // autogen a signing key
+			Generate:   true,
+			Monolithic: true,
+		})
+		cfg.MediaAPI.Defaults(config.DefaultOpts{ // autogen a media path
+			Generate:   true,
+			Monolithic: true,
+		})
 		cfg.Global.ServerName = "test"
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
@@ -49,7 +58,10 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 		}
 		return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), close
 	case test.DBTypeSQLite:
-		cfg.Defaults(true) // sets a sqlite db per component
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: false, // because we need a database per component
+		})
 		cfg.Global.ServerName = "test"
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
@@ -82,7 +94,10 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) {
 	if cfg == nil {
 		cfg = &config.Dendrite{}
-		cfg.Defaults(true)
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: true,
+		})
 	}
 	cfg.Global.JetStream.InMemory = true
 	base := base.NewBaseDendrite(cfg, "Tests")

From 304acd7adc4929f64af930d0b5801e33ba101193 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 1 Sep 2022 14:49:42 +0100
Subject: [PATCH 317/391] Version 0.9.6 (#2689)

Changelog and version bump.
---
 CHANGES.md          | 20 ++++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 0f57bffcb..7d0575663 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,25 @@
 # Changelog
 
+## Dendrite 0.9.6 (2022-09-01)
+
+### Features
+
+* The appservice API has been refactored for improved performance and stability
+  * The appservice database has been deprecated, as the roomserver output stream is now used as the data source instead
+* The `generate-config` tool has been updated to support additional scenarios, i.e. for CI configuration generation and generating both monolith and polylith skeleton config files
+
+### Fixes
+
+* The username length check has been fixed on new account creation
+* The length of the `type`, `sender`, `state_key` and `room_id` fields in events are now verified by number of codepoints rather than bytes, fixing the "Cat Overflow" bug
+* UTF-16 surrogate handling in the canonical JSON implementation has been fixed
+* A race condition when starting the keyserver has been fixed
+* A race condition when configuring HTTP servers and routing at startup has been fixed
+* A bug where the incorrect limit was used for lazy-loading memberships has been fixed
+* The number of push notifications will now be sent to the push gateway
+* A missing index causing slow performance on the sync API send-to-device table has been added (contributed by [PiotrKozimor](https://github.com/PiotrKozimor))
+* Event auth will now correctly check for the existence of the `"creator"` field in create events
+
 ## Dendrite 0.9.5 (2022-08-25)
 
 ### Fixes
diff --git a/internal/version.go b/internal/version.go
index 108b8ab0f..c9a9f3af5 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 5
+	VersionPatch = 6
 	VersionTag   = "" // example: "rc1"
 )
 

From fea869b41fc2a9f4db70ef664442a856c095f7c7 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 1 Sep 2022 17:12:27 +0100
Subject: [PATCH 318/391] Update P2P demos

---
 build/gobind-pinecone/monolith.go           | 43 ++++++----
 build/gobind-yggdrasil/monolith.go          | 42 +++++++++-
 cmd/dendrite-demo-yggdrasil/main.go         | 59 +++++++++++---
 cmd/dendrite-demo-yggdrasil/yggconn/node.go | 48 +++++------
 go.mod                                      | 19 +++--
 go.sum                                      | 88 +++++++++------------
 6 files changed, 176 insertions(+), 123 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 44eab4cd7..b51e97786 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -22,10 +22,10 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -45,6 +45,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -204,27 +205,37 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
 
 // nolint:gocyclo
 func (m *DendriteMonolith) Start() {
-	var err error
 	var sk ed25519.PrivateKey
 	var pk ed25519.PublicKey
-	keyfile := fmt.Sprintf("%s/p2p.key", m.StorageDirectory)
-	if _, err = os.Stat(keyfile); os.IsNotExist(err) {
-		if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
-			panic(err)
+
+	keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
+	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
+		oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
+		if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
+			if err = test.NewMatrixKey(keyfile); err != nil {
+				panic("failed to generate a new PEM key: " + err.Error())
+			}
+			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+				panic("failed to load PEM key: " + err.Error())
+			}
+		} else {
+			if sk, err = os.ReadFile(oldkeyfile); err != nil {
+				panic("failed to read the old private key: " + err.Error())
+			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
+			if err = test.SaveMatrixKey(keyfile, sk); err != nil {
+				panic("failed to convert the private key to PEM format: " + err.Error())
+			}
 		}
-		if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil {
-			panic(err)
+	} else {
+		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+			panic("failed to load PEM key: " + err.Error())
 		}
-	} else if err == nil {
-		if sk, err = ioutil.ReadFile(keyfile); err != nil {
-			panic(err)
-		}
-		if len(sk) != ed25519.PrivateKeySize {
-			panic("the private key is not long enough")
-		}
-		pk = sk.Public().(ed25519.PublicKey)
 	}
 
+	var err error
 	m.listener, err = net.Listen("tcp", "localhost:65432")
 	if err != nil {
 		panic(err)
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index 896f3d14f..c671ae098 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -2,10 +2,14 @@ package gobind
 
 import (
 	"context"
+	"crypto/ed25519"
 	"crypto/tls"
+	"encoding/hex"
 	"fmt"
 	"net"
 	"net/http"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -22,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
@@ -63,19 +68,50 @@ func (m *DendriteMonolith) DisconnectMulticastPeers() {
 }
 
 func (m *DendriteMonolith) Start() {
+	var pk ed25519.PublicKey
+	var sk ed25519.PrivateKey
+
 	m.logger = logrus.Logger{
 		Out: BindLogger{},
 	}
 	m.logger.SetOutput(BindLogger{})
 	logrus.SetOutput(BindLogger{})
 
+	keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
+	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
+		oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
+		if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
+			if err = test.NewMatrixKey(keyfile); err != nil {
+				panic("failed to generate a new PEM key: " + err.Error())
+			}
+			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+				panic("failed to load PEM key: " + err.Error())
+			}
+		} else {
+			if sk, err = os.ReadFile(oldkeyfile); err != nil {
+				panic("failed to read the old private key: " + err.Error())
+			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
+			if err := test.SaveMatrixKey(keyfile, sk); err != nil {
+				panic("failed to convert the private key to PEM format: " + err.Error())
+			}
+		}
+	} else {
+		var err error
+		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+			panic("failed to load PEM key: " + err.Error())
+		}
+	}
+
 	var err error
 	m.listener, err = net.Listen("tcp", "localhost:65432")
 	if err != nil {
 		panic(err)
 	}
 
-	ygg, err := yggconn.Setup("dendrite", m.StorageDirectory, "")
+	ygg, err := yggconn.Setup(sk, "dendrite", m.StorageDirectory, "", "")
 	if err != nil {
 		panic(err)
 	}
@@ -86,8 +122,8 @@ func (m *DendriteMonolith) Start() {
 		Generate:   true,
 		Monolithic: true,
 	})
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Global.PrivateKey = ygg.PrivateKey()
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory))
 	cfg.Global.JetStream.InMemory = true
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index d2fddf8bc..7445b6bfe 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -16,7 +16,9 @@ package main
 
 import (
 	"context"
+	"crypto/ed25519"
 	"crypto/tls"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"net"
@@ -42,6 +44,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/mscs"
+	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/sirupsen/logrus"
 
@@ -49,19 +52,18 @@ import (
 )
 
 var (
-	instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance")
-	instancePort = flag.Int("port", 8008, "the port that the client API will listen on")
-	instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to")
+	instanceName   = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance")
+	instancePort   = flag.Int("port", 8008, "the port that the client API will listen on")
+	instancePeer   = flag.String("peer", "", "the static Yggdrasil peers to connect to, comma separated-list")
+	instanceListen = flag.String("listen", "tcp://:0", "the port Yggdrasil peers can connect to")
 )
 
 func main() {
 	flag.Parse()
 	internal.SetupPprof()
 
-	ygg, err := yggconn.Setup(*instanceName, ".", *instancePeer)
-	if err != nil {
-		panic(err)
-	}
+	var pk ed25519.PublicKey
+	var sk ed25519.PrivateKey
 
 	// iterate through the cli args and check if the config flag was set
 	configFlagSet := false
@@ -74,6 +76,34 @@ func main() {
 
 	cfg := &config.Dendrite{}
 
+	keyfile := *instanceName + ".pem"
+	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
+		oldkeyfile := *instanceName + ".key"
+		if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
+			if err = test.NewMatrixKey(keyfile); err != nil {
+				panic("failed to generate a new PEM key: " + err.Error())
+			}
+			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+				panic("failed to load PEM key: " + err.Error())
+			}
+		} else {
+			if sk, err = os.ReadFile(oldkeyfile); err != nil {
+				panic("failed to read the old private key: " + err.Error())
+			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
+			if err := test.SaveMatrixKey(keyfile, sk); err != nil {
+				panic("failed to convert the private key to PEM format: " + err.Error())
+			}
+		}
+	} else {
+		var err error
+		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+			panic("failed to load PEM key: " + err.Error())
+		}
+	}
+
 	// use custom config if config flag is set
 	if configFlagSet {
 		cfg = setup.ParseFlags(true)
@@ -82,6 +112,7 @@ func main() {
 			Generate:   true,
 			Monolithic: true,
 		})
+		cfg.Global.PrivateKey = sk
 		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
 		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
@@ -93,19 +124,23 @@ func main() {
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
 		cfg.ClientAPI.RegistrationDisabled = false
 		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-		if err = cfg.Derive(); err != nil {
+		if err := cfg.Derive(); err != nil {
 			panic(err)
 		}
 	}
 
-	// always override ServerName, PrivateKey and KeyID
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Global.PrivateKey = ygg.PrivateKey()
-	cfg.Global.KeyID = signing.KeyID
+	pk = sk.Public().(ed25519.PublicKey)
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck
 
+	ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
+	if err != nil {
+		panic(err)
+	}
+
 	federation := ygg.CreateFederationClient(base)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
index ff3c73ec8..83b4cdf9e 100644
--- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go
@@ -18,15 +18,13 @@ import (
 	"context"
 	"crypto/ed25519"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
-	"log"
 	"net"
-	"os"
 	"strings"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/neilalexander/utp"
+	"github.com/sirupsen/logrus"
 
 	ironwoodtypes "github.com/Arceliar/ironwood/types"
 	yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config"
@@ -57,48 +55,38 @@ func (n *Node) DialerContext(ctx context.Context, _, address string) (net.Conn,
 	return n.utpSocket.DialAddrContext(ctx, pk)
 }
 
-func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
+func Setup(sk ed25519.PrivateKey, instanceName, storageDirectory, peerURI, listenURI string) (*Node, error) {
 	n := &Node{
 		core:      &yggdrasilcore.Core{},
 		config:    yggdrasildefaults.GenerateConfig(),
 		multicast: &yggdrasilmulticast.Multicast{},
-		log:       gologme.New(os.Stdout, "YGG ", log.Flags()),
+		log:       gologme.New(logrus.StandardLogger().Writer(), "", 0),
 		incoming:  make(chan net.Conn),
 	}
 
-	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
-	if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
-		yggconf, e := os.ReadFile(yggfile)
-		if e != nil {
-			panic(err)
-		}
-		if err := json.Unmarshal([]byte(yggconf), &n.config); err != nil {
-			panic(err)
-		}
+	options := []yggdrasilcore.SetupOption{
+		yggdrasilcore.AdminListenAddress("none"),
+	}
+	if listenURI != "" {
+		options = append(options, yggdrasilcore.ListenAddress(listenURI))
 	}
-
-	n.config.Peers = []string{}
 	if peerURI != "" {
-		n.config.Peers = append(n.config.Peers, peerURI)
+		for _, uri := range strings.Split(peerURI, ",") {
+			options = append(options, yggdrasilcore.Peer{
+				URI: uri,
+			})
+		}
 	}
-	n.config.AdminListen = "none"
 
-	j, err := json.MarshalIndent(n.config, "", "  ")
-	if err != nil {
+	var err error
+	if n.core, err = yggdrasilcore.New(sk, options...); err != nil {
 		panic(err)
 	}
-	if e := os.WriteFile(yggfile, j, 0600); e != nil {
-		n.log.Printf("Couldn't write private key to file '%s': %s\n", yggfile, e)
-	}
-
 	n.log.EnableLevel("error")
 	n.log.EnableLevel("warn")
 	n.log.EnableLevel("info")
-	if err = n.core.Start(n.config, n.log); err != nil {
-		panic(err)
-	}
-	n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core)
-	if err != nil {
+	n.core.SetLogger(n.log)
+	if n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core); err != nil {
 		panic(err)
 	}
 	if err = n.multicast.Init(n.core, n.config, n.log, nil); err != nil {
@@ -108,7 +96,7 @@ func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
 		panic(err)
 	}
 
-	n.log.Println("Public key:", n.core.PublicKey())
+	n.log.Printf("Public key: %x", n.core.PublicKey())
 	go n.listenFromYgg()
 
 	return n, nil
diff --git a/go.mod b/go.mod
index 8bf8f454d..67a7771ec 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/matrix-org/dendrite
 
 require (
-	github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98
+	github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/MFAshby/stdemuxerhook v1.0.0
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2
-	github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9
+	github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
 	github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c
@@ -40,12 +40,12 @@ require (
 	github.com/tidwall/sjson v1.2.4
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
-	github.com/yggdrasil-network/yggdrasil-go v0.4.3
+	github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c
 	go.uber.org/atomic v1.9.0
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
 	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
-	golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
-	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
+	golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105
+	golang.org/x/net v0.0.0-20220728211354-c7608f3a8462
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
@@ -64,7 +64,7 @@ require (
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/frankban/quicktest v1.14.3 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
@@ -87,7 +87,7 @@ require (
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.17.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
@@ -98,12 +98,11 @@ require (
 	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
-	golang.org/x/tools v0.1.10 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
diff --git a/go.sum b/go.sum
index 8b8baabcf..dcfed5eaa 100644
--- a/go.sum
+++ b/go.sum
@@ -38,9 +38,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
-github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 h1:PsaZb47k7WB1V+AlGpb+W7SM+ZOhp16vVevg5gl9YkU=
-github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
+github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a h1:yfbnOyqPcx2gi5cFIJ2rlPz5M6rFPHT/c8FgZmFjCdc=
+github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ=
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -59,10 +58,8 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
-github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
+github.com/RyanCarrier/dijkstra v1.1.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
 github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
-github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
-github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -98,7 +95,6 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -132,8 +128,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
-github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
@@ -142,8 +136,9 @@ github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03D
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
 github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -218,7 +213,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo=
 github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4=
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -260,7 +254,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORR
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -269,10 +262,8 @@ github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
@@ -324,8 +315,6 @@ github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lucas-clemente/quic-go v0.28.1 h1:Uo0lvVxWg5la9gflIF9lwa39ONq85Xq2D91YNEIslzU=
 github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
-github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
-github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
@@ -345,17 +334,13 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 h1:esbNn9hg//tAStA6TogatAJAursw23A+yfVRQsdiv70=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
-github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY=
-github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE=
+github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ=
+github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
-github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
 github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA=
@@ -368,7 +353,6 @@ github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
 github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -410,8 +394,9 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
@@ -459,8 +444,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -535,16 +518,17 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/yggdrasil-network/yggdrasil-go v0.4.3 h1:LNS7kNpKzFlxQ9xmD5tfmMEvzwa+utBoD6pV9t2a8q4=
-github.com/yggdrasil-network/yggdrasil-go v0.4.3/go.mod h1:A1/8kOQT7vzBxlkQtLf1KzJR0cbfL/2zjOCiYOAdjjo=
+github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c h1:/cTmA6pV2Z20BT/FGSmnb5BmJ8eRbDP0HbCB5IO1aKw=
+github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c/go.mod h1:cIwhYwX9yT9Bcei59O0oOBSaj+kQP+9aVQUMWHh5R00=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -569,6 +553,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -603,10 +588,8 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mobile v0.0.0-20220112015953-858099ff7816/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
-golang.org/x/mobile v0.0.0-20220325161704-447654d348e3/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
-golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd h1:x1GptNaTtxPAlTVIAJk61fuXg0y17h09DTxyb+VNC/k=
-golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
+golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 h1:3vUV5x5+3LfQbgk7paCM6INOaJG9xXQbn79xoNkwfIk=
+golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -614,8 +597,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -657,11 +640,12 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 h1:UreQrH7DbFXSi9ZFox6FNT3WBooWmdANpU+IfkT1T4I=
+golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -682,6 +666,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -730,12 +715,10 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -743,13 +726,17 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80=
 golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -822,19 +809,16 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
-golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
-golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326/go.mod h1:SDoazCvdy7RDjBPNEMBwrXhomlmtG7svs8mgwWEqtVI=
-golang.zx2c4.com/wireguard v0.0.0-20211012062646-82d2aa87aa62/go.mod h1:id8Oh3eCCmpj9uVGWVjsUAl6UPX5ysMLzu6QxJU2UOU=
-golang.zx2c4.com/wireguard v0.0.0-20211017052713-f87e87af0d9a/go.mod h1:id8Oh3eCCmpj9uVGWVjsUAl6UPX5ysMLzu6QxJU2UOU=
-golang.zx2c4.com/wireguard/windows v0.4.12/go.mod h1:PW4y+d9oY83XU9rRwRwrJDwEMuhVjMxu2gfD1cfzS7w=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
+golang.zx2c4.com/wireguard v0.0.0-20220703234212-c31a7b1ab478/go.mod h1:bVQfyl2sCM/QIIGHpWbFGfHPuDvqnCNkT6MQLTCjO/U=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=

From 350a5e5393503780c6ab0a8983daae5301e845e2 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 2 Sep 2022 16:03:33 +0200
Subject: [PATCH 319/391] Only send (invite) events to the AS if it's
 interested in it (#2692)

This should hopefully fix #2691 (and maybe also #2690)
---
 appservice/consumers/roomserver.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index a30944e75..a4bcfa7d9 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 			}
 
 		case api.OutputTypeNewInviteEvent:
-			if output.NewInviteEvent == nil {
+			if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) {
 				continue
 			}
 			events = append(events, output.NewInviteEvent.Event)
@@ -281,7 +281,9 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e
 			case err != nil:
 				continue
 			case membership.Membership == gomatrixserverlib.Join:
-				return true
+				if appservice.IsInterestedInUserID(*ev.StateKey) {
+					return true
+				}
 			}
 		}
 	} else {

From d1f87e63f1b744825f40e59b8202933608480a70 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 5 Sep 2022 13:16:14 +0100
Subject: [PATCH 320/391] Move `SetLatestEvents` call

---
 roomserver/internal/input/input_latest_events.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index d6efad79d..ec6626720 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -178,6 +178,10 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 		u.newStateNID = u.oldStateNID
 	}
 
+	if err = u.updater.SetLatestEvents(u.roomInfo.RoomNID, u.latest, u.stateAtEvent.EventNID, u.newStateNID); err != nil {
+		return fmt.Errorf("u.updater.SetLatestEvents: %w", err)
+	}
+
 	update, err := u.makeOutputNewRoomEvent()
 	if err != nil {
 		return fmt.Errorf("u.makeOutputNewRoomEvent: %w", err)
@@ -196,10 +200,6 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 		return fmt.Errorf("u.api.WriteOutputEvents: %w", err)
 	}
 
-	if err = u.updater.SetLatestEvents(u.roomInfo.RoomNID, u.latest, u.stateAtEvent.EventNID, u.newStateNID); err != nil {
-		return fmt.Errorf("u.updater.SetLatestEvents: %w", err)
-	}
-
 	if err = u.updater.MarkEventAsSent(u.stateAtEvent.EventNID); err != nil {
 		return fmt.Errorf("u.updater.MarkEventAsSent: %w", err)
 	}

From 1c1d09abd4009e3fcc76b71ad158fb16e5e776cc Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 5 Sep 2022 14:17:04 +0100
Subject: [PATCH 321/391] Optimise `resolve-state` tool

---
 cmd/resolve-state/main.go | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go
index 6852ab4fb..f8bb130c7 100644
--- a/cmd/resolve-state/main.go
+++ b/cmd/resolve-state/main.go
@@ -76,9 +76,14 @@ func main() {
 			panic(err)
 		}
 
-		var eventNIDs []types.EventNID
+		eventNIDMap := map[types.EventNID]struct{}{}
 		for _, entry := range append(removed, added...) {
-			eventNIDs = append(eventNIDs, entry.EventNID)
+			eventNIDMap[entry.EventNID] = struct{}{}
+		}
+
+		eventNIDs := make([]types.EventNID, 0, len(eventNIDMap))
+		for eventNID := range eventNIDMap {
+			eventNIDs = append(eventNIDs, eventNID)
 		}
 
 		var eventEntries []types.Event
@@ -129,12 +134,17 @@ func main() {
 		stateEntries = append(stateEntries, entries...)
 	}
 
-	var eventNIDs []types.EventNID
+	eventNIDMap := map[types.EventNID]struct{}{}
 	for _, entry := range stateEntries {
-		eventNIDs = append(eventNIDs, entry.EventNID)
+		eventNIDMap[entry.EventNID] = struct{}{}
 	}
 
-	fmt.Println("Fetching", len(eventNIDs), "state events")
+	eventNIDs := make([]types.EventNID, 0, len(eventNIDMap))
+	for eventNID := range eventNIDMap {
+		eventNIDs = append(eventNIDs, eventNID)
+	}
+
+	fmt.Println("Fetching", len(eventNIDMap), "state events")
 	eventEntries, err := roomserverDB.Events(ctx, eventNIDs)
 	if err != nil {
 		panic(err)

From bea3dbe77bac078f07927df54c63f145df8a0caf Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 5 Sep 2022 17:00:54 +0100
Subject: [PATCH 322/391] State and auth tweaks (update to
 matrix-org/gomatrixserverlib#335)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 67a7771ec..b7f1920ca 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29
 	github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index dcfed5eaa..3212c5d70 100644
--- a/go.sum
+++ b/go.sum
@@ -332,8 +332,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 h1:esbNn9hg//tAStA6TogatAJAursw23A+yfVRQsdiv70=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 h1:55lVTCUNuq/WvTH6usVwrliqnxbziOjOCHeQs1tcSeg=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From ecee5f10f49487f0b6628fea065914ea3e685a79 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 5 Sep 2022 17:08:54 +0100
Subject: [PATCH 323/391] Tweak logging for detected state resets

---
 roomserver/internal/input/input_latest_events.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index ec6626720..7a4105ab8 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -275,7 +275,7 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
 	}
 
-	if removed := len(u.removed) - len(u.added); removed > 0 {
+	if removed := len(u.removed) - len(u.added); !u.rewritesState && removed > 0 {
 		logrus.WithFields(logrus.Fields{
 			"event_id":      u.event.EventID(),
 			"room_id":       u.event.RoomID(),
@@ -283,7 +283,7 @@ func (u *latestEventsUpdater) latestState() error {
 			"new_state_nid": u.newStateNID,
 			"old_latest":    u.oldLatest.EventIDs(),
 			"new_latest":    u.latest.EventIDs(),
-		}).Errorf("Unexpected state deletion (removing %d events)", removed)
+		}).Warnf("State reset detected (removing %d events)", removed)
 	}
 
 	// Also work out the state before the event removes and the event

From cd22ba22b01fe5197bf8e4630c8414dd413225af Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 5 Sep 2022 17:25:11 +0100
Subject: [PATCH 324/391] Improve Sentry reporting

---
 internal/httputil/httpapi.go                     |  1 +
 roomserver/internal/input/input.go               |  6 ++++++
 roomserver/internal/input/input_latest_events.go | 10 ++++++++++
 3 files changed, 17 insertions(+)

diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index e0436c60a..ab8319bbb 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -60,6 +60,7 @@ func MakeAuthAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			hub.Scope().SetTag("username", device.UserID)
 			hub.Scope().SetTag("user_id", device.UserID)
 			hub.Scope().SetTag("device_id", device.ID)
 		}
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 429cc4bd2..a8a3e0248 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -188,6 +188,9 @@ func (w *worker) _next() {
 	// Look up what the next event is that's waiting to be processed.
 	ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute)
 	defer cancel()
+	if scope := sentry.CurrentHub().Scope(); scope != nil {
+		scope.SetTag("room_id", w.roomID)
+	}
 	msgs, err := w.subscription.Fetch(1, nats.Context(ctx))
 	switch err {
 	case nil:
@@ -239,6 +242,9 @@ func (w *worker) _next() {
 		return
 	}
 
+	if scope := sentry.CurrentHub().Scope(); scope != nil {
+		scope.SetTag("event_id", inputRoomEvent.Event.EventID())
+	}
 	roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Inc()
 	defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec()
 
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 7a4105ab8..7e42db1b7 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/opentracing/opentracing-go"
@@ -284,6 +285,15 @@ func (u *latestEventsUpdater) latestState() error {
 			"old_latest":    u.oldLatest.EventIDs(),
 			"new_latest":    u.latest.EventIDs(),
 		}).Warnf("State reset detected (removing %d events)", removed)
+		sentry.WithScope(func(scope *sentry.Scope) {
+			scope.SetLevel("warning")
+			scope.SetTag("event_id", u.event.EventID())
+			scope.SetTag("old_state_nid", fmt.Sprintf("%d", u.oldStateNID))
+			scope.SetTag("new_state_nid", fmt.Sprintf("%d", u.newStateNID))
+			scope.SetTag("old_latest", u.oldLatest.EventIDs())
+			scope.SetTag("new_latest", u.latest.EventIDs())
+			sentry.CaptureMessage("State reset detected")
+		})
 	}
 
 	// Also work out the state before the event removes and the event

From 5992b4c7ed2779e12c8b9da7bc96b2b6398cf04c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 5 Sep 2022 17:35:43 +0100
Subject: [PATCH 325/391] Set Sentry user where known

---
 internal/httputil/httpapi.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index ab8319bbb..5612d78c1 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -60,7 +60,10 @@ func MakeAuthAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
-			hub.Scope().SetTag("username", device.UserID)
+			hub.Scope().SetUser(sentry.User{
+				Username:  device.UserID,
+				IPAddress: req.RemoteAddr,
+			})
 			hub.Scope().SetTag("user_id", device.UserID)
 			hub.Scope().SetTag("device_id", device.ID)
 		}

From fd7661f69cfecb426f0170a27a4e414b178a1cdf Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 5 Sep 2022 17:37:17 +0100
Subject: [PATCH 326/391] Tweak Sentry again

---
 internal/httputil/httpapi.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
index 5612d78c1..36dcaf453 100644
--- a/internal/httputil/httpapi.go
+++ b/internal/httputil/httpapi.go
@@ -61,8 +61,7 @@ func MakeAuthAPI(
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
 			hub.Scope().SetUser(sentry.User{
-				Username:  device.UserID,
-				IPAddress: req.RemoteAddr,
+				Username: device.UserID,
 			})
 			hub.Scope().SetTag("user_id", device.UserID)
 			hub.Scope().SetTag("device_id", device.ID)

From 70cd9a902c84f15817b191999340d003ccbc41eb Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 6 Sep 2022 17:19:21 +0100
Subject: [PATCH 327/391] State resolution fixes in `applyAndAuthEvent` (update
 to matrix-org/gomatrixserverlib#336)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index b7f1920ca..623e8f397 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267
 	github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 3212c5d70..d30bc39f3 100644
--- a/go.sum
+++ b/go.sum
@@ -332,8 +332,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 h1:55lVTCUNuq/WvTH6usVwrliqnxbziOjOCHeQs1tcSeg=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267 h1:+Ix9XC5ah7cofDaEFHWrupBQWUQEqz9bOkHcZvhzw2s=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 489ccc1c600836fda8b66b783e1f510e67c3666d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 6 Sep 2022 17:25:17 +0100
Subject: [PATCH 328/391] Stronger default power levels

---
 internal/eventutil/eventcontent.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/internal/eventutil/eventcontent.go b/internal/eventutil/eventcontent.go
index 4ecb5fb56..e3c80f1d0 100644
--- a/internal/eventutil/eventcontent.go
+++ b/internal/eventutil/eventcontent.go
@@ -53,6 +53,9 @@ func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLev
 		"m.room.history_visibility": 100,
 		"m.room.canonical_alias":    50,
 		"m.room.avatar":             50,
+		"m.room.tombstone":          100,
+		"m.room.encryption":         100,
+		"m.room.server_acl":         100,
 	}
 	c.Users = map[string]int64{roomCreator: 100}
 	return c

From 847032df36086ff6f383bcc4c332c391e86eaec8 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 7 Sep 2022 09:11:36 +0100
Subject: [PATCH 329/391] Update to matrix-org/gomatrixserverlib@637a173

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 623e8f397..c737e467c 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661
 	github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index d30bc39f3..0b331befe 100644
--- a/go.sum
+++ b/go.sum
@@ -332,8 +332,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267 h1:+Ix9XC5ah7cofDaEFHWrupBQWUQEqz9bOkHcZvhzw2s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661 h1:dww9rH0HVfAO9JOBD1nxq26GHKbEw07thAJTu1DrAQs=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 440eb0f3a2e5aecddebe3e093d5740ce5c4505c2 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 7 Sep 2022 11:44:27 +0200
Subject: [PATCH 330/391] Handle errors differently in the `DeviceListUpdater`
 (#2695)

`If a device list update goes missing, the server resyncs on the next
one` was failing because a previous test would receive a `waitTime` of
1h, resulting in the test timing out.
This now tries to handle the returned errors differently, e.g. by using
the default `waitTime` of 2s. Also doesn't try further users in the
list, if one of the errors would cause a longer `waitTime`.
---
 federationapi/storage/shared/storage_edus.go |  6 +--
 internal/httputil/http.go                    |  6 +--
 keyserver/internal/device_list_update.go     | 48 +++++++++++++++-----
 sytest-whitelist                             |  3 +-
 4 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/federationapi/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go
index ce9632ed3..e0c740c11 100644
--- a/federationapi/storage/shared/storage_edus.go
+++ b/federationapi/storage/shared/storage_edus.go
@@ -53,9 +53,9 @@ func (d *Database) AssociateEDUWithDestination(
 		// Keep EDUs for at least x minutes before deleting them
 		expiresAt = gomatrixserverlib.AsTimestamp(time.Now().Add(duration))
 	}
-	// We forcibly set m.direct_to_device events to 0, as we always want them
-	// to be delivered. (required for E2EE)
-	if eduType == gomatrixserverlib.MDirectToDevice {
+	// We forcibly set m.direct_to_device and m.device_list_update events
+	// to 0, as we always want them to be delivered. (required for E2EE)
+	if eduType == gomatrixserverlib.MDirectToDevice || eduType == gomatrixserverlib.MDeviceListUpdate {
 		expiresAt = 0
 	}
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
diff --git a/internal/httputil/http.go b/internal/httputil/http.go
index 1e07ee33c..ad26de512 100644
--- a/internal/httputil/http.go
+++ b/internal/httputil/http.go
@@ -24,7 +24,7 @@ import (
 	"net/url"
 	"strings"
 
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 )
 
@@ -81,8 +81,8 @@ func PostJSON[reqtype, restype any, errtype error](
 			return fmt.Errorf("HTTP %d from %s (no response body)", res.StatusCode, apiURL)
 		}
 		var reserr errtype
-		if err = json.Unmarshal(body, reserr); err != nil {
-			return fmt.Errorf("HTTP %d from %s", res.StatusCode, apiURL)
+		if err = json.Unmarshal(body, &reserr); err != nil {
+			return fmt.Errorf("HTTP %d from %s - %w", res.StatusCode, apiURL, err)
 		}
 		return reserr
 	}
diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 304b67b23..8530f9078 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -19,9 +19,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"hash/fnv"
+	"net"
 	"sync"
 	"time"
 
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -388,6 +390,8 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 		return waitTime, true
 	}
 	failCount := 0
+
+userLoop:
 	for _, userID := range userIDs {
 		if ctx.Err() != nil {
 			// we've timed out, give up and go to the back of the queue to let another server be processed.
@@ -397,19 +401,35 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 		res, err := u.fedClient.GetUserDevices(ctx, serverName, userID)
 		if err != nil {
 			failCount += 1
-			fcerr, ok := err.(*fedsenderapi.FederationClientError)
-			if ok {
-				if fcerr.RetryAfter > 0 {
-					waitTime = fcerr.RetryAfter
-				} else if fcerr.Blacklisted {
+			switch e := err.(type) {
+			case *fedsenderapi.FederationClientError:
+				if e.RetryAfter > 0 {
+					waitTime = e.RetryAfter
+					break userLoop
+				} else if e.Blacklisted {
 					waitTime = time.Hour * 8
-				} else {
-					// For all other errors (DNS resolution, network etc.) wait 1 hour.
-					waitTime = time.Hour
+					break userLoop
 				}
-			} else {
-				waitTime = time.Hour
-				logger.WithError(err).WithField("user_id", userID).Debug("GetUserDevices returned unknown error type")
+			case net.Error:
+				// Use the default waitTime, if it's a timeout.
+				// It probably doesn't make sense to try further users.
+				if !e.Timeout() {
+					waitTime = time.Minute * 10
+					logrus.WithError(e).Error("GetUserDevices returned net.Error")
+					break userLoop
+				}
+			case gomatrix.HTTPError:
+				// The remote server returned an error, give it some time to recover
+				if e.Code >= 500 {
+					waitTime = time.Minute * 10
+					logrus.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError")
+					break userLoop
+				}
+			default:
+				// Something else failed
+				waitTime = time.Minute * 10
+				logger.WithError(err).WithField("user_id", userID).Debugf("GetUserDevices returned unknown error type: %T", err)
+				break userLoop
 			}
 			continue
 		}
@@ -437,7 +457,11 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 		}
 	}
 	if failCount > 0 {
-		logger.WithField("total", len(userIDs)).WithField("failed", failCount).WithField("wait", waitTime).Warn("Failed to query device keys for some users")
+		logger.WithFields(logrus.Fields{
+			"total":   len(userIDs),
+			"failed":  failCount,
+			"skipped": len(userIDs) - failCount,
+		}).Warn("Failed to query device keys for some users")
 	}
 	for _, userID := range userIDs {
 		// always clear the channel to unblock Update calls regardless of success/failure
diff --git a/sytest-whitelist b/sytest-whitelist
index 5c8896b99..fc10bd3cc 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -741,4 +741,5 @@ Newly joined room includes presence in incremental sync
 User in private room doesn't appear in user directory
 User joining then leaving public room appears and dissappears from directory
 User in remote room doesn't appear in user directory after server left room
-User in shared private room does appear in user directory until leave
\ No newline at end of file
+User in shared private room does appear in user directory until leave
+If a device list update goes missing, the server resyncs on the next one
\ No newline at end of file

From 2cfcfddeccb435709824e1beb8d6731eff7c8b86 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 7 Sep 2022 11:45:12 +0200
Subject: [PATCH 331/391] Add a SigningKeyUpdate producer (#2697)

This adds a new stream for signing key updates, this should ensure we
don't lose any updates over federation.
---
 federationapi/federationapi.go          |   7 +-
 federationapi/producers/syncapi.go      |  20 ++++-
 federationapi/routing/send.go           |  47 ++--------
 keyserver/consumers/signingkeyupdate.go | 110 ++++++++++++++++++++++++
 keyserver/keyserver.go                  |  10 ++-
 setup/jetstream/streams.go              |   6 ++
 6 files changed, 153 insertions(+), 47 deletions(-)
 create mode 100644 keyserver/consumers/signingkeyupdate.go

diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index ff01b1952..4a13c9d9b 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -18,6 +18,8 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
@@ -33,10 +35,10 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/federationapi/routing"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
@@ -66,6 +68,7 @@ func AddPublicRoutes(
 		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
 		TopicPresenceEvent:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
 		TopicDeviceListUpdate:  cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
+		TopicSigningKeyUpdate:  cfg.Matrix.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
 		ServerName:             cfg.Matrix.ServerName,
 		UserAPI:                userAPI,
 	}
diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go
index 43dd08dd8..0825bccbb 100644
--- a/federationapi/producers/syncapi.go
+++ b/federationapi/producers/syncapi.go
@@ -21,12 +21,13 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/syncapi/types"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // SyncAPIProducer produces events for the sync API server to consume
@@ -36,6 +37,7 @@ type SyncAPIProducer struct {
 	TopicTypingEvent       string
 	TopicPresenceEvent     string
 	TopicDeviceListUpdate  string
+	TopicSigningKeyUpdate  string
 	JetStream              nats.JetStreamContext
 	ServerName             gomatrixserverlib.ServerName
 	UserAPI                userapi.UserInternalAPI
@@ -178,3 +180,15 @@ func (p *SyncAPIProducer) SendDeviceListUpdate(
 	_, err = p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err
 }
+
+func (p *SyncAPIProducer) SendSigningKeyUpdate(
+	ctx context.Context, data gomatrixserverlib.RawJSON, origin gomatrixserverlib.ServerName,
+) (err error) {
+	m := nats.NewMsg(p.TopicSigningKeyUpdate)
+	m.Header.Set("origin", string(origin))
+	m.Data = data
+
+	log.Debugf("Sending signing key update")
+	_, err = p.JetStream.PublishMsg(m, nats.Context(ctx))
+	return err
+}
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 87b6fa33e..3d9319964 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -22,6 +22,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/producers"
@@ -31,10 +36,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -391,7 +392,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				}
 			}
 		case types.MSigningKeyUpdate:
-			if err := t.processSigningKeyUpdate(ctx, e); err != nil {
+			if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil {
 				logrus.WithError(err).Errorf("Failed to process signing key update")
 			}
 		case gomatrixserverlib.MPresence:
@@ -431,42 +432,6 @@ func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) e
 	return nil
 }
 
-func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverlib.EDU) error {
-	var updatePayload keyapi.CrossSigningKeyUpdate
-	if err := json.Unmarshal(e.Content, &updatePayload); err != nil {
-		util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
-			"user_id": updatePayload.UserID,
-		}).Debug("Failed to unmarshal signing key update")
-		return err
-	}
-	if _, serverName, err := gomatrixserverlib.SplitID('@', updatePayload.UserID); err != nil {
-		return nil
-	} else if serverName == t.ourServerName {
-		return nil
-	} else if serverName != t.Origin {
-		return nil
-	}
-	keys := gomatrixserverlib.CrossSigningKeys{}
-	if updatePayload.MasterKey != nil {
-		keys.MasterKey = *updatePayload.MasterKey
-	}
-	if updatePayload.SelfSigningKey != nil {
-		keys.SelfSigningKey = *updatePayload.SelfSigningKey
-	}
-	uploadReq := &keyapi.PerformUploadDeviceKeysRequest{
-		CrossSigningKeys: keys,
-		UserID:           updatePayload.UserID,
-	}
-	uploadRes := &keyapi.PerformUploadDeviceKeysResponse{}
-	if err := t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil {
-		return err
-	}
-	if uploadRes.Error != nil {
-		return uploadRes.Error
-	}
-	return nil
-}
-
 // processReceiptEvent sends receipt events to JetStream
 func (t *txnReq) processReceiptEvent(ctx context.Context,
 	userID, roomID, receiptType string,
diff --git a/keyserver/consumers/signingkeyupdate.go b/keyserver/consumers/signingkeyupdate.go
new file mode 100644
index 000000000..366e259b4
--- /dev/null
+++ b/keyserver/consumers/signingkeyupdate.go
@@ -0,0 +1,110 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
+
+	keyapi "github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/keyserver/internal"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
+)
+
+// SigningKeyUpdateConsumer consumes signing key updates that came in over federation.
+type SigningKeyUpdateConsumer struct {
+	ctx       context.Context
+	jetstream nats.JetStreamContext
+	durable   string
+	topic     string
+	keyAPI    *internal.KeyInternalAPI
+	cfg       *config.KeyServer
+}
+
+// NewSigningKeyUpdateConsumer creates a new SigningKeyUpdateConsumer. Call Start() to begin consuming from key servers.
+func NewSigningKeyUpdateConsumer(
+	process *process.ProcessContext,
+	cfg *config.KeyServer,
+	js nats.JetStreamContext,
+	keyAPI *internal.KeyInternalAPI,
+) *SigningKeyUpdateConsumer {
+	return &SigningKeyUpdateConsumer{
+		ctx:       process.Context(),
+		jetstream: js,
+		durable:   cfg.Matrix.JetStream.Prefixed("KeyServerSigningKeyConsumer"),
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
+		keyAPI:    keyAPI,
+		cfg:       cfg,
+	}
+}
+
+// Start consuming from key servers
+func (t *SigningKeyUpdateConsumer) Start() error {
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, 1,
+		t.onMessage, nats.DeliverAll(), nats.ManualAck(),
+	)
+}
+
+// onMessage is called in response to a message received on the
+// signing key update events topic from the key server.
+func (t *SigningKeyUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
+	msg := msgs[0] // Guaranteed to exist if onMessage is called
+	var updatePayload keyapi.CrossSigningKeyUpdate
+	if err := json.Unmarshal(msg.Data, &updatePayload); err != nil {
+		logrus.WithError(err).Errorf("Failed to read from signing key update input topic")
+		return true
+	}
+	origin := gomatrixserverlib.ServerName(msg.Header.Get("origin"))
+	if _, serverName, err := gomatrixserverlib.SplitID('@', updatePayload.UserID); err != nil {
+		logrus.WithError(err).Error("failed to split user id")
+		return true
+	} else if serverName == t.cfg.Matrix.ServerName {
+		logrus.Warn("dropping device key update from ourself")
+		return true
+	} else if serverName != origin {
+		logrus.Warnf("dropping device key update, %s != %s", serverName, origin)
+		return true
+	}
+
+	keys := gomatrixserverlib.CrossSigningKeys{}
+	if updatePayload.MasterKey != nil {
+		keys.MasterKey = *updatePayload.MasterKey
+	}
+	if updatePayload.SelfSigningKey != nil {
+		keys.SelfSigningKey = *updatePayload.SelfSigningKey
+	}
+	uploadReq := &keyapi.PerformUploadDeviceKeysRequest{
+		CrossSigningKeys: keys,
+		UserID:           updatePayload.UserID,
+	}
+	uploadRes := &keyapi.PerformUploadDeviceKeysResponse{}
+	if err := t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil {
+		logrus.WithError(err).Error("failed to upload device keys")
+		return false
+	}
+	if uploadRes.Error != nil {
+		logrus.WithError(uploadRes.Error).Error("failed to upload device keys")
+		return true
+	}
+
+	return true
+}
diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go
index 75cdaedb4..5124b777e 100644
--- a/keyserver/keyserver.go
+++ b/keyserver/keyserver.go
@@ -16,6 +16,8 @@ package keyserver
 
 import (
 	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
 	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/keyserver/consumers"
@@ -26,7 +28,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/sirupsen/logrus"
 )
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
@@ -72,5 +73,12 @@ func NewInternalAPI(
 		logrus.WithError(err).Panic("failed to start device list consumer")
 	}
 
+	sigConsumer := consumers.NewSigningKeyUpdateConsumer(
+		base.ProcessContext, cfg, js, ap,
+	)
+	if err := sigConsumer.Start(); err != nil {
+		logrus.WithError(err).Panic("failed to start signing key consumer")
+	}
+
 	return ap
 }
diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go
index 110808b1b..a06a12662 100644
--- a/setup/jetstream/streams.go
+++ b/setup/jetstream/streams.go
@@ -17,6 +17,7 @@ const (
 var (
 	InputRoomEvent          = "InputRoomEvent"
 	InputDeviceListUpdate   = "InputDeviceListUpdate"
+	InputSigningKeyUpdate   = "InputSigningKeyUpdate"
 	OutputRoomEvent         = "OutputRoomEvent"
 	OutputSendToDeviceEvent = "OutputSendToDeviceEvent"
 	OutputKeyChangeEvent    = "OutputKeyChangeEvent"
@@ -51,6 +52,11 @@ var streams = []*nats.StreamConfig{
 		Retention: nats.InterestPolicy,
 		Storage:   nats.FileStorage,
 	},
+	{
+		Name:      InputSigningKeyUpdate,
+		Retention: nats.InterestPolicy,
+		Storage:   nats.FileStorage,
+	},
 	{
 		Name:      OutputRoomEvent,
 		Retention: nats.InterestPolicy,

From 4e352390b6827870a2fb19e01c59bd5c267dd7e0 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 7 Sep 2022 12:13:02 +0200
Subject: [PATCH 332/391] Re-add waitTime if we're not blacklisted and no
 RetryAfter was specified.

---
 keyserver/internal/device_list_update.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 8530f9078..b70d6a5e1 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -405,11 +405,12 @@ userLoop:
 			case *fedsenderapi.FederationClientError:
 				if e.RetryAfter > 0 {
 					waitTime = e.RetryAfter
-					break userLoop
 				} else if e.Blacklisted {
 					waitTime = time.Hour * 8
-					break userLoop
+				} else {
+					waitTime = time.Hour
 				}
+				break userLoop
 			case net.Error:
 				// Use the default waitTime, if it's a timeout.
 				// It probably doesn't make sense to try further users.

From 7e8c605f98458f829fc914c3476cf4999e317bcd Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 7 Sep 2022 12:16:04 +0200
Subject: [PATCH 333/391] Avoid unneeded JSON operations (#2698)

We were `json.Unmarshal`ing the EDU and `json.Marshal`ing right before
sending the EDU to the stream. Those are now removed and the consumer
does `json.Unmarshal` once.
---
 federationapi/producers/syncapi.go      | 10 ++-----
 federationapi/routing/send.go           | 22 ++------------
 keyserver/consumers/devicelistupdate.go | 38 ++++++++++++++++---------
 3 files changed, 31 insertions(+), 39 deletions(-)

diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go
index 0825bccbb..86c8c10a3 100644
--- a/federationapi/producers/syncapi.go
+++ b/federationapi/producers/syncapi.go
@@ -167,15 +167,11 @@ func (p *SyncAPIProducer) SendPresence(
 }
 
 func (p *SyncAPIProducer) SendDeviceListUpdate(
-	ctx context.Context, deviceListUpdate *gomatrixserverlib.DeviceListUpdateEvent,
+	ctx context.Context, deviceListUpdate gomatrixserverlib.RawJSON, origin string,
 ) (err error) {
 	m := nats.NewMsg(p.TopicDeviceListUpdate)
-	m.Header.Set(jetstream.UserID, deviceListUpdate.UserID)
-	m.Data, err = json.Marshal(deviceListUpdate)
-	if err != nil {
-		return fmt.Errorf("json.Marshal: %w", err)
-	}
-
+	m.Header.Set("origin", origin)
+	m.Data = deviceListUpdate
 	log.Debugf("Sending device list update: %+v", m.Header)
 	_, err = p.JetStream.PublishMsg(m, nats.Context(ctx))
 	return err
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 3d9319964..a9714c65a 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -359,7 +359,9 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				}
 			}
 		case gomatrixserverlib.MDeviceListUpdate:
-			t.processDeviceListUpdate(ctx, e)
+			if err := t.producer.SendDeviceListUpdate(ctx, e.Content, e.Origin); err != nil {
+				util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate")
+			}
 		case gomatrixserverlib.MReceipt:
 			// https://matrix.org/docs/spec/server_server/r0.1.4#receipts
 			payload := map[string]types.FederationReceiptMRead{}
@@ -454,21 +456,3 @@ func (t *txnReq) processReceiptEvent(ctx context.Context,
 
 	return nil
 }
-
-func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverlib.EDU) {
-	var payload gomatrixserverlib.DeviceListUpdateEvent
-	if err := json.Unmarshal(e.Content, &payload); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal device list update event")
-		return
-	}
-	if _, serverName, err := gomatrixserverlib.SplitID('@', payload.UserID); err != nil {
-		return
-	} else if serverName == t.ourServerName {
-		return
-	} else if serverName != t.Origin {
-		return
-	}
-	if err := t.producer.SendDeviceListUpdate(ctx, &payload); err != nil {
-		util.GetLogger(ctx).WithError(err).WithField("user_id", payload.UserID).Error("failed to InputDeviceListUpdate")
-	}
-}
diff --git a/keyserver/consumers/devicelistupdate.go b/keyserver/consumers/devicelistupdate.go
index d15f94267..575e41281 100644
--- a/keyserver/consumers/devicelistupdate.go
+++ b/keyserver/consumers/devicelistupdate.go
@@ -18,22 +18,24 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/keyserver/internal"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/sirupsen/logrus"
 )
 
 // DeviceListUpdateConsumer consumes device list updates that came in over federation.
 type DeviceListUpdateConsumer struct {
-	ctx       context.Context
-	jetstream nats.JetStreamContext
-	durable   string
-	topic     string
-	updater   *internal.DeviceListUpdater
+	ctx        context.Context
+	jetstream  nats.JetStreamContext
+	durable    string
+	topic      string
+	updater    *internal.DeviceListUpdater
+	serverName gomatrixserverlib.ServerName
 }
 
 // NewDeviceListUpdateConsumer creates a new DeviceListConsumer. Call Start() to begin consuming from key servers.
@@ -44,11 +46,12 @@ func NewDeviceListUpdateConsumer(
 	updater *internal.DeviceListUpdater,
 ) *DeviceListUpdateConsumer {
 	return &DeviceListUpdateConsumer{
-		ctx:       process.Context(),
-		jetstream: js,
-		durable:   cfg.Matrix.JetStream.Prefixed("KeyServerInputDeviceListConsumer"),
-		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
-		updater:   updater,
+		ctx:        process.Context(),
+		jetstream:  js,
+		durable:    cfg.Matrix.JetStream.Prefixed("KeyServerInputDeviceListConsumer"),
+		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
+		updater:    updater,
+		serverName: cfg.Matrix.ServerName,
 	}
 }
 
@@ -69,6 +72,15 @@ func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M
 		logrus.WithError(err).Errorf("Failed to read from device list update input topic")
 		return true
 	}
+	origin := gomatrixserverlib.ServerName(msg.Header.Get("origin"))
+	if _, serverName, err := gomatrixserverlib.SplitID('@', m.UserID); err != nil {
+		return true
+	} else if serverName == t.serverName {
+		return true
+	} else if serverName != origin {
+		return true
+	}
+
 	err := t.updater.Update(ctx, m)
 	if err != nil {
 		logrus.WithFields(logrus.Fields{

From 0d697f675484cfd990af640aad6fa54c44ecd208 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 7 Sep 2022 16:14:09 +0200
Subject: [PATCH 334/391] Add HTTP status code to FederationClientError (#2699)

Also ensures we wait on more HTTP status codes.
---
 federationapi/api/api.go                 |  4 +++-
 federationapi/inthttp/server.go          | 10 ++++++++--
 keyserver/internal/device_list_update.go | 21 +++++++++++++--------
 sytest-blacklist                         |  3 +++
 sytest-whitelist                         |  1 -
 5 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index 292ed55ad..f25538784 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -5,9 +5,10 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/matrix-org/dendrite/federationapi/types"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/federationapi/types"
 )
 
 // FederationInternalAPI is used to query information from the federation sender.
@@ -108,6 +109,7 @@ type FederationClientError struct {
 	Err         string
 	RetryAfter  time.Duration
 	Blacklisted bool
+	Code        int // HTTP Status code from the remote server
 }
 
 func (e FederationClientError) Error() string {
diff --git a/federationapi/inthttp/server.go b/federationapi/inthttp/server.go
index a8b829a71..58ea9ddc6 100644
--- a/federationapi/inthttp/server.go
+++ b/federationapi/inthttp/server.go
@@ -6,10 +6,12 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/internal/httputil"
 )
 
 // AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux.
@@ -229,6 +231,10 @@ func federationClientError(err error) error {
 		return &ferr
 	case *api.FederationClientError:
 		return ferr
+	case gomatrix.HTTPError:
+		return &api.FederationClientError{
+			Code: ferr.Code,
+		}
 	default:
 		return &api.FederationClientError{
 			Err: err.Error(),
diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index b70d6a5e1..41534fe82 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -407,10 +407,13 @@ userLoop:
 					waitTime = e.RetryAfter
 				} else if e.Blacklisted {
 					waitTime = time.Hour * 8
-				} else {
+					break userLoop
+				} else if e.Code >= 300 {
+					// We didn't get a real FederationClientError (e.g. in polylith mode, where gomatrix.HTTPError
+					// are "converted" to FederationClientError), but we probably shouldn't hit them every $waitTime seconds.
 					waitTime = time.Hour
+					break userLoop
 				}
-				break userLoop
 			case net.Error:
 				// Use the default waitTime, if it's a timeout.
 				// It probably doesn't make sense to try further users.
@@ -420,9 +423,10 @@ userLoop:
 					break userLoop
 				}
 			case gomatrix.HTTPError:
-				// The remote server returned an error, give it some time to recover
-				if e.Code >= 500 {
-					waitTime = time.Minute * 10
+				// The remote server returned an error, give it some time to recover.
+				// This is to avoid spamming remote servers, which may not be Matrix servers anymore.
+				if e.Code >= 300 {
+					waitTime = time.Hour
 					logrus.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError")
 					break userLoop
 				}
@@ -459,9 +463,10 @@ userLoop:
 	}
 	if failCount > 0 {
 		logger.WithFields(logrus.Fields{
-			"total":   len(userIDs),
-			"failed":  failCount,
-			"skipped": len(userIDs) - failCount,
+			"total":    len(userIDs),
+			"failed":   failCount,
+			"skipped":  len(userIDs) - failCount,
+			"waittime": waitTime,
 		}).Warn("Failed to query device keys for some users")
 	}
 	for _, userID := range userIDs {
diff --git a/sytest-blacklist b/sytest-blacklist
index bcc345f6e..5b2e973a6 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -49,3 +49,6 @@ Notifications can be viewed with GET /notifications
 
 If remote user leaves room we no longer receive device updates
 Guest users can join guest_access rooms
+
+# This will fail in HTTP API mode, so blacklisted for now
+If a device list update goes missing, the server resyncs on the next one
\ No newline at end of file
diff --git a/sytest-whitelist b/sytest-whitelist
index fc10bd3cc..31940b884 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -742,4 +742,3 @@ User in private room doesn't appear in user directory
 User joining then leaving public room appears and dissappears from directory
 User in remote room doesn't appear in user directory after server left room
 User in shared private room does appear in user directory until leave
-If a device list update goes missing, the server resyncs on the next one
\ No newline at end of file

From 5014b35bd7aaad10444aeb5d0204b338929f4e55 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 7 Sep 2022 16:23:22 +0100
Subject: [PATCH 335/391] Update state reset capture to Sentry

---
 roomserver/internal/input/input_latest_events.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 7e42db1b7..7a7524fb8 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -287,11 +287,14 @@ func (u *latestEventsUpdater) latestState() error {
 		}).Warnf("State reset detected (removing %d events)", removed)
 		sentry.WithScope(func(scope *sentry.Scope) {
 			scope.SetLevel("warning")
-			scope.SetTag("event_id", u.event.EventID())
-			scope.SetTag("old_state_nid", fmt.Sprintf("%d", u.oldStateNID))
-			scope.SetTag("new_state_nid", fmt.Sprintf("%d", u.newStateNID))
-			scope.SetTag("old_latest", u.oldLatest.EventIDs())
-			scope.SetTag("new_latest", u.latest.EventIDs())
+			scope.SetContexts(map[string]interface{}{
+				"Event ID":      u.event.EventID(),
+				"Old state NID": fmt.Sprintf("%d", u.oldStateNID),
+				"New state NID": fmt.Sprintf("%d", u.newStateNID),
+				"Old latest":    u.oldLatest.EventIDs(),
+				"New latest":    u.latest.EventIDs(),
+				"State removed": removed,
+			})
 			sentry.CaptureMessage("State reset detected")
 		})
 	}

From 31f4ae8997af7e939f505107341b86b2abd3fd9a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 7 Sep 2022 16:24:43 +0100
Subject: [PATCH 336/391] Use a single context instead

---
 roomserver/internal/input/input_latest_events.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 7a7524fb8..205a33e83 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -287,7 +287,7 @@ func (u *latestEventsUpdater) latestState() error {
 		}).Warnf("State reset detected (removing %d events)", removed)
 		sentry.WithScope(func(scope *sentry.Scope) {
 			scope.SetLevel("warning")
-			scope.SetContexts(map[string]interface{}{
+			scope.SetContext("State reset", map[string]interface{}{
 				"Event ID":      u.event.EventID(),
 				"Old state NID": fmt.Sprintf("%d", u.oldStateNID),
 				"New state NID": fmt.Sprintf("%d", u.newStateNID),

From d5876abbe9f5484768f603ec91a567b8650e6e73 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 7 Sep 2022 18:15:54 +0200
Subject: [PATCH 337/391] Fulltext implementation incl. config (#2480)

This adds the main component of the fulltext search.
This PR doesn't do anything yet, besides creating an empty fulltextindex
folder if enabled. Indexing events is done in a separate PR.
---
 dendrite-sample.monolith.yaml        |   4 +
 dendrite-sample.polylith.yaml        |   4 +
 docs/installation/7_configuration.md |  13 ++
 go.mod                               |  27 ++-
 go.sum                               |  75 ++++++++
 internal/fulltext/bleve.go           | 164 ++++++++++++++++++
 internal/fulltext/bleve_test.go      | 250 +++++++++++++++++++++++++++
 internal/fulltext/bleve_wasm.go      |  65 +++++++
 setup/base/base.go                   |  12 ++
 setup/config/config_syncapi.go       |  26 +++
 setup/jetstream/streams.go           |   1 +
 test/testrig/base.go                 |   8 +-
 12 files changed, 647 insertions(+), 2 deletions(-)
 create mode 100644 internal/fulltext/bleve.go
 create mode 100644 internal/fulltext/bleve_test.go
 create mode 100644 internal/fulltext/bleve_wasm.go

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index f753c3d9b..195d11610 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -268,6 +268,10 @@ sync_api:
   # address of the client. This is likely required if Dendrite is running behind
   # a reverse proxy server.
   # real_ip_header: X-Real-IP
+  fulltext:
+    enabled: false
+    index_path: "./fulltextindex"
+    language: "en" # more possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
 
 # Configuration for the User API.
 user_api:
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 3caf91434..65d60e150 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -319,6 +319,10 @@ sync_api:
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
+  fulltext:
+    enabled: false
+    index_path: "./fulltextindex"
+    language: "en" # more possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
 
   # This option controls which HTTP header to inspect to find the real remote IP
   # address of the client. This is likely required if Dendrite is running behind
diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md
index b1c747414..8fbe71c40 100644
--- a/docs/installation/7_configuration.md
+++ b/docs/installation/7_configuration.md
@@ -138,6 +138,19 @@ room_server:
     conn_max_lifetime: -1
 ```
 
+## Fulltext search
+
+Dendrite supports experimental fulltext indexing using [Bleve](https://github.com/blevesearch/bleve), it is configured in the `sync_api` section as follows. Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, to ensure the returned results match the expections. A full list of possible languages can be found [here](https://github.com/blevesearch/bleve/tree/master/analysis/lang).
+
+```yaml
+sync_api:
+  # ...
+  fulltext:
+    enabled: false
+    index_path: "./fulltextindex"
+    language: "en"
+```
+
 ## Other sections
 
 There are other options which may be useful so review them all. In particular, if you are
diff --git a/go.mod b/go.mod
index c737e467c..572453974 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/blevesearch/bleve/v2 v2.3.4
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d
 	github.com/docker/docker v20.10.16+incompatible
@@ -16,7 +17,6 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
-	github.com/kardianos/minwinsvc v1.0.0
 	github.com/lib/pq v1.10.5
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
@@ -56,7 +56,24 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/RoaringBitmap/roaring v0.9.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.2.0 // indirect
+	github.com/blevesearch/bleve_index_api v1.0.3 // indirect
+	github.com/blevesearch/geo v0.1.13 // indirect
+	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
+	github.com/blevesearch/gtreap v0.1.1 // indirect
+	github.com/blevesearch/mmap-go v1.0.4 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.1.2 // indirect
+	github.com/blevesearch/segment v0.9.0 // indirect
+	github.com/blevesearch/snowballstem v0.9.0 // indirect
+	github.com/blevesearch/upsidedown_store_api v1.0.1 // indirect
+	github.com/blevesearch/vellum v1.0.8 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.5 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.5 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.5 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.5 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.5 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -67,11 +84,15 @@ require (
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
+	github.com/kardianos/minwinsvc v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/lucas-clemente/quic-go v0.28.1 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
@@ -82,7 +103,10 @@ require (
 	github.com/miekg/dns v1.1.49 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/nats-io/jwt/v2 v2.3.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
@@ -98,6 +122,7 @@ require (
 	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
+	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
diff --git a/go.sum b/go.sum
index 0b331befe..00392cb65 100644
--- a/go.sum
+++ b/go.sum
@@ -58,6 +58,8 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
+github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo=
+github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
 github.com/RyanCarrier/dijkstra v1.1.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
 github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -79,10 +81,49 @@ github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikz
 github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
 github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/blevesearch/bleve/v2 v2.3.4 h1:SSb7/cwGzo85LWX1jchIsXM8ZiNNMX3shT5lROM63ew=
+github.com/blevesearch/bleve/v2 v2.3.4/go.mod h1:Ot0zYum8XQRfPcwhae8bZmNyYubynsoMjVvl1jPqL30=
+github.com/blevesearch/bleve_index_api v1.0.3 h1:DDSWaPXOZZJ2BB73ZTWjKxydAugjwywcqU+91AAqcAg=
+github.com/blevesearch/bleve_index_api v1.0.3/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
+github.com/blevesearch/geo v0.1.13 h1:RsY1vfFm81iv1g+uoCQtsOFvKAhZnpOdTOK8JRA6pqw=
+github.com/blevesearch/geo v0.1.13/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc=
+github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
+github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
+github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
+github.com/blevesearch/goleveldb v1.0.1/go.mod h1:WrU8ltZbIp0wAoig/MHbrPCXSOLpe79nz5lv5nqfYrQ=
+github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
+github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
+github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
+github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
+github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.2 h1:TAte9VZLWda5WAVlZTTZ+GCzEHqGJb4iB2aiZSA6Iv8=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.2/go.mod h1:rvoQXZGq8drq7vXbNeyiRzdEOwZkjkiYGf1822i6CRA=
+github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
+github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
+github.com/blevesearch/snowball v0.6.1/go.mod h1:ZF0IBg5vgpeoUhnMza2v0A/z8m1cWPlwhke08LpNusg=
+github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
+github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
+github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU=
+github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q=
+github.com/blevesearch/vellum v1.0.8 h1:iMGh4lfxza4BnWO/UJTMPlI3HsK9YawjPv+TteVa9ck=
+github.com/blevesearch/vellum v1.0.8/go.mod h1:+cpRi/tqq49xUYSQN2P7A5zNSNrS+MscLeeaZ3J46UA=
+github.com/blevesearch/zapx/v11 v11.3.5 h1:eBQWQ7huA+mzm0sAGnZDwgGGli7S45EO+N+ObFWssbI=
+github.com/blevesearch/zapx/v11 v11.3.5/go.mod h1:5UdIa/HRMdeRCiLQOyFESsnqBGiip7vQmYReA9toevU=
+github.com/blevesearch/zapx/v12 v12.3.5 h1:5pX2hU+R1aZihT7ac1dNWh1n4wqkIM9pZzWp0ANED9s=
+github.com/blevesearch/zapx/v12 v12.3.5/go.mod h1:ANcthYRZQycpbRut/6ArF5gP5HxQyJqiFcuJCBju/ss=
+github.com/blevesearch/zapx/v13 v13.3.5 h1:eJ3gbD+Nu8p36/O6lhfdvWQ4pxsGYSuTOBrLLPVWJ74=
+github.com/blevesearch/zapx/v13 v13.3.5/go.mod h1:FV+dRnScFgKnRDIp08RQL4JhVXt1x2HE3AOzqYa6fjo=
+github.com/blevesearch/zapx/v14 v14.3.5 h1:hEvVjZaagFCvOUJrlFQ6/Z6Jjy0opM3g7TMEo58TwP4=
+github.com/blevesearch/zapx/v14 v14.3.5/go.mod h1:954A/eKFb+pg/ncIYWLWCKY+mIjReM9FGTGIO2Wu1cU=
+github.com/blevesearch/zapx/v15 v15.3.5 h1:NVD0qq8vRk66ImJn1KloXT5ckqPDUZT7VbVJs9jKlac=
+github.com/blevesearch/zapx/v15 v15.3.5/go.mod h1:QMUh2hXCaYIWFKPYGavq/Iga2zbHWZ9DZAa9uFbWyvg=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
@@ -102,7 +143,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI=
 github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
+github.com/couchbase/moss v0.2.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -180,6 +227,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
+github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -213,6 +262,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo=
 github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4=
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -233,6 +284,7 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -264,12 +316,15 @@ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslC
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -315,6 +370,7 @@ github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lucas-clemente/quic-go v0.28.1 h1:Uo0lvVxWg5la9gflIF9lwa39ONq85Xq2D91YNEIslzU=
 github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
@@ -353,6 +409,8 @@ github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
 github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -365,6 +423,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
+github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
@@ -391,12 +451,14 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
@@ -411,6 +473,7 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -482,7 +545,12 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
@@ -512,6 +580,7 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK
 github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
@@ -520,6 +589,7 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c h1:/cTmA6pV2Z20BT/FGSmnb5BmJ8eRbDP0HbCB5IO1aKw=
 github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c/go.mod h1:cIwhYwX9yT9Bcei59O0oOBSaj+kQP+9aVQUMWHh5R00=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -529,6 +599,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -542,6 +614,7 @@ golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -672,6 +745,8 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go
new file mode 100644
index 000000000..b07c0e51d
--- /dev/null
+++ b/internal/fulltext/bleve.go
@@ -0,0 +1,164 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !wasm
+// +build !wasm
+
+package fulltext
+
+import (
+	"strings"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Search contains all existing bleve.Index
+type Search struct {
+	FulltextIndex bleve.Index
+}
+
+// IndexElement describes the layout of an element to index
+type IndexElement struct {
+	EventID        string
+	RoomID         string
+	Content        string
+	ContentType    string
+	StreamPosition int64
+}
+
+// SetContentType sets i.ContentType given an identifier
+func (i *IndexElement) SetContentType(v string) {
+	switch v {
+	case "m.room.message":
+		i.ContentType = "content.body"
+	case gomatrixserverlib.MRoomName:
+		i.ContentType = "content.name"
+	case gomatrixserverlib.MRoomTopic:
+		i.ContentType = "content.topic"
+	}
+}
+
+// New opens a new/existing fulltext index
+func New(cfg config.Fulltext) (fts *Search, err error) {
+	fts = &Search{}
+	fts.FulltextIndex, err = openIndex(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return fts, nil
+}
+
+// Close closes the fulltext index
+func (f *Search) Close() error {
+	return f.FulltextIndex.Close()
+}
+
+// Index indexes the given elements
+func (f *Search) Index(elements ...IndexElement) error {
+	batch := f.FulltextIndex.NewBatch()
+
+	for _, element := range elements {
+		err := batch.Index(element.EventID, element)
+		if err != nil {
+			return err
+		}
+	}
+	return f.FulltextIndex.Batch(batch)
+}
+
+// Delete deletes an indexed element by the eventID
+func (f *Search) Delete(eventID string) error {
+	return f.FulltextIndex.Delete(eventID)
+}
+
+// Search searches the index given a search term, roomIDs and keys.
+func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error) {
+	qry := bleve.NewConjunctionQuery()
+	termQuery := bleve.NewBooleanQuery()
+
+	terms := strings.Split(term, " ")
+	for _, term := range terms {
+		matchQuery := bleve.NewMatchQuery(term)
+		matchQuery.SetField("Content")
+		termQuery.AddMust(matchQuery)
+	}
+	qry.AddQuery(termQuery)
+
+	roomQuery := bleve.NewBooleanQuery()
+	for _, roomID := range roomIDs {
+		roomSearch := bleve.NewMatchQuery(roomID)
+		roomSearch.SetField("RoomID")
+		roomQuery.AddShould(roomSearch)
+	}
+	if len(roomIDs) > 0 {
+		qry.AddQuery(roomQuery)
+	}
+	keyQuery := bleve.NewBooleanQuery()
+	for _, key := range keys {
+		keySearch := bleve.NewMatchQuery(key)
+		keySearch.SetField("ContentType")
+		keyQuery.AddShould(keySearch)
+	}
+	if len(keys) > 0 {
+		qry.AddQuery(keyQuery)
+	}
+
+	s := bleve.NewSearchRequestOptions(qry, limit, from, false)
+	s.Fields = []string{"*"}
+	s.SortBy([]string{"_score"})
+	if orderByStreamPos {
+		s.SortBy([]string{"-StreamPosition"})
+	}
+
+	return f.FulltextIndex.Search(s)
+}
+
+func openIndex(cfg config.Fulltext) (bleve.Index, error) {
+	m := getMapping(cfg)
+	if cfg.InMemory {
+		return bleve.NewMemOnly(m)
+	}
+	if index, err := bleve.Open(string(cfg.IndexPath)); err == nil {
+		return index, nil
+	}
+
+	index, err := bleve.New(string(cfg.IndexPath), m)
+	if err != nil {
+		return nil, err
+	}
+	return index, nil
+}
+
+func getMapping(cfg config.Fulltext) *mapping.IndexMappingImpl {
+	enFieldMapping := bleve.NewTextFieldMapping()
+	enFieldMapping.Analyzer = cfg.Language
+
+	eventMapping := bleve.NewDocumentMapping()
+	eventMapping.AddFieldMappingsAt("Content", enFieldMapping)
+	eventMapping.AddFieldMappingsAt("StreamPosition", bleve.NewNumericFieldMapping())
+
+	// Index entries as is
+	idFieldMapping := bleve.NewKeywordFieldMapping()
+	eventMapping.AddFieldMappingsAt("ContentType", idFieldMapping)
+	eventMapping.AddFieldMappingsAt("RoomID", idFieldMapping)
+	eventMapping.AddFieldMappingsAt("EventID", idFieldMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("Event", eventMapping)
+	indexMapping.DefaultType = "Event"
+	return indexMapping
+}
diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go
new file mode 100644
index 000000000..84a282423
--- /dev/null
+++ b/internal/fulltext/bleve_test.go
@@ -0,0 +1,250 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fulltext_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/internal/fulltext"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search {
+	t.Helper()
+	cfg := config.Fulltext{}
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
+	if tempDir != "" {
+		cfg.IndexPath = config.Path(tempDir)
+		cfg.InMemory = false
+	}
+	fts, err := fulltext.New(cfg)
+	if err != nil {
+		t.Fatal("failed to open fulltext index:", err)
+	}
+	return fts
+}
+
+func mustAddTestData(t *testing.T, fts *fulltext.Search, firstStreamPos int64) (eventIDs, roomIDs []string) {
+	t.Helper()
+	// create some more random data
+	var batchItems []fulltext.IndexElement
+	streamPos := firstStreamPos
+
+	wantRoomID := util.RandomString(16)
+
+	for i := 0; i < 30; i++ {
+		streamPos++
+		eventID := util.RandomString(16)
+		// Create more data for the first room
+		if i > 15 {
+			wantRoomID = util.RandomString(16)
+		}
+		e := fulltext.IndexElement{
+			EventID:        eventID,
+			RoomID:         wantRoomID,
+			Content:        "lorem ipsum",
+			StreamPosition: streamPos,
+		}
+		e.SetContentType("m.room.message")
+		batchItems = append(batchItems, e)
+		roomIDs = append(roomIDs, wantRoomID)
+		eventIDs = append(eventIDs, eventID)
+	}
+	e := fulltext.IndexElement{
+		EventID:        util.RandomString(16),
+		RoomID:         wantRoomID,
+		Content:        "Roomname testing",
+		StreamPosition: streamPos,
+	}
+	e.SetContentType(gomatrixserverlib.MRoomName)
+	batchItems = append(batchItems, e)
+	e = fulltext.IndexElement{
+		EventID:        util.RandomString(16),
+		RoomID:         wantRoomID,
+		Content:        "Room topic fulltext",
+		StreamPosition: streamPos,
+	}
+	e.SetContentType(gomatrixserverlib.MRoomTopic)
+	batchItems = append(batchItems, e)
+	if err := fts.Index(batchItems...); err != nil {
+		t.Fatalf("failed to batch insert elements: %v", err)
+	}
+	return eventIDs, roomIDs
+}
+
+func TestOpen(t *testing.T) {
+	dataDir := t.TempDir()
+	fts := mustOpenIndex(t, dataDir)
+	if err := fts.Close(); err != nil {
+		t.Fatal("unable to close fulltext index", err)
+	}
+
+	// open existing index
+	fts = mustOpenIndex(t, dataDir)
+	defer fts.Close()
+}
+
+func TestIndex(t *testing.T) {
+	fts := mustOpenIndex(t, "")
+	defer fts.Close()
+
+	// add some data
+	var streamPos int64 = 1
+	roomID := util.RandomString(8)
+	eventID := util.RandomString(16)
+	e := fulltext.IndexElement{
+		EventID:        eventID,
+		RoomID:         roomID,
+		Content:        "lorem ipsum",
+		StreamPosition: streamPos,
+	}
+	e.SetContentType("m.room.message")
+
+	if err := fts.Index(e); err != nil {
+		t.Fatal("failed to index element", err)
+	}
+
+	// create some more random data
+	mustAddTestData(t, fts, streamPos)
+}
+
+func TestDelete(t *testing.T) {
+	fts := mustOpenIndex(t, "")
+	defer fts.Close()
+	eventIDs, roomIDs := mustAddTestData(t, fts, 0)
+	res1, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = fts.Delete(eventIDs[0]); err != nil {
+		t.Fatal(err)
+	}
+
+	res2, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if res1.Total <= res2.Total {
+		t.Fatalf("got unexpected result: %d <= %d", res1.Total, res2.Total)
+	}
+}
+
+func TestSearch(t *testing.T) {
+	type args struct {
+		term             string
+		keys             []string
+		limit            int
+		from             int
+		orderByStreamPos bool
+		roomIndex        []int
+	}
+	tests := []struct {
+		name      string
+		args      args
+		wantCount int
+		wantErr   bool
+	}{
+		{
+			name:      "Can search for many results in one room",
+			wantCount: 16,
+			args: args{
+				term:      "lorem",
+				roomIndex: []int{0},
+				limit:     20,
+			},
+		},
+		{
+			name:      "Can search for one result in one room",
+			wantCount: 1,
+			args: args{
+				term:      "lorem",
+				roomIndex: []int{16},
+				limit:     20,
+			},
+		},
+		{
+			name:      "Can search for many results in multiple rooms",
+			wantCount: 17,
+			args: args{
+				term:      "lorem",
+				roomIndex: []int{0, 16},
+				limit:     20,
+			},
+		},
+		{
+			name:      "Can search for many results in all rooms, reversed",
+			wantCount: 30,
+			args: args{
+				term:             "lorem",
+				limit:            30,
+				orderByStreamPos: true,
+			},
+		},
+		{
+			name:      "Can search for specific search room name",
+			wantCount: 1,
+			args: args{
+				term:      "testing",
+				roomIndex: []int{},
+				limit:     20,
+				keys:      []string{"content.name"},
+			},
+		},
+		{
+			name:      "Can search for specific search room topic",
+			wantCount: 1,
+			args: args{
+				term:      "fulltext",
+				roomIndex: []int{},
+				limit:     20,
+				keys:      []string{"content.topic"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := mustOpenIndex(t, "")
+			eventIDs, roomIDs := mustAddTestData(t, f, 0)
+			var searchRooms []string
+			for _, x := range tt.args.roomIndex {
+				searchRooms = append(searchRooms, roomIDs[x])
+			}
+			t.Logf("searching in rooms: %v - %v\n", searchRooms, tt.args.keys)
+
+			got, err := f.Search(tt.args.term, searchRooms, tt.args.keys, tt.args.limit, tt.args.from, tt.args.orderByStreamPos)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Search() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(len(got.Hits), tt.wantCount) {
+				t.Errorf("Search() got = %v, want %v", len(got.Hits), tt.wantCount)
+			}
+			if tt.args.orderByStreamPos {
+				if got.Hits[0].ID != eventIDs[29] {
+					t.Fatalf("expected ID %s, got %s", eventIDs[29], got.Hits[0].ID)
+				}
+			}
+		})
+	}
+}
diff --git a/internal/fulltext/bleve_wasm.go b/internal/fulltext/bleve_wasm.go
new file mode 100644
index 000000000..a69a8926e
--- /dev/null
+++ b/internal/fulltext/bleve_wasm.go
@@ -0,0 +1,65 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fulltext
+
+import (
+	"github.com/matrix-org/dendrite/setup/config"
+	"time"
+)
+
+type Search struct{}
+type IndexElement struct {
+	EventID        string
+	RoomID         string
+	Content        string
+	ContentType    string
+	StreamPosition int64
+}
+
+type SearchResult struct {
+	Status   interface{}   `json:"status"`
+	Request  *interface{}  `json:"request"`
+	Hits     []interface{} `json:"hits"`
+	Total    uint64        `json:"total_hits"`
+	MaxScore float64       `json:"max_score"`
+	Took     time.Duration `json:"took"`
+	Facets   interface{}   `json:"facets"`
+}
+
+func (i *IndexElement) SetContentType(v string) {}
+
+func New(cfg config.Fulltext) (fts *Search, err error) {
+	return &Search{}, nil
+}
+
+func (f *Search) Close() error {
+	return nil
+}
+
+func (f *Search) Index(e IndexElement) error {
+	return nil
+}
+
+func (f *Search) BatchIndex(elements []IndexElement) error {
+	return nil
+}
+
+func (f *Search) Delete(eventID string) error {
+	return nil
+}
+
+func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error) {
+	return SearchResult{}, nil
+}
diff --git a/setup/base/base.go b/setup/base/base.go
index 87f415764..3070bed5f 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -38,6 +38,7 @@ import (
 	"golang.org/x/net/http2/h2c"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/fulltext"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushgateway"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -90,6 +91,7 @@ type BaseDendrite struct {
 	Database               *sql.DB
 	DatabaseWriter         sqlutil.Writer
 	EnableMetrics          bool
+	Fulltext               *fulltext.Search
 	startupLock            sync.Mutex
 }
 
@@ -150,6 +152,15 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		logrus.WithError(err).Panicf("failed to start opentracing")
 	}
 
+	var fts *fulltext.Search
+	isSyncOrMonolith := componentName == "syncapi" || isMonolith
+	if cfg.SyncAPI.Fulltext.Enabled && isSyncOrMonolith {
+		fts, err = fulltext.New(cfg.SyncAPI.Fulltext)
+		if err != nil {
+			logrus.WithError(err).Panicf("failed to create full text")
+		}
+	}
+
 	if cfg.Global.Sentry.Enabled {
 		logrus.Info("Setting up Sentry for debugging...")
 		err = sentry.Init(sentry.ClientOptions{
@@ -247,6 +258,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		Database:               db,     // set if monolith with global connection pool only
 		DatabaseWriter:         writer, // set if monolith with global connection pool only
 		EnableMetrics:          enableMetrics,
+		Fulltext:               fts,
 	}
 }
 
diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go
index 4e302ae09..196afb82c 100644
--- a/setup/config/config_syncapi.go
+++ b/setup/config/config_syncapi.go
@@ -9,6 +9,8 @@ type SyncAPI struct {
 	Database DatabaseOptions `yaml:"database,omitempty"`
 
 	RealIPHeader string `yaml:"real_ip_header"`
+
+	Fulltext Fulltext `yaml:"fulltext"`
 }
 
 func (c *SyncAPI) Defaults(opts DefaultOpts) {
@@ -18,6 +20,7 @@ func (c *SyncAPI) Defaults(opts DefaultOpts) {
 		c.ExternalAPI.Listen = "http://localhost:8073"
 		c.Database.Defaults(20)
 	}
+	c.Fulltext.Defaults(opts)
 	if opts.Generate {
 		if !opts.Monolithic {
 			c.Database.ConnectionString = "file:syncapi.db"
@@ -26,6 +29,7 @@ func (c *SyncAPI) Defaults(opts DefaultOpts) {
 }
 
 func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	c.Fulltext.Verify(configErrs, isMonolith)
 	if isMonolith { // polylith required configs below
 		return
 	}
@@ -36,3 +40,25 @@ func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect))
 	checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen))
 }
+
+type Fulltext struct {
+	Enabled   bool   `yaml:"enabled"`
+	IndexPath Path   `yaml:"index_path"`
+	InMemory  bool   `yaml:"in_memory"` // only useful in tests
+	Language  string `yaml:"language"`  // the language to use when analysing content
+}
+
+func (f *Fulltext) Defaults(opts DefaultOpts) {
+	f.Enabled = false
+	f.IndexPath = "./fulltextindex"
+	f.Language = "en"
+	if opts.Generate {
+		f.Enabled = true
+		f.InMemory = true
+	}
+}
+
+func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "syncapi.fulltext.index_path", string(f.IndexPath))
+	checkNotEmpty(configErrs, "syncapi.fulltext.language", f.Language)
+}
diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go
index a06a12662..c07d3a0b4 100644
--- a/setup/jetstream/streams.go
+++ b/setup/jetstream/streams.go
@@ -29,6 +29,7 @@ var (
 	OutputReadUpdate        = "OutputReadUpdate"
 	RequestPresence         = "GetPresence"
 	OutputPresenceEvent     = "OutputPresenceEvent"
+	InputFulltextReindex    = "InputFulltextReindex"
 )
 
 var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+")
diff --git a/test/testrig/base.go b/test/testrig/base.go
index 9747ea600..10cc2407b 100644
--- a/test/testrig/base.go
+++ b/test/testrig/base.go
@@ -22,10 +22,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/nats-io/nats.go"
+
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/nats-io/nats.go"
 )
 
 func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) {
@@ -45,6 +46,10 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 			Generate:   true,
 			Monolithic: true,
 		})
+		cfg.SyncAPI.Fulltext.Defaults(config.DefaultOpts{ // use in memory fts
+			Generate:   true,
+			Monolithic: true,
+		})
 		cfg.Global.ServerName = "test"
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
@@ -100,6 +105,7 @@ func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nat
 		})
 	}
 	cfg.Global.JetStream.InMemory = true
+	cfg.SyncAPI.Fulltext.InMemory = true
 	base := base.NewBaseDendrite(cfg, "Tests")
 	js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream)
 	return base, js, jc

From 42a82091a8932fd92e36ba41c644c2058ab7dce6 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 8 Sep 2022 12:03:44 +0200
Subject: [PATCH 338/391] Fix issue with stale device lists (#2702)

We were only sending the last entry to the worker, so most likely missed
updates.
---
 federationapi/inthttp/server.go          | 13 +++++++++++--
 keyserver/internal/device_list_update.go | 13 +++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/federationapi/inthttp/server.go b/federationapi/inthttp/server.go
index 58ea9ddc6..7aa0e4801 100644
--- a/federationapi/inthttp/server.go
+++ b/federationapi/inthttp/server.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/http"
+	"net/url"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/gomatrix"
@@ -235,9 +236,17 @@ func federationClientError(err error) error {
 		return &api.FederationClientError{
 			Code: ferr.Code,
 		}
-	default:
+	case *url.Error: // e.g. certificate error, unable to connect
 		return &api.FederationClientError{
-			Err: err.Error(),
+			Err:  ferr.Error(),
+			Code: 400,
+		}
+	default:
+		// We don't know what exactly failed, but we probably don't
+		// want to retry the request immediately in the device list updater
+		return &api.FederationClientError{
+			Err:  err.Error(),
+			Code: 400,
 		}
 	}
 }
diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 41534fe82..525f8a99d 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -167,6 +167,7 @@ func (u *DeviceListUpdater) Start() error {
 		step = (time.Second * 120) / time.Duration(max)
 	}
 	for _, userID := range staleLists {
+		userID := userID // otherwise we are only sending the last entry
 		time.AfterFunc(offset, func() {
 			u.notifyWorkers(userID)
 		})
@@ -396,11 +397,19 @@ userLoop:
 		if ctx.Err() != nil {
 			// we've timed out, give up and go to the back of the queue to let another server be processed.
 			failCount += 1
+			waitTime = time.Minute * 10
 			break
 		}
 		res, err := u.fedClient.GetUserDevices(ctx, serverName, userID)
 		if err != nil {
 			failCount += 1
+			select {
+			case <-ctx.Done():
+				// we've timed out, give up and go to the back of the queue to let another server be processed.
+				waitTime = time.Minute * 10
+				break userLoop
+			default:
+			}
 			switch e := err.(type) {
 			case *fedsenderapi.FederationClientError:
 				if e.RetryAfter > 0 {
@@ -419,7 +428,7 @@ userLoop:
 				// It probably doesn't make sense to try further users.
 				if !e.Timeout() {
 					waitTime = time.Minute * 10
-					logrus.WithError(e).Error("GetUserDevices returned net.Error")
+					logger.WithError(e).Error("GetUserDevices returned net.Error")
 					break userLoop
 				}
 			case gomatrix.HTTPError:
@@ -427,7 +436,7 @@ userLoop:
 				// This is to avoid spamming remote servers, which may not be Matrix servers anymore.
 				if e.Code >= 300 {
 					waitTime = time.Hour
-					logrus.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError")
+					logger.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError")
 					break userLoop
 				}
 			default:

From 64472d9aab85bf56fb345570459c372b78a14b8c Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 9 Sep 2022 13:56:33 +0200
Subject: [PATCH 339/391] Update getting pushrules, add tests, tweak pushrules
 (#2705)

This PR
- adds tests for `evaluatePushrules`
- removes the need for the UserAPI on the `OutputStreamEventConsumer`
(for easier testing)
- adds a method to get the pushrules from the database
- adds a new default pushrule for `m.reaction` events (and some other
tweaks)
---
 internal/pushrules/default_content.go         |   6 +
 internal/pushrules/default_override.go        |  23 +++-
 internal/pushrules/default_underride.go       |  30 +++-
 internal/pushrules/evaluate_test.go           |  32 +++--
 internal/pushrules/util.go                    |   2 +-
 userapi/consumers/syncapi_streamevent.go      |  13 +-
 userapi/consumers/syncapi_streamevent_test.go | 129 ++++++++++++++++++
 userapi/internal/api.go                       |  57 +-------
 userapi/storage/interface.go                  |   2 +
 userapi/storage/shared/storage.go             |  38 +++++-
 userapi/userapi.go                            |   5 +-
 11 files changed, 255 insertions(+), 82 deletions(-)
 create mode 100644 userapi/consumers/syncapi_streamevent_test.go

diff --git a/internal/pushrules/default_content.go b/internal/pushrules/default_content.go
index 158afd18b..8982dd587 100644
--- a/internal/pushrules/default_content.go
+++ b/internal/pushrules/default_content.go
@@ -16,6 +16,12 @@ func mRuleContainsUserNameDefinition(localpart string) *Rule {
 		Default: true,
 		Enabled: true,
 		Pattern: localpart,
+		Conditions: []*Condition{
+			{
+				Kind: EventMatchCondition,
+				Key:  "content.body",
+			},
+		},
 		Actions: []*Action{
 			{Kind: NotifyAction},
 			{
diff --git a/internal/pushrules/default_override.go b/internal/pushrules/default_override.go
index 6f66fd66a..a9788df2f 100644
--- a/internal/pushrules/default_override.go
+++ b/internal/pushrules/default_override.go
@@ -7,8 +7,9 @@ func defaultOverrideRules(userID string) []*Rule {
 		mRuleInviteForMeDefinition(userID),
 		&mRuleMemberEventDefinition,
 		&mRuleContainsDisplayNameDefinition,
-		&mRuleTombstoneDefinition,
 		&mRuleRoomNotifDefinition,
+		&mRuleTombstoneDefinition,
+		&mRuleReactionDefinition,
 	}
 }
 
@@ -20,6 +21,7 @@ const (
 	MRuleContainsDisplayName = ".m.rule.contains_display_name"
 	MRuleTombstone           = ".m.rule.tombstone"
 	MRuleRoomNotif           = ".m.rule.roomnotif"
+	MRuleReaction            = ".m.rule.reaction"
 )
 
 var (
@@ -96,7 +98,7 @@ var (
 			{
 				Kind:  SetTweakAction,
 				Tweak: HighlightTweak,
-				Value: false,
+				Value: true,
 			},
 		},
 	}
@@ -120,10 +122,25 @@ var (
 			{
 				Kind:  SetTweakAction,
 				Tweak: HighlightTweak,
-				Value: false,
+				Value: true,
 			},
 		},
 	}
+	mRuleReactionDefinition = Rule{
+		RuleID:  MRuleReaction,
+		Default: true,
+		Enabled: true,
+		Conditions: []*Condition{
+			{
+				Kind:    EventMatchCondition,
+				Key:     "type",
+				Pattern: "m.reaction",
+			},
+		},
+		Actions: []*Action{
+			{Kind: DontNotifyAction},
+		},
+	}
 )
 
 func mRuleInviteForMeDefinition(userID string) *Rule {
diff --git a/internal/pushrules/default_underride.go b/internal/pushrules/default_underride.go
index de72bd526..8da449a19 100644
--- a/internal/pushrules/default_underride.go
+++ b/internal/pushrules/default_underride.go
@@ -10,8 +10,8 @@ const (
 
 var defaultUnderrideRules = []*Rule{
 	&mRuleCallDefinition,
-	&mRuleEncryptedRoomOneToOneDefinition,
 	&mRuleRoomOneToOneDefinition,
+	&mRuleEncryptedRoomOneToOneDefinition,
 	&mRuleMessageDefinition,
 	&mRuleEncryptedDefinition,
 }
@@ -59,6 +59,11 @@ var (
 		},
 		Actions: []*Action{
 			{Kind: NotifyAction},
+			{
+				Kind:  SetTweakAction,
+				Tweak: SoundTweak,
+				Value: "default",
+			},
 			{
 				Kind:  SetTweakAction,
 				Tweak: HighlightTweak,
@@ -88,6 +93,11 @@ var (
 				Tweak: HighlightTweak,
 				Value: false,
 			},
+			{
+				Kind:  SetTweakAction,
+				Tweak: HighlightTweak,
+				Value: false,
+			},
 		},
 	}
 	mRuleMessageDefinition = Rule{
@@ -101,7 +111,14 @@ var (
 				Pattern: "m.room.message",
 			},
 		},
-		Actions: []*Action{{Kind: NotifyAction}},
+		Actions: []*Action{
+			{Kind: NotifyAction},
+			{
+				Kind:  SetTweakAction,
+				Tweak: HighlightTweak,
+				Value: false,
+			},
+		},
 	}
 	mRuleEncryptedDefinition = Rule{
 		RuleID:  MRuleEncrypted,
@@ -114,6 +131,13 @@ var (
 				Pattern: "m.room.encrypted",
 			},
 		},
-		Actions: []*Action{{Kind: NotifyAction}},
+		Actions: []*Action{
+			{Kind: NotifyAction},
+			{
+				Kind:  SetTweakAction,
+				Tweak: HighlightTweak,
+				Value: false,
+			},
+		},
 	}
 )
diff --git a/internal/pushrules/evaluate_test.go b/internal/pushrules/evaluate_test.go
index 50e703365..eabd02415 100644
--- a/internal/pushrules/evaluate_test.go
+++ b/internal/pushrules/evaluate_test.go
@@ -24,24 +24,28 @@ func TestRuleSetEvaluatorMatchEvent(t *testing.T) {
 		Default: false,
 		Enabled: true,
 	}
+	defaultRuleset := DefaultGlobalRuleSet("test", "test")
 	tsts := []struct {
 		Name    string
 		RuleSet RuleSet
 		Want    *Rule
+		Event   *gomatrixserverlib.Event
 	}{
-		{"empty", RuleSet{}, nil},
-		{"defaultCanWin", RuleSet{Override: []*Rule{defaultEnabled}}, defaultEnabled},
-		{"userWins", RuleSet{Override: []*Rule{defaultEnabled, userEnabled}}, userEnabled},
-		{"defaultOverrideWins", RuleSet{Override: []*Rule{defaultEnabled}, Underride: []*Rule{userEnabled}}, defaultEnabled},
-		{"overrideContent", RuleSet{Override: []*Rule{userEnabled}, Content: []*Rule{userEnabled2}}, userEnabled},
-		{"overrideRoom", RuleSet{Override: []*Rule{userEnabled}, Room: []*Rule{userEnabled2}}, userEnabled},
-		{"overrideSender", RuleSet{Override: []*Rule{userEnabled}, Sender: []*Rule{userEnabled2}}, userEnabled},
-		{"overrideUnderride", RuleSet{Override: []*Rule{userEnabled}, Underride: []*Rule{userEnabled2}}, userEnabled},
+		{"empty", RuleSet{}, nil, ev},
+		{"defaultCanWin", RuleSet{Override: []*Rule{defaultEnabled}}, defaultEnabled, ev},
+		{"userWins", RuleSet{Override: []*Rule{defaultEnabled, userEnabled}}, userEnabled, ev},
+		{"defaultOverrideWins", RuleSet{Override: []*Rule{defaultEnabled}, Underride: []*Rule{userEnabled}}, defaultEnabled, ev},
+		{"overrideContent", RuleSet{Override: []*Rule{userEnabled}, Content: []*Rule{userEnabled2}}, userEnabled, ev},
+		{"overrideRoom", RuleSet{Override: []*Rule{userEnabled}, Room: []*Rule{userEnabled2}}, userEnabled, ev},
+		{"overrideSender", RuleSet{Override: []*Rule{userEnabled}, Sender: []*Rule{userEnabled2}}, userEnabled, ev},
+		{"overrideUnderride", RuleSet{Override: []*Rule{userEnabled}, Underride: []*Rule{userEnabled2}}, userEnabled, ev},
+		{"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"type":"m.reaction"}`)},
+		{"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"type":"m.receipt"}`)},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
-			rse := NewRuleSetEvaluator(nil, &tst.RuleSet)
-			got, err := rse.MatchEvent(ev)
+			rse := NewRuleSetEvaluator(fakeEvaluationContext{3}, &tst.RuleSet)
+			got, err := rse.MatchEvent(tst.Event)
 			if err != nil {
 				t.Fatalf("MatchEvent failed: %v", err)
 			}
@@ -128,7 +132,7 @@ func TestConditionMatches(t *testing.T) {
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
-			got, err := conditionMatches(&tst.Cond, mustEventFromJSON(t, tst.EventJSON), &fakeEvaluationContext{})
+			got, err := conditionMatches(&tst.Cond, mustEventFromJSON(t, tst.EventJSON), &fakeEvaluationContext{2})
 			if err != nil {
 				t.Fatalf("conditionMatches failed: %v", err)
 			}
@@ -139,10 +143,10 @@ func TestConditionMatches(t *testing.T) {
 	}
 }
 
-type fakeEvaluationContext struct{}
+type fakeEvaluationContext struct{ memberCount int }
 
-func (fakeEvaluationContext) UserDisplayName() string       { return "Dear User" }
-func (fakeEvaluationContext) RoomMemberCount() (int, error) { return 2, nil }
+func (fakeEvaluationContext) UserDisplayName() string         { return "Dear User" }
+func (f fakeEvaluationContext) RoomMemberCount() (int, error) { return f.memberCount, nil }
 func (fakeEvaluationContext) HasPowerLevel(userID, levelKey string) (bool, error) {
 	return userID == "@poweruser:example.com" && levelKey == "powerlevel", nil
 }
diff --git a/internal/pushrules/util.go b/internal/pushrules/util.go
index 027d35ef6..8ab4eab94 100644
--- a/internal/pushrules/util.go
+++ b/internal/pushrules/util.go
@@ -11,7 +11,7 @@ import (
 // kind and a tweaks map. Returns a nil map if it would have been
 // empty.
 func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) {
-	var kind ActionKind
+	kind := UnknownAction
 	tweaks := map[string]interface{}{}
 
 	for _, a := range as {
diff --git a/userapi/consumers/syncapi_streamevent.go b/userapi/consumers/syncapi_streamevent.go
index 3ac6f58d0..f3b2bf27f 100644
--- a/userapi/consumers/syncapi_streamevent.go
+++ b/userapi/consumers/syncapi_streamevent.go
@@ -29,7 +29,6 @@ import (
 type OutputStreamEventConsumer struct {
 	ctx          context.Context
 	cfg          *config.UserAPI
-	userAPI      api.UserInternalAPI
 	rsAPI        rsapi.UserRoomserverAPI
 	jetstream    nats.JetStreamContext
 	durable      string
@@ -45,7 +44,6 @@ func NewOutputStreamEventConsumer(
 	js nats.JetStreamContext,
 	store storage.Database,
 	pgClient pushgateway.Client,
-	userAPI api.UserInternalAPI,
 	rsAPI rsapi.UserRoomserverAPI,
 	syncProducer *producers.SyncAPI,
 ) *OutputStreamEventConsumer {
@@ -57,7 +55,6 @@ func NewOutputStreamEventConsumer(
 		durable:      cfg.Matrix.JetStream.Durable("UserAPISyncAPIStreamEventConsumer"),
 		topic:        cfg.Matrix.JetStream.Prefixed(jetstream.OutputStreamEvent),
 		pgClient:     pgClient,
-		userAPI:      userAPI,
 		rsAPI:        rsAPI,
 		syncProducer: syncProducer,
 	}
@@ -305,7 +302,7 @@ func (s *OutputStreamEventConsumer) notifyLocal(ctx context.Context, event *goma
 			"event_id":  event.EventID(),
 			"room_id":   event.RoomID(),
 			"localpart": mem.Localpart,
-		}).Tracef("Push rule evaluation rejected the event")
+		}).Debugf("Push rule evaluation rejected the event")
 		return nil
 	}
 
@@ -348,7 +345,7 @@ func (s *OutputStreamEventConsumer) notifyLocal(ctx context.Context, event *goma
 		"localpart":  mem.Localpart,
 		"num_urls":   len(devicesByURLAndFormat),
 		"num_unread": userNumUnreadNotifs,
-	}).Tracef("Notifying single member")
+	}).Debugf("Notifying single member")
 
 	// Push gateways are out of our control, and we cannot risk
 	// looking up the server on a misbehaving push gateway. Each user
@@ -422,8 +419,8 @@ func (s *OutputStreamEventConsumer) evaluatePushRules(ctx context.Context, event
 			return nil, fmt.Errorf("user %s is ignored", sender)
 		}
 	}
-	var res api.QueryPushRulesResponse
-	if err = s.userAPI.QueryPushRules(ctx, &api.QueryPushRulesRequest{UserID: mem.UserID}, &res); err != nil {
+	ruleSets, err := s.db.QueryPushRules(ctx, mem.Localpart)
+	if err != nil {
 		return nil, err
 	}
 
@@ -434,7 +431,7 @@ func (s *OutputStreamEventConsumer) evaluatePushRules(ctx context.Context, event
 		roomID:   event.RoomID(),
 		roomSize: roomSize,
 	}
-	eval := pushrules.NewRuleSetEvaluator(ec, &res.RuleSets.Global)
+	eval := pushrules.NewRuleSetEvaluator(ec, &ruleSets.Global)
 	rule, err := eval.MatchEvent(event.Event)
 	if err != nil {
 		return nil, err
diff --git a/userapi/consumers/syncapi_streamevent_test.go b/userapi/consumers/syncapi_streamevent_test.go
new file mode 100644
index 000000000..48ea0fe11
--- /dev/null
+++ b/userapi/consumers/syncapi_streamevent_test.go
@@ -0,0 +1,129 @@
+package consumers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/matrix-org/dendrite/internal/pushrules"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi/storage"
+)
+
+func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := storage.NewUserAPIDatabase(nil, &config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, "", 4, 0, 0, "")
+	if err != nil {
+		t.Fatalf("failed to create new user db: %v", err)
+	}
+	return db, close
+}
+
+func mustCreateEvent(t *testing.T, content string) *gomatrixserverlib.HeaderedEvent {
+	t.Helper()
+	ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(content), false, gomatrixserverlib.RoomVersionV10)
+	if err != nil {
+		t.Fatalf("failed to create event: %v", err)
+	}
+	return ev.Headered(gomatrixserverlib.RoomVersionV10)
+}
+
+func Test_evaluatePushRules(t *testing.T) {
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateDatabase(t, dbType)
+		defer close()
+		consumer := OutputStreamEventConsumer{db: db}
+
+		testCases := []struct {
+			name         string
+			eventContent string
+			wantAction   pushrules.ActionKind
+			wantActions  []*pushrules.Action
+			wantNotify   bool
+		}{
+			{
+				name:         "m.receipt doesn't notify",
+				eventContent: `{"type":"m.receipt"}`,
+				wantAction:   pushrules.UnknownAction,
+				wantActions:  nil,
+			},
+			{
+				name:         "m.reaction doesn't notify",
+				eventContent: `{"type":"m.reaction"}`,
+				wantAction:   pushrules.DontNotifyAction,
+				wantActions: []*pushrules.Action{
+					{
+						Kind: pushrules.DontNotifyAction,
+					},
+				},
+			},
+			{
+				name:         "m.room.message notifies",
+				eventContent: `{"type":"m.room.message"}`,
+				wantNotify:   true,
+				wantAction:   pushrules.NotifyAction,
+				wantActions: []*pushrules.Action{
+					{Kind: pushrules.NotifyAction},
+					{
+						Kind:  pushrules.SetTweakAction,
+						Tweak: pushrules.HighlightTweak,
+						Value: false,
+					},
+				},
+			},
+			{
+				name:         "m.room.message highlights",
+				eventContent: `{"type":"m.room.message", "content": {"body": "test"} }`,
+				wantNotify:   true,
+				wantAction:   pushrules.NotifyAction,
+				wantActions: []*pushrules.Action{
+					{Kind: pushrules.NotifyAction},
+					{
+						Kind:  pushrules.SetTweakAction,
+						Tweak: pushrules.SoundTweak,
+						Value: "default",
+					},
+					{
+						Kind:  pushrules.SetTweakAction,
+						Tweak: pushrules.HighlightTweak,
+						Value: true,
+					},
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				actions, err := consumer.evaluatePushRules(ctx, mustCreateEvent(t, tc.eventContent), &localMembership{
+					UserID:    "@test:localhost",
+					Localpart: "test",
+					Domain:    "localhost",
+				}, 10)
+				if err != nil {
+					t.Fatalf("failed to evaluate push rules: %v", err)
+				}
+				assert.Equal(t, tc.wantActions, actions)
+				gotAction, _, err := pushrules.ActionsToTweaks(actions)
+				if err != nil {
+					t.Fatalf("failed to get actions: %v", err)
+				}
+				if gotAction != tc.wantAction {
+					t.Fatalf("expected action to be '%s', got '%s'", tc.wantAction, gotAction)
+				}
+				// this is taken from `notifyLocal`
+				if tc.wantNotify && gotAction != pushrules.NotifyAction && gotAction != pushrules.CoalesceAction {
+					t.Fatalf("expected to notify but didn't")
+				}
+			})
+
+		}
+	})
+}
diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 2f28ee906..dcbb73614 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -30,7 +30,6 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/internal/pushrules"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
@@ -760,57 +759,15 @@ func (a *UserInternalAPI) PerformPushRulesPut(
 }
 
 func (a *UserInternalAPI) QueryPushRules(ctx context.Context, req *api.QueryPushRulesRequest, res *api.QueryPushRulesResponse) error {
-	userReq := api.QueryAccountDataRequest{
-		UserID:   req.UserID,
-		DataType: pushRulesAccountDataType,
+	localpart, _, err := gomatrixserverlib.SplitID('@', req.UserID)
+	if err != nil {
+		return fmt.Errorf("failed to split user ID %q for push rules", req.UserID)
 	}
-	var userRes api.QueryAccountDataResponse
-	if err := a.QueryAccountData(ctx, &userReq, &userRes); err != nil {
-		return err
+	pushRules, err := a.DB.QueryPushRules(ctx, localpart)
+	if err != nil {
+		return fmt.Errorf("failed to query push rules: %w", err)
 	}
-	bs, ok := userRes.GlobalAccountData[pushRulesAccountDataType]
-	if ok {
-		// Legacy Dendrite users will have completely empty push rules, so we should
-		// detect that situation and set some defaults.
-		var rules struct {
-			G struct {
-				Content   []json.RawMessage `json:"content"`
-				Override  []json.RawMessage `json:"override"`
-				Room      []json.RawMessage `json:"room"`
-				Sender    []json.RawMessage `json:"sender"`
-				Underride []json.RawMessage `json:"underride"`
-			} `json:"global"`
-		}
-		if err := json.Unmarshal([]byte(bs), &rules); err == nil {
-			count := len(rules.G.Content) + len(rules.G.Override) +
-				len(rules.G.Room) + len(rules.G.Sender) + len(rules.G.Underride)
-			ok = count > 0
-		}
-	}
-	if !ok {
-		// If we didn't find any default push rules then we should just generate some
-		// fresh ones.
-		localpart, _, err := gomatrixserverlib.SplitID('@', req.UserID)
-		if err != nil {
-			return fmt.Errorf("failed to split user ID %q for push rules", req.UserID)
-		}
-		pushRuleSets := pushrules.DefaultAccountRuleSets(localpart, a.ServerName)
-		prbs, err := json.Marshal(pushRuleSets)
-		if err != nil {
-			return fmt.Errorf("failed to marshal default push rules: %w", err)
-		}
-		if err := a.DB.SaveAccountData(ctx, localpart, "", pushRulesAccountDataType, json.RawMessage(prbs)); err != nil {
-			return fmt.Errorf("failed to save default push rules: %w", err)
-		}
-		res.RuleSets = pushRuleSets
-		return nil
-	}
-	var data pushrules.AccountRuleSets
-	if err := json.Unmarshal([]byte(bs), &data); err != nil {
-		util.GetLogger(ctx).WithError(err).Error("json.Unmarshal of push rules failed")
-		return err
-	}
-	res.RuleSets = &data
+	res.RuleSets = pushRules
 	return nil
 }
 
diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go
index f7cd1810a..fbac463e2 100644
--- a/userapi/storage/interface.go
+++ b/userapi/storage/interface.go
@@ -20,6 +20,7 @@ import (
 	"errors"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/pushrules"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 	"github.com/matrix-org/dendrite/userapi/types"
@@ -53,6 +54,7 @@ type AccountData interface {
 	// If no account data could be found, returns nil
 	// Returns an error if there was an issue with the retrieval
 	GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data json.RawMessage, err error)
+	QueryPushRules(ctx context.Context, localpart string) (*pushrules.AccountRuleSets, error)
 }
 
 type Device interface {
diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index 0cf713dac..d968b47a9 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -26,10 +26,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/matrix-org/dendrite/userapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 
+	"github.com/matrix-org/dendrite/userapi/types"
+
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/internal/pushrules"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -177,6 +178,41 @@ func (d *Database) createAccount(
 	return account, nil
 }
 
+func (d *Database) QueryPushRules(
+	ctx context.Context,
+	localpart string,
+) (*pushrules.AccountRuleSets, error) {
+	data, err := d.AccountDatas.SelectAccountDataByType(ctx, localpart, "", "m.push_rules")
+	if err != nil {
+		return nil, err
+	}
+
+	// If we didn't find any default push rules then we should just generate some
+	// fresh ones.
+	if len(data) == 0 {
+		pushRuleSets := pushrules.DefaultAccountRuleSets(localpart, d.ServerName)
+		prbs, err := json.Marshal(pushRuleSets)
+		if err != nil {
+			return nil, fmt.Errorf("failed to marshal default push rules: %w", err)
+		}
+		err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+			if dbErr := d.AccountDatas.InsertAccountData(ctx, txn, localpart, "", "m.push_rules", prbs); dbErr != nil {
+				return fmt.Errorf("failed to save default push rules: %w", dbErr)
+			}
+			return nil
+		})
+
+		return pushRuleSets, err
+	}
+
+	var pushRules pushrules.AccountRuleSets
+	if err := json.Unmarshal(data, &pushRules); err != nil {
+		return nil, err
+	}
+
+	return &pushRules, nil
+}
+
 // SaveAccountData saves new account data for a given user and a given room.
 // If the account data is not specific to a room, the room ID should be an empty string
 // If an account data already exists for a given set (user, room, data type), it will
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 2e86d6aa7..23855a89f 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -18,6 +18,8 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/internal/pushgateway"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
@@ -31,7 +33,6 @@ import (
 	"github.com/matrix-org/dendrite/userapi/producers"
 	"github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/dendrite/userapi/util"
-	"github.com/sirupsen/logrus"
 )
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
@@ -90,7 +91,7 @@ func NewInternalAPI(
 	}
 
 	eventConsumer := consumers.NewOutputStreamEventConsumer(
-		base.ProcessContext, cfg, js, db, pgClient, userAPI, rsAPI, syncProducer,
+		base.ProcessContext, cfg, js, db, pgClient, rsAPI, syncProducer,
 	)
 	if err := eventConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start user API streamed event consumer")

From 34e1dc210bf4698d2b46a0be71e6dfc10a5a6365 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Sep 2022 12:59:28 +0100
Subject: [PATCH 340/391] Use database writer in user API
 `DeleteOldNotifications`

---
 userapi/storage/shared/storage.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index d968b47a9..acc65212c 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -735,7 +735,9 @@ func (d *Database) GetRoomNotificationCounts(ctx context.Context, localpart, roo
 }
 
 func (d *Database) DeleteOldNotifications(ctx context.Context) error {
-	return d.Notifications.Clean(ctx, nil)
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.Notifications.Clean(ctx, txn)
+	})
 }
 
 func (d *Database) UpsertPusher(

From 646de03d60fa1ca78dbf0b4d5418600d540fb881 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Sep 2022 13:06:42 +0100
Subject: [PATCH 341/391] More writer fixes in the Sync API

---
 syncapi/storage/postgres/ignores_table.go     |  9 ++++----
 .../postgres/notification_data_table.go       | 12 +++++-----
 syncapi/storage/shared/syncserver.go          | 22 +++++++++++++------
 syncapi/storage/sqlite3/ignores_table.go      |  9 ++++----
 .../sqlite3/notification_data_table.go        | 10 ++++-----
 syncapi/storage/tables/interface.go           | 10 ++++-----
 6 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/syncapi/storage/postgres/ignores_table.go b/syncapi/storage/postgres/ignores_table.go
index 055a1a237..97660725c 100644
--- a/syncapi/storage/postgres/ignores_table.go
+++ b/syncapi/storage/postgres/ignores_table.go
@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
@@ -61,10 +62,10 @@ func NewPostgresIgnoresTable(db *sql.DB) (tables.Ignores, error) {
 }
 
 func (s *ignoresStatements) SelectIgnores(
-	ctx context.Context, userID string,
+	ctx context.Context, txn *sql.Tx, userID string,
 ) (*types.IgnoredUsers, error) {
 	var ignoresData []byte
-	err := s.selectIgnoresStmt.QueryRowContext(ctx, userID).Scan(&ignoresData)
+	err := sqlutil.TxStmt(txn, s.selectIgnoresStmt).QueryRowContext(ctx, userID).Scan(&ignoresData)
 	if err != nil {
 		return nil, err
 	}
@@ -76,12 +77,12 @@ func (s *ignoresStatements) SelectIgnores(
 }
 
 func (s *ignoresStatements) UpsertIgnores(
-	ctx context.Context, userID string, ignores *types.IgnoredUsers,
+	ctx context.Context, txn *sql.Tx, userID string, ignores *types.IgnoredUsers,
 ) error {
 	ignoresJSON, err := json.Marshal(ignores)
 	if err != nil {
 		return err
 	}
-	_, err = s.upsertIgnoresStmt.ExecContext(ctx, userID, ignoresJSON)
+	_, err = sqlutil.TxStmt(txn, s.upsertIgnoresStmt).ExecContext(ctx, userID, ignoresJSON)
 	return err
 }
diff --git a/syncapi/storage/postgres/notification_data_table.go b/syncapi/storage/postgres/notification_data_table.go
index 9cd8b7362..708c3a9b4 100644
--- a/syncapi/storage/postgres/notification_data_table.go
+++ b/syncapi/storage/postgres/notification_data_table.go
@@ -70,13 +70,13 @@ const selectUserUnreadNotificationCountsSQL = `SELECT
 
 const selectMaxNotificationIDSQL = `SELECT CASE COUNT(*) WHEN 0 THEN 0 ELSE MAX(id) END FROM syncapi_notification_data`
 
-func (r *notificationDataStatements) UpsertRoomUnreadCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
-	err = r.upsertRoomUnreadCounts.QueryRowContext(ctx, userID, roomID, notificationCount, highlightCount).Scan(&pos)
+func (r *notificationDataStatements) UpsertRoomUnreadCounts(ctx context.Context, txn *sql.Tx, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
+	err = sqlutil.TxStmt(txn, r.upsertRoomUnreadCounts).QueryRowContext(ctx, userID, roomID, notificationCount, highlightCount).Scan(&pos)
 	return
 }
 
-func (r *notificationDataStatements) SelectUserUnreadCounts(ctx context.Context, userID string, fromExcl, toIncl types.StreamPosition) (map[string]*eventutil.NotificationData, error) {
-	rows, err := r.selectUserUnreadCounts.QueryContext(ctx, userID, fromExcl, toIncl)
+func (r *notificationDataStatements) SelectUserUnreadCounts(ctx context.Context, txn *sql.Tx, userID string, fromExcl, toIncl types.StreamPosition) (map[string]*eventutil.NotificationData, error) {
+	rows, err := sqlutil.TxStmt(txn, r.selectUserUnreadCounts).QueryContext(ctx, userID, fromExcl, toIncl)
 	if err != nil {
 		return nil, err
 	}
@@ -101,8 +101,8 @@ func (r *notificationDataStatements) SelectUserUnreadCounts(ctx context.Context,
 	return roomCounts, rows.Err()
 }
 
-func (r *notificationDataStatements) SelectMaxID(ctx context.Context) (int64, error) {
+func (r *notificationDataStatements) SelectMaxID(ctx context.Context, txn *sql.Tx) (int64, error) {
 	var id int64
-	err := r.selectMaxID.QueryRowContext(ctx).Scan(&id)
+	err := sqlutil.TxStmt(txn, r.selectMaxID).QueryRowContext(ctx).Scan(&id)
 	return id, err
 }
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index b06d2c6a9..778ad8b18 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -108,7 +108,7 @@ func (d *Database) MaxStreamPositionForAccountData(ctx context.Context) (types.S
 }
 
 func (d *Database) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) {
-	id, err := d.NotificationData.SelectMaxID(ctx)
+	id, err := d.NotificationData.SelectMaxID(ctx, nil)
 	if err != nil {
 		return 0, fmt.Errorf("d.NotificationData.SelectMaxID: %w", err)
 	}
@@ -1029,15 +1029,15 @@ func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, stream
 }
 
 func (d *Database) UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
-	err = d.Writer.Do(nil, nil, func(_ *sql.Tx) error {
-		pos, err = d.NotificationData.UpsertRoomUnreadCounts(ctx, userID, roomID, notificationCount, highlightCount)
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		pos, err = d.NotificationData.UpsertRoomUnreadCounts(ctx, txn, userID, roomID, notificationCount, highlightCount)
 		return err
 	})
 	return
 }
 
 func (d *Database) GetUserUnreadNotificationCounts(ctx context.Context, userID string, from, to types.StreamPosition) (map[string]*eventutil.NotificationData, error) {
-	return d.NotificationData.SelectUserUnreadCounts(ctx, userID, from, to)
+	return d.NotificationData.SelectUserUnreadCounts(ctx, nil, userID, from, to)
 }
 
 func (d *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) {
@@ -1052,15 +1052,23 @@ func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID s
 }
 
 func (d *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) {
-	return d.Ignores.SelectIgnores(ctx, userID)
+	return d.Ignores.SelectIgnores(ctx, nil, userID)
 }
 
 func (d *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error {
-	return d.Ignores.UpsertIgnores(ctx, userID, ignores)
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.Ignores.UpsertIgnores(ctx, txn, userID, ignores)
+	})
 }
 
 func (d *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
-	return d.Presence.UpsertPresence(ctx, nil, userID, statusMsg, presence, lastActiveTS, fromSync)
+	var pos types.StreamPosition
+	var err error
+	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		pos, err = d.Presence.UpsertPresence(ctx, txn, userID, statusMsg, presence, lastActiveTS, fromSync)
+		return nil
+	})
+	return pos, err
 }
 
 func (d *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
diff --git a/syncapi/storage/sqlite3/ignores_table.go b/syncapi/storage/sqlite3/ignores_table.go
index f4afca55e..5ee1a9fa0 100644
--- a/syncapi/storage/sqlite3/ignores_table.go
+++ b/syncapi/storage/sqlite3/ignores_table.go
@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
@@ -61,10 +62,10 @@ func NewSqliteIgnoresTable(db *sql.DB) (tables.Ignores, error) {
 }
 
 func (s *ignoresStatements) SelectIgnores(
-	ctx context.Context, userID string,
+	ctx context.Context, txn *sql.Tx, userID string,
 ) (*types.IgnoredUsers, error) {
 	var ignoresData []byte
-	err := s.selectIgnoresStmt.QueryRowContext(ctx, userID).Scan(&ignoresData)
+	err := sqlutil.TxStmt(txn, s.selectIgnoresStmt).QueryRowContext(ctx, userID).Scan(&ignoresData)
 	if err != nil {
 		return nil, err
 	}
@@ -76,12 +77,12 @@ func (s *ignoresStatements) SelectIgnores(
 }
 
 func (s *ignoresStatements) UpsertIgnores(
-	ctx context.Context, userID string, ignores *types.IgnoredUsers,
+	ctx context.Context, txn *sql.Tx, userID string, ignores *types.IgnoredUsers,
 ) error {
 	ignoresJSON, err := json.Marshal(ignores)
 	if err != nil {
 		return err
 	}
-	_, err = s.upsertIgnoresStmt.ExecContext(ctx, userID, ignoresJSON)
+	_, err = sqlutil.TxStmt(txn, s.upsertIgnoresStmt).ExecContext(ctx, userID, ignoresJSON)
 	return err
 }
diff --git a/syncapi/storage/sqlite3/notification_data_table.go b/syncapi/storage/sqlite3/notification_data_table.go
index eaa11a8c0..66d4d4381 100644
--- a/syncapi/storage/sqlite3/notification_data_table.go
+++ b/syncapi/storage/sqlite3/notification_data_table.go
@@ -72,7 +72,7 @@ const selectUserUnreadNotificationCountsSQL = `SELECT
 
 const selectMaxNotificationIDSQL = `SELECT CASE COUNT(*) WHEN 0 THEN 0 ELSE MAX(id) END FROM syncapi_notification_data`
 
-func (r *notificationDataStatements) UpsertRoomUnreadCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
+func (r *notificationDataStatements) UpsertRoomUnreadCounts(ctx context.Context, txn *sql.Tx, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
 	pos, err = r.streamIDStatements.nextNotificationID(ctx, nil)
 	if err != nil {
 		return
@@ -81,8 +81,8 @@ func (r *notificationDataStatements) UpsertRoomUnreadCounts(ctx context.Context,
 	return
 }
 
-func (r *notificationDataStatements) SelectUserUnreadCounts(ctx context.Context, userID string, fromExcl, toIncl types.StreamPosition) (map[string]*eventutil.NotificationData, error) {
-	rows, err := r.selectUserUnreadCounts.QueryContext(ctx, userID, fromExcl, toIncl)
+func (r *notificationDataStatements) SelectUserUnreadCounts(ctx context.Context, txn *sql.Tx, userID string, fromExcl, toIncl types.StreamPosition) (map[string]*eventutil.NotificationData, error) {
+	rows, err := sqlutil.TxStmt(txn, r.selectUserUnreadCounts).QueryContext(ctx, userID, fromExcl, toIncl)
 	if err != nil {
 		return nil, err
 	}
@@ -107,8 +107,8 @@ func (r *notificationDataStatements) SelectUserUnreadCounts(ctx context.Context,
 	return roomCounts, rows.Err()
 }
 
-func (r *notificationDataStatements) SelectMaxID(ctx context.Context) (int64, error) {
+func (r *notificationDataStatements) SelectMaxID(ctx context.Context, txn *sql.Tx) (int64, error) {
 	var id int64
-	err := r.selectMaxID.QueryRowContext(ctx).Scan(&id)
+	err := sqlutil.TxStmt(txn, r.selectMaxID).QueryRowContext(ctx).Scan(&id)
 	return id, err
 }
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 468d26aca..193881b44 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -189,14 +189,14 @@ type Memberships interface {
 }
 
 type NotificationData interface {
-	UpsertRoomUnreadCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (types.StreamPosition, error)
-	SelectUserUnreadCounts(ctx context.Context, userID string, fromExcl, toIncl types.StreamPosition) (map[string]*eventutil.NotificationData, error)
-	SelectMaxID(ctx context.Context) (int64, error)
+	UpsertRoomUnreadCounts(ctx context.Context, txn *sql.Tx, userID, roomID string, notificationCount, highlightCount int) (types.StreamPosition, error)
+	SelectUserUnreadCounts(ctx context.Context, txn *sql.Tx, userID string, fromExcl, toIncl types.StreamPosition) (map[string]*eventutil.NotificationData, error)
+	SelectMaxID(ctx context.Context, txn *sql.Tx) (int64, error)
 }
 
 type Ignores interface {
-	SelectIgnores(ctx context.Context, userID string) (*types.IgnoredUsers, error)
-	UpsertIgnores(ctx context.Context, userID string, ignores *types.IgnoredUsers) error
+	SelectIgnores(ctx context.Context, txn *sql.Tx, userID string) (*types.IgnoredUsers, error)
+	UpsertIgnores(ctx context.Context, txn *sql.Tx, userID string, ignores *types.IgnoredUsers) error
 }
 
 type Presence interface {

From 8196b296570de229eb42501ade5aabbe7ba124f9 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 9 Sep 2022 14:14:52 +0200
Subject: [PATCH 342/391] Change detection of already executed migrations
 (#2665)

This changes the detection of already executed migrations for the
roomserver state block and keychange refactor. It now uses schema tables
provided by the database engine to check if the column was already
removed. We now also store the migration in the migrations table.

This should stop e.g. Postgres from logging errors like `ERROR: column
"event_nid" does not exist at character 8`.
---
 internal/sqlutil/migrate.go                   | 21 ++++++-
 .../storage/postgres/key_changes_table.go     | 46 +++++++++------
 .../storage/sqlite3/key_changes_table.go      | 44 ++++++++++----
 roomserver/storage/postgres/storage.go        | 58 +++++++++++--------
 roomserver/storage/sqlite3/storage.go         | 44 +++++++++-----
 5 files changed, 141 insertions(+), 72 deletions(-)

diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go
index 18020a902..98e7d8938 100644
--- a/internal/sqlutil/migrate.go
+++ b/internal/sqlutil/migrate.go
@@ -21,8 +21,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/internal"
 )
 
 const createDBMigrationsSQL = "" +
@@ -95,11 +96,11 @@ func (m *Migrator) Up(ctx context.Context) error {
 		for i := range m.migrations {
 			now := time.Now().UTC().Format(time.RFC3339)
 			migration := m.migrations[i]
-			logrus.Debugf("Executing database migration '%s'", migration.Version)
 			// Skip migration if it was already executed
 			if _, ok := executedMigrations[migration.Version]; ok {
 				continue
 			}
+			logrus.Debugf("Executing database migration '%s'", migration.Version)
 			err = migration.Up(ctx, txn)
 			if err != nil {
 				return fmt.Errorf("unable to execute migration '%s': %w", migration.Version, err)
@@ -140,3 +141,19 @@ func (m *Migrator) ExecutedMigrations(ctx context.Context) (map[string]struct{},
 
 	return result, rows.Err()
 }
+
+// InsertMigration creates the migrations table if it doesn't exist and
+// inserts a migration given their name to the database.
+// This should only be used when manually inserting migrations.
+func InsertMigration(ctx context.Context, db *sql.DB, migrationName string) error {
+	_, err := db.ExecContext(ctx, createDBMigrationsSQL)
+	if err != nil {
+		return fmt.Errorf("unable to create db_migrations: %w", err)
+	}
+	_, err = db.ExecContext(ctx, insertVersionSQL,
+		migrationName,
+		time.Now().Format(time.RFC3339),
+		internal.VersionString(),
+	)
+	return err
+}
diff --git a/keyserver/storage/postgres/key_changes_table.go b/keyserver/storage/postgres/key_changes_table.go
index 004f15d82..e91b048d5 100644
--- a/keyserver/storage/postgres/key_changes_table.go
+++ b/keyserver/storage/postgres/key_changes_table.go
@@ -17,8 +17,9 @@ package postgres
 import (
 	"context"
 	"database/sql"
+	"errors"
 
-	"github.com/lib/pq"
+	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -63,30 +64,37 @@ func NewPostgresKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) {
 		return s, err
 	}
 
+	if err = executeMigration(context.Background(), db); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func executeMigration(ctx context.Context, db *sql.DB) error {
 	// TODO: Remove when we are sure we are not having goose artefacts in the db
 	// This forces an error, which indicates the migration is already applied, since the
 	// column partition was removed from the table
-	var count int
-	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan(&count)
-	if err == nil {
-		m := sqlutil.NewMigrator(db)
-		m.AddMigrations(sqlutil.Migration{
-			Version: "keyserver: refactor key changes",
-			Up:      deltas.UpRefactorKeyChanges,
-		})
-		return s, m.Up(context.Background())
-	} else {
-		switch e := err.(type) {
-		case *pq.Error:
-			// ignore undefined_column (42703) errors, as this is expected at this point
-			if e.Code != "42703" {
-				return nil, err
+	migrationName := "keyserver: refactor key changes"
+
+	var cName string
+	err := db.QueryRowContext(ctx, "select column_name from information_schema.columns where table_name = 'keyserver_key_changes' AND column_name = 'partition'").Scan(&cName)
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
+			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
+				// not a fatal error, log and continue
+				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
 			}
-		default:
-			return nil, err
+			return nil
 		}
+		return err
 	}
-	return s, nil
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: migrationName,
+		Up:      deltas.UpRefactorKeyChanges,
+	})
+
+	return m.Up(ctx)
 }
 
 func (s *keyChangesStatements) Prepare() (err error) {
diff --git a/keyserver/storage/sqlite3/key_changes_table.go b/keyserver/storage/sqlite3/key_changes_table.go
index 217fa7a5d..b68df5552 100644
--- a/keyserver/storage/sqlite3/key_changes_table.go
+++ b/keyserver/storage/sqlite3/key_changes_table.go
@@ -17,6 +17,9 @@ package sqlite3
 import (
 	"context"
 	"database/sql"
+	"errors"
+
+	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -58,23 +61,40 @@ func NewSqliteKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) {
 	if err != nil {
 		return s, err
 	}
-	// TODO: Remove when we are sure we are not having goose artefacts in the db
-	// This forces an error, which indicates the migration is already applied, since the
-	// column partition was removed from the table
-	var count int
-	err = db.QueryRow("SELECT partition FROM keyserver_key_changes LIMIT 1;").Scan(&count)
-	if err == nil {
-		m := sqlutil.NewMigrator(db)
-		m.AddMigrations(sqlutil.Migration{
-			Version: "keyserver: refactor key changes",
-			Up:      deltas.UpRefactorKeyChanges,
-		})
-		return s, m.Up(context.Background())
+
+	if err = executeMigration(context.Background(), db); err != nil {
+		return nil, err
 	}
 
 	return s, nil
 }
 
+func executeMigration(ctx context.Context, db *sql.DB) error {
+	// TODO: Remove when we are sure we are not having goose artefacts in the db
+	// This forces an error, which indicates the migration is already applied, since the
+	// column partition was removed from the table
+	migrationName := "keyserver: refactor key changes"
+
+	var cName string
+	err := db.QueryRowContext(ctx, `SELECT p.name FROM sqlite_master AS m JOIN pragma_table_info(m.name) AS p WHERE m.name = 'keyserver_key_changes' AND p.name = 'partition'`).Scan(&cName)
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
+			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
+				// not a fatal error, log and continue
+				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
+			}
+			return nil
+		}
+		return err
+	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: migrationName,
+		Up:      deltas.UpRefactorKeyChanges,
+	})
+	return m.Up(ctx)
+}
+
 func (s *keyChangesStatements) Prepare() (err error) {
 	if s.upsertKeyChangeStmt, err = s.db.Prepare(upsertKeyChangeSQL); err != nil {
 		return err
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index f47a64c80..26178df83 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -16,10 +16,13 @@
 package postgres
 
 import (
+	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 
-	"github.com/lib/pq"
+	"github.com/sirupsen/logrus"
+
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
 
@@ -52,30 +55,8 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 
 	// Special case, since this migration uses several tables, so it needs to
 	// be sure that all tables are created first.
-	// TODO: Remove when we are sure we are not having goose artefacts in the db
-	// This forces an error, which indicates the migration is already applied, since the
-	// column event_nid was removed from the table
-	var eventNID int
-	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan(&eventNID)
-	if err == nil {
-		m := sqlutil.NewMigrator(db)
-		m.AddMigrations(sqlutil.Migration{
-			Version: "roomserver: state blocks refactor",
-			Up:      deltas.UpStateBlocksRefactor,
-		})
-		if err = m.Up(base.Context()); err != nil {
-			return nil, err
-		}
-	} else {
-		switch e := err.(type) {
-		case *pq.Error:
-			// ignore undefined_column (42703) errors, as this is expected at this point
-			if e.Code != "42703" {
-				return nil, err
-			}
-		default:
-			return nil, err
-		}
+	if err = executeMigration(base.Context(), db); err != nil {
+		return nil, err
 	}
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
@@ -87,6 +68,33 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 	return &d, nil
 }
 
+func executeMigration(ctx context.Context, db *sql.DB) error {
+	// TODO: Remove when we are sure we are not having goose artefacts in the db
+	// This forces an error, which indicates the migration is already applied, since the
+	// column event_nid was removed from the table
+	migrationName := "roomserver: state blocks refactor"
+
+	var cName string
+	err := db.QueryRowContext(ctx, "select column_name from information_schema.columns where table_name = 'roomserver_state_block' AND column_name = 'event_nid'").Scan(&cName)
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
+			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
+				// not a fatal error, log and continue
+				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
+			}
+			return nil
+		}
+		return err
+	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: migrationName,
+		Up:      deltas.UpStateBlocksRefactor,
+	})
+
+	return m.Up(ctx)
+}
+
 func (d *Database) create(db *sql.DB) error {
 	if err := CreateEventStateKeysTable(db); err != nil {
 		return err
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 9f8a1b118..c7bc00393 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -18,9 +18,11 @@ package sqlite3
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -61,20 +63,8 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 
 	// Special case, since this migration uses several tables, so it needs to
 	// be sure that all tables are created first.
-	// TODO: Remove when we are sure we are not having goose artefacts in the db
-	// This forces an error, which indicates the migration is already applied, since the
-	// column event_nid was removed from the table
-	var eventNID int
-	err = db.QueryRow("SELECT event_nid FROM roomserver_state_block LIMIT 1;").Scan(&eventNID)
-	if err == nil {
-		m := sqlutil.NewMigrator(db)
-		m.AddMigrations(sqlutil.Migration{
-			Version: "roomserver: state blocks refactor",
-			Up:      deltas.UpStateBlocksRefactor,
-		})
-		if err = m.Up(base.Context()); err != nil {
-			return nil, err
-		}
+	if err = executeMigration(base.Context(), db); err != nil {
+		return nil, err
 	}
 
 	// Then prepare the statements. Now that the migrations have run, any columns referred
@@ -86,6 +76,32 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
 	return &d, nil
 }
 
+func executeMigration(ctx context.Context, db *sql.DB) error {
+	// TODO: Remove when we are sure we are not having goose artefacts in the db
+	// This forces an error, which indicates the migration is already applied, since the
+	// column event_nid was removed from the table
+	migrationName := "roomserver: state blocks refactor"
+
+	var cName string
+	err := db.QueryRowContext(ctx, `SELECT p.name FROM sqlite_master AS m JOIN pragma_table_info(m.name) AS p WHERE m.name = 'roomserver_state_block' AND p.name = 'event_nid'`).Scan(&cName)
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
+			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
+				// not a fatal error, log and continue
+				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
+			}
+			return nil
+		}
+		return err
+	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: migrationName,
+		Up:      deltas.UpStateBlocksRefactor,
+	})
+	return m.Up(ctx)
+}
+
 func (d *Database) create(db *sql.DB) error {
 	if err := CreateEventStateKeysTable(db); err != nil {
 		return err

From c0e17bbe1bfbfdb93b7ab6baffc4a53d94ad7937 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Sep 2022 13:13:04 +0100
Subject: [PATCH 343/391] Fix transactions around assigning NIDs

---
 .../storage/shared/membership_updater.go      |  6 +--
 roomserver/storage/shared/storage.go          | 44 ++++++++++++-------
 2 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go
index 07fb697f9..f9c889cb1 100644
--- a/roomserver/storage/shared/membership_updater.go
+++ b/roomserver/storage/shared/membership_updater.go
@@ -26,11 +26,11 @@ func NewMembershipUpdater(
 	var targetUserNID types.EventStateKeyNID
 	var err error
 	err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error {
-		roomNID, err = d.assignRoomNID(ctx, roomID, roomVersion)
+		roomNID, err = d.assignRoomNID(ctx, txn, roomID, roomVersion)
 		if err != nil {
 			return err
 		}
-		targetUserNID, err = d.assignStateKeyNID(ctx, targetUserID)
+		targetUserNID, err = d.assignStateKeyNID(ctx, txn, targetUserID)
 		if err != nil {
 			return err
 		}
@@ -101,7 +101,7 @@ func (u *MembershipUpdater) Update(newMembership tables.MembershipState, event *
 	var inserted bool    // Did the query result in a membership change?
 	var retired []string // Did we retire any updates in the process?
 	return inserted, retired, u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
-		senderUserNID, err := u.d.assignStateKeyNID(u.ctx, event.Sender())
+		senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, event.Sender())
 		if err != nil {
 			return fmt.Errorf("u.d.AssignStateKeyNID: %w", err)
 		}
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index f35592a76..00a17e5cb 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -402,7 +402,7 @@ func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error {
 func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomforgotten bool, err error) {
 	var requestSenderUserNID types.EventStateKeyNID
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		requestSenderUserNID, err = d.assignStateKeyNID(ctx, requestSenderUserID)
+		requestSenderUserNID, err = d.assignStateKeyNID(ctx, txn, requestSenderUserID)
 		return err
 	})
 	if err != nil {
@@ -596,7 +596,9 @@ func (d *Database) storeEvent(
 	if updater != nil && updater.txn != nil {
 		txn = updater.txn
 	}
-	err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error {
+	// First writer is with a database-provided transaction, so that NIDs are assigned
+	// globally outside of the updater context, to help avoid races.
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		// TODO: Here we should aim to have two different code paths for new rooms
 		// vs existing ones.
 
@@ -611,11 +613,11 @@ func (d *Database) storeEvent(
 			return fmt.Errorf("extractRoomVersionFromCreateEvent: %w", err)
 		}
 
-		if roomNID, err = d.assignRoomNID(ctx, event.RoomID(), roomVersion); err != nil {
+		if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion); err != nil {
 			return fmt.Errorf("d.assignRoomNID: %w", err)
 		}
 
-		if eventTypeNID, err = d.assignEventTypeNID(ctx, event.Type()); err != nil {
+		if eventTypeNID, err = d.assignEventTypeNID(ctx, txn, event.Type()); err != nil {
 			return fmt.Errorf("d.assignEventTypeNID: %w", err)
 		}
 
@@ -623,11 +625,19 @@ func (d *Database) storeEvent(
 		// Assigned a numeric ID for the state_key if there is one present.
 		// Otherwise set the numeric ID for the state_key to 0.
 		if eventStateKey != nil {
-			if eventStateKeyNID, err = d.assignStateKeyNID(ctx, *eventStateKey); err != nil {
+			if eventStateKeyNID, err = d.assignStateKeyNID(ctx, txn, *eventStateKey); err != nil {
 				return fmt.Errorf("d.assignStateKeyNID: %w", err)
 			}
 		}
 
+		return nil
+	})
+	if err != nil {
+		return 0, 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.Writer.Do: %w", err)
+	}
+	// Second writer is using the database-provided transaction, probably from the
+	// room updater, for easy roll-back if required.
+	err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error {
 		if eventNID, stateNID, err = d.EventsTable.InsertEvent(
 			ctx,
 			txn,
@@ -749,48 +759,48 @@ func (d *Database) MissingAuthPrevEvents(
 }
 
 func (d *Database) assignRoomNID(
-	ctx context.Context, roomID string, roomVersion gomatrixserverlib.RoomVersion,
+	ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion,
 ) (types.RoomNID, error) {
 	// Check if we already have a numeric ID in the database.
-	roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID)
+	roomNID, err := d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
 	if err == sql.ErrNoRows {
 		// We don't have a numeric ID so insert one into the database.
-		roomNID, err = d.RoomsTable.InsertRoomNID(ctx, nil, roomID, roomVersion)
+		roomNID, err = d.RoomsTable.InsertRoomNID(ctx, txn, roomID, roomVersion)
 		if err == sql.ErrNoRows {
 			// We raced with another insert so run the select again.
-			roomNID, err = d.RoomsTable.SelectRoomNID(ctx, nil, roomID)
+			roomNID, err = d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
 		}
 	}
 	return roomNID, err
 }
 
 func (d *Database) assignEventTypeNID(
-	ctx context.Context, eventType string,
+	ctx context.Context, txn *sql.Tx, eventType string,
 ) (types.EventTypeNID, error) {
 	// Check if we already have a numeric ID in the database.
-	eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, nil, eventType)
+	eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
 	if err == sql.ErrNoRows {
 		// We don't have a numeric ID so insert one into the database.
-		eventTypeNID, err = d.EventTypesTable.InsertEventTypeNID(ctx, nil, eventType)
+		eventTypeNID, err = d.EventTypesTable.InsertEventTypeNID(ctx, txn, eventType)
 		if err == sql.ErrNoRows {
 			// We raced with another insert so run the select again.
-			eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, nil, eventType)
+			eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
 		}
 	}
 	return eventTypeNID, err
 }
 
 func (d *Database) assignStateKeyNID(
-	ctx context.Context, eventStateKey string,
+	ctx context.Context, txn *sql.Tx, eventStateKey string,
 ) (types.EventStateKeyNID, error) {
 	// Check if we already have a numeric ID in the database.
-	eventStateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, eventStateKey)
+	eventStateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
 	if err == sql.ErrNoRows {
 		// We don't have a numeric ID so insert one into the database.
-		eventStateKeyNID, err = d.EventStateKeysTable.InsertEventStateKeyNID(ctx, nil, eventStateKey)
+		eventStateKeyNID, err = d.EventStateKeysTable.InsertEventStateKeyNID(ctx, txn, eventStateKey)
 		if err == sql.ErrNoRows {
 			// We raced with another insert so run the select again.
-			eventStateKeyNID, err = d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, eventStateKey)
+			eventStateKeyNID, err = d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
 		}
 	}
 	return eventStateKeyNID, err

From e1bc4f6a1ed446380408fad902ed1e314bcd99c5 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Sep 2022 13:31:55 +0100
Subject: [PATCH 344/391] Fix database transaction for keyserver
 `DeleteDeviceKeys`

---
 keyserver/storage/shared/storage.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go
index 0e587b5a8..5beeed0f1 100644
--- a/keyserver/storage/shared/storage.go
+++ b/keyserver/storage/shared/storage.go
@@ -158,7 +158,7 @@ func (d *Database) MarkDeviceListStale(ctx context.Context, userID string, isSta
 // DeleteDeviceKeys removes the device keys for a given user/device, and any accompanying
 // cross-signing signatures relating to that device.
 func (d *Database) DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []gomatrixserverlib.KeyID) error {
-	return d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		for _, deviceID := range deviceIDs {
 			if err := d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget(ctx, txn, userID, deviceID); err != nil && err != sql.ErrNoRows {
 				return fmt.Errorf("d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget: %w", err)

From 6ee758df63fc2c5f107954e9b436d361cc147741 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Sep 2022 13:50:50 +0100
Subject: [PATCH 345/391] Optimise shared users query in Synx API slightly by
 removing a potential sort

---
 syncapi/storage/postgres/current_room_state_table.go | 2 +-
 syncapi/storage/sqlite3/current_room_state_table.go  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 58f404511..37294e46f 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -111,7 +111,7 @@ const selectEventsWithEventIDsSQL = "" +
 
 const selectSharedUsersSQL = "" +
 	"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
-	"	SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
+	"	SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
 	") AND state_key = ANY($2) AND membership IN ('join', 'invite');"
 
 type currentRoomStateStatements struct {
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 3a10b2325..2765c14b2 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -95,7 +95,7 @@ const selectEventsWithEventIDsSQL = "" +
 
 const selectSharedUsersSQL = "" +
 	"SELECT state_key FROM syncapi_current_room_state WHERE room_id IN(" +
-	"	SELECT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
+	"	SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
 	") AND state_key IN ($2) AND membership IN ('join', 'invite');"
 
 type currentRoomStateStatements struct {

From 955e69a3b7c3f3bb066cb47be6c49d4b03ff2891 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Sep 2022 14:18:45 +0100
Subject: [PATCH 346/391] Optimise `SharedUsers` again by using complete
 composite index

---
 syncapi/storage/postgres/current_room_state_table.go | 2 +-
 syncapi/storage/sqlite3/current_room_state_table.go  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 37294e46f..083d10b88 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -112,7 +112,7 @@ const selectEventsWithEventIDsSQL = "" +
 const selectSharedUsersSQL = "" +
 	"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
 	"	SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
-	") AND state_key = ANY($2) AND membership IN ('join', 'invite');"
+	") AND type = 'm.room.member' AND state_key = ANY($2) AND membership IN ('join', 'invite');"
 
 type currentRoomStateStatements struct {
 	upsertRoomStateStmt                *sql.Stmt
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index 2765c14b2..b88c11f88 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -96,7 +96,7 @@ const selectEventsWithEventIDsSQL = "" +
 const selectSharedUsersSQL = "" +
 	"SELECT state_key FROM syncapi_current_room_state WHERE room_id IN(" +
 	"	SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" +
-	") AND state_key IN ($2) AND membership IN ('join', 'invite');"
+	") AND type = 'm.room.member' AND state_key IN ($2) AND membership IN ('join', 'invite');"
 
 type currentRoomStateStatements struct {
 	db                                 *sql.DB

From e8687f6f8255a978a8200b19c7a28581397ad455 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 9 Sep 2022 15:01:13 +0100
Subject: [PATCH 347/391] Version 0.9.7 (#2707)

Changelog and version bump.
---
 CHANGES.md          | 21 +++++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 7d0575663..024272eb9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,26 @@
 # Changelog
 
+## Dendrite 0.9.7 (2022-09-09)
+
+### Features
+
+* Initial supporting code to enable full-text search has been merged (although not ready for use yet)
+* Newly created rooms now have higher default power levels for enabling encryption, setting server ACLs or sending tombstone events
+* Incoming signing key updates over federation are now queued in JetStream for processing, so that they cannot be dropped accidentally
+
+### Fixes
+
+* A race condition between the roomserver output events being generated, forward extremities being updated and room info being updated has been fixed
+* Appservices will no longer receive invite events which they are not interested in, which caused heavy load in some cases or excessive request sizes in others
+* A bug in state resolution v2 where events could incorrectly be classified as control events has been fixed
+* A bug in state resolution v2 where some specific events with unexpected non-empty state keys are dropped has been fixed
+* A bug in state resolution v2 when fetching auth events vs partial state has been fixed
+* Stale device lists should now be handled correctly for all user IDs, which may help with E2EE reliability
+* A number of database writer issues have been fixed in the user API and sync API, which should help to reduce `database is locked` errors with SQLite databases
+* Database migrations should now be detected more reliably to prevent unexpected errors at startup
+* A number of minor database transaction issues have been fixed, particularly for assigning NIDs in the roomserver, cleaning up device keys and cleaning up notifications
+* The database query for finding shared users in the sync API has been optimised, using significantly less CPU time as a result
+
 ## Dendrite 0.9.6 (2022-09-01)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index c9a9f3af5..ed2ae442f 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 6
+	VersionPatch = 7
 	VersionTag   = "" // example: "rc1"
 )
 

From af9a204cc0b997bf0b7fd6ea434a7c04157bdf28 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 9 Sep 2022 17:19:29 +0200
Subject: [PATCH 348/391] Only verify if fulltext indexing is enabled

---
 setup/config/config_syncapi.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go
index 196afb82c..c890b0054 100644
--- a/setup/config/config_syncapi.go
+++ b/setup/config/config_syncapi.go
@@ -59,6 +59,9 @@ func (f *Fulltext) Defaults(opts DefaultOpts) {
 }
 
 func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	if !f.Enabled {
+		return
+	}
 	checkNotEmpty(configErrs, "syncapi.fulltext.index_path", string(f.IndexPath))
 	checkNotEmpty(configErrs, "syncapi.fulltext.language", f.Language)
 }

From 5997c3245286b4a8df57054b1f311e5957836910 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Sun, 11 Sep 2022 15:39:11 +0100
Subject: [PATCH 349/391] Update to
 matrix-org/gomatrixserverlib@dec87dbaa40700a8c8f7c299c53f911047cf2cb9

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 572453974..68e48bef6 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407
 	github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 00392cb65..7196104fd 100644
--- a/go.sum
+++ b/go.sum
@@ -390,6 +390,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661 h1:dww9rH0HVfAO9JOBD1nxq26GHKbEw07thAJTu1DrAQs=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407 h1:UciyfR3UTWnpqFBvEAMwGmZpjjO2hemFkWmwa/tB+fw=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ=
 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 3a9dde28fda81067a2127258b09ff7c9accd5256 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Sep 2022 10:19:02 +0100
Subject: [PATCH 350/391] P2P demo tweaks

---
 build/gobind-pinecone/monolith.go   |  8 ++++++++
 build/gobind-yggdrasil/monolith.go  |  8 ++++++++
 cmd/dendrite-demo-pinecone/main.go  | 10 +++++++++-
 cmd/dendrite-demo-yggdrasil/main.go |  9 ++++++++-
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index b51e97786..91bbb6873 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -218,6 +218,9 @@ func (m *DendriteMonolith) Start() {
 			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 				panic("failed to load PEM key: " + err.Error())
 			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
 		} else {
 			if sk, err = os.ReadFile(oldkeyfile); err != nil {
 				panic("failed to read the old private key: " + err.Error())
@@ -233,8 +236,13 @@ func (m *DendriteMonolith) Start() {
 		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 			panic("failed to load PEM key: " + err.Error())
 		}
+		if len(sk) != ed25519.PrivateKeySize {
+			panic("the private key is not long enough")
+		}
 	}
 
+	pk = sk.Public().(ed25519.PublicKey)
+
 	var err error
 	m.listener, err = net.Listen("tcp", "localhost:65432")
 	if err != nil {
diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go
index c671ae098..248b6c324 100644
--- a/build/gobind-yggdrasil/monolith.go
+++ b/build/gobind-yggdrasil/monolith.go
@@ -87,6 +87,9 @@ func (m *DendriteMonolith) Start() {
 			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 				panic("failed to load PEM key: " + err.Error())
 			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
 		} else {
 			if sk, err = os.ReadFile(oldkeyfile); err != nil {
 				panic("failed to read the old private key: " + err.Error())
@@ -103,8 +106,13 @@ func (m *DendriteMonolith) Start() {
 		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 			panic("failed to load PEM key: " + err.Error())
 		}
+		if len(sk) != ed25519.PrivateKeySize {
+			panic("the private key is not long enough")
+		}
 	}
 
+	pk = sk.Public().(ed25519.PublicKey)
+
 	var err error
 	m.listener, err = net.Listen("tcp", "localhost:65432")
 	if err != nil {
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 83b52132a..ff9cb5aa1 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -98,6 +98,9 @@ func main() {
 				if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 					panic("failed to load PEM key: " + err.Error())
 				}
+				if len(sk) != ed25519.PrivateKeySize {
+					panic("the private key is not long enough")
+				}
 			} else {
 				if sk, err = os.ReadFile(oldkeyfile); err != nil {
 					panic("failed to read the old private key: " + err.Error())
@@ -114,7 +117,13 @@ func main() {
 			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 				panic("failed to load PEM key: " + err.Error())
 			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
 		}
+
+		pk = sk.Public().(ed25519.PublicKey)
+
 		cfg.Defaults(config.DefaultOpts{
 			Generate:   true,
 			Monolithic: true,
@@ -136,7 +145,6 @@ func main() {
 		}
 	}
 
-	pk = sk.Public().(ed25519.PublicKey)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 7445b6bfe..868d9826a 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -86,6 +86,9 @@ func main() {
 			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 				panic("failed to load PEM key: " + err.Error())
 			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
 		} else {
 			if sk, err = os.ReadFile(oldkeyfile); err != nil {
 				panic("failed to read the old private key: " + err.Error())
@@ -102,8 +105,13 @@ func main() {
 		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
 			panic("failed to load PEM key: " + err.Error())
 		}
+		if len(sk) != ed25519.PrivateKeySize {
+			panic("the private key is not long enough")
+		}
 	}
 
+	pk = sk.Public().(ed25519.PublicKey)
+
 	// use custom config if config flag is set
 	if configFlagSet {
 		cfg = setup.ParseFlags(true)
@@ -129,7 +137,6 @@ func main() {
 		}
 	}
 
-	pk = sk.Public().(ed25519.PublicKey)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 

From e9af30b3fe8ca1c426ed63822bf9fd0e4b2c9572 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Sep 2022 10:35:04 +0100
Subject: [PATCH 351/391] Update to matrix-org/pinecone@b215925

---
 go.mod | 4 ++--
 go.sum | 6 ++----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 68e48bef6..258197ad0 100644
--- a/go.mod
+++ b/go.mod
@@ -17,12 +17,13 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
+	github.com/kardianos/minwinsvc v1.0.0
 	github.com/lib/pq v1.10.5
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407
-	github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed
+	github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
 	github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c
@@ -92,7 +93,6 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
-	github.com/kardianos/minwinsvc v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/lucas-clemente/quic-go v0.28.1 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
diff --git a/go.sum b/go.sum
index 7196104fd..6670711cc 100644
--- a/go.sum
+++ b/go.sum
@@ -388,12 +388,10 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661 h1:dww9rH0HVfAO9JOBD1nxq26GHKbEw07thAJTu1DrAQs=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407 h1:UciyfR3UTWnpqFBvEAMwGmZpjjO2hemFkWmwa/tB+fw=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
-github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ=
-github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
+github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534 h1:XuJYAJNkdG3zj9cO0yQSvL+Sp2xogsTOuZRx7PwdtoA=
+github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 7595fbf58c67fd8f0f879498f865f4871e652ddb Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Sep 2022 13:24:03 +0100
Subject: [PATCH 352/391] Update to NATS Server v2.9.0

---
 go.mod | 10 +++++-----
 go.sum | 19 ++++++++++---------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/go.mod b/go.mod
index 258197ad0..a3e52c1a4 100644
--- a/go.mod
+++ b/go.mod
@@ -26,8 +26,8 @@ require (
 	github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
-	github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c
-	github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995
+	github.com/nats-io/nats-server/v2 v2.9.0
+	github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
@@ -43,7 +43,7 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
+	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
 	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
 	golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105
 	golang.org/x/net v0.0.0-20220728211354-c7608f3a8462
@@ -124,9 +124,9 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect
+	golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77 // indirect
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
-	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
+	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
diff --git a/go.sum b/go.sum
index 6670711cc..4ab56010c 100644
--- a/go.sum
+++ b/go.sum
@@ -429,10 +429,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
 github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
-github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c h1:U5qngWGZ7E/nQxz0544IpIEdKFUUaOJxQN2LHCYLGhg=
-github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c/go.mod h1:+f++B/5jpr71JATt7b5KCX+G7bt43iWx1OYWGkpE/Kk=
-github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995 h1:CUcSQR8jwa9//qNgN/t3tW53DObnTPQ/G/K+qnS7yRc=
-github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/nats-io/nats-server/v2 v2.9.0 h1:DLWu+7/VgGOoChcDKytnUZPAmudpv7o/MhKmNrnH1RE=
+github.com/nats-io/nats-server/v2 v2.9.0/go.mod h1:BWKY6217RvhI+FDoOLZ2BH+hOC37xeKRBlQ1Lz7teKI=
+github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0 h1:dPUKD6Iv8M1y9MU8PK6H4a4/12yx5/CbaYWz/Z1arY8=
+github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
 github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -627,8 +627,9 @@ golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
+golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -812,8 +813,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80=
-golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77 h1:C1tElbkWrsSkn3IRl1GCW/gETw1TywWIPgwZtXTZbYg=
+golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
@@ -831,8 +832,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
-golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
+golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From 2792d0490f3771488bad346981b8c26479a872c3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Sep 2022 12:30:41 +0100
Subject: [PATCH 353/391] Fix missing signature check on the
 `/get_missing_events` response

---
 roomserver/internal/input/input_missing.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go
index 0dd2b64c0..d789c3a14 100644
--- a/roomserver/internal/input/input_missing.go
+++ b/roomserver/internal/input/input_missing.go
@@ -553,11 +553,14 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e *gomatrixserve
 
 	// Make sure events from the missingResp are using the cache - missing events
 	// will be added and duplicates will be removed.
-	logger.Debugf("get_missing_events returned %d events", len(missingResp.Events))
 	missingEvents := make([]*gomatrixserverlib.Event, 0, len(missingResp.Events))
 	for _, ev := range missingResp.Events.UntrustedEvents(roomVersion) {
+		if err = ev.VerifyEventSignatures(ctx, t.keys); err != nil {
+			continue
+		}
 		missingEvents = append(missingEvents, t.cacheAndReturn(ev))
 	}
+	logger.Debugf("get_missing_events returned %d events (%d passed signature checks)", len(missingResp.Events), len(missingEvents))
 
 	// topologically sort and sanity check that we are making forward progress
 	newEvents = gomatrixserverlib.ReverseTopologicalOrdering(missingEvents, gomatrixserverlib.TopologicalOrderByPrevEvents)

From 47b2a5d6b8d2704950170cc636ed3dabdad59c06 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Sep 2022 13:41:07 +0100
Subject: [PATCH 354/391] Version 0.9.8

---
 CHANGES.md          | 14 ++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 024272eb9..79d2fe2e1 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,19 @@
 # Changelog
 
+## Dendrite 0.9.8 (2022-09-12)
+
+### Important
+
+* This is a **security release** to fix a vulnerability where missing events retrieved from other servers did not have their signatures verified in all cases, affecting all versions of Dendrite before 0.9.8. Upgrading to this version is highly recommended. For more information, [see here](https://github.com/matrix-org/dendrite/security/advisories/GHSA-pfw4-xjgm-267c).
+
+### Features
+
+* The built-in NATS Server has been updated to the final 2.9.0 release version
+
+### Fixes
+
+* Dendrite will now correctly verify the signatures of events retrieved using `/get_missing_events`
+
 ## Dendrite 0.9.7 (2022-09-09)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index ed2ae442f..ef7b879c2 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 7
+	VersionPatch = 8
 	VersionTag   = "" // example: "rc1"
 )
 

From 62afb936a5d282d0f500645c3442c1c294d37bdb Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 12 Sep 2022 15:27:45 +0100
Subject: [PATCH 355/391] Update to matrix-org/gomatrixserverlib@7b96db4

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index a3e52c1a4..d151b2995 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a
 	github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index 4ab56010c..3cad3f530 100644
--- a/go.sum
+++ b/go.sum
@@ -388,8 +388,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407 h1:UciyfR3UTWnpqFBvEAMwGmZpjjO2hemFkWmwa/tB+fw=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220911125436-dec87dbaa407/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a h1:XeGBDZZsUe4kgj3myl0EiuDNVWxszJecMTrON3Wn9sI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534 h1:XuJYAJNkdG3zj9cO0yQSvL+Sp2xogsTOuZRx7PwdtoA=
 github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 100fa9b2354efa05b4fbff3a3cb745ea7783d41c Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 13 Sep 2022 08:07:43 +0200
Subject: [PATCH 356/391] Check unique constraint errors when manually
 inserting migrations (#2712)

This should avoid unnecessary logging on startup if the migration (were
we need `InsertMigration`) was already executed.
This now checks for "unique constraint errors" for SQLite and Postgres
and fails the startup process if the migration couldn't be manually
inserted for some other reason.
---
 internal/sqlutil/migrate.go                   |  5 +++++
 ...{postgres_wasm.go => unique_constraint.go} | 21 ++++++++++++++++---
 ...{postgres.go => unique_constraint_wasm.go} | 17 +++++++++------
 .../storage/postgres/key_changes_table.go     |  6 ++----
 .../storage/sqlite3/key_changes_table.go      |  6 ++----
 roomserver/storage/postgres/storage.go        |  5 +----
 roomserver/storage/sqlite3/storage.go         |  4 +---
 7 files changed, 40 insertions(+), 24 deletions(-)
 rename internal/sqlutil/{postgres_wasm.go => unique_constraint.go} (62%)
 rename internal/sqlutil/{postgres.go => unique_constraint_wasm.go} (74%)

diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go
index 98e7d8938..b6a8b1f25 100644
--- a/internal/sqlutil/migrate.go
+++ b/internal/sqlutil/migrate.go
@@ -155,5 +155,10 @@ func InsertMigration(ctx context.Context, db *sql.DB, migrationName string) erro
 		time.Now().Format(time.RFC3339),
 		internal.VersionString(),
 	)
+	// If the migration was already executed, we'll get a unique constraint error,
+	// return nil instead, to avoid unnecessary logging.
+	if IsUniqueConstraintViolationErr(err) {
+		return nil
+	}
 	return err
 }
diff --git a/internal/sqlutil/postgres_wasm.go b/internal/sqlutil/unique_constraint.go
similarity index 62%
rename from internal/sqlutil/postgres_wasm.go
rename to internal/sqlutil/unique_constraint.go
index 34086f450..4a1b7fd94 100644
--- a/internal/sqlutil/postgres_wasm.go
+++ b/internal/sqlutil/unique_constraint.go
@@ -12,12 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build wasm
-// +build wasm
+//go:build !wasm
+// +build !wasm
 
 package sqlutil
 
-// IsUniqueConstraintViolationErr no-ops for this architecture
+import (
+	"github.com/lib/pq"
+	"github.com/mattn/go-sqlite3"
+)
+
+// IsUniqueConstraintViolationErr returns true if the error is an unique_violation error
 func IsUniqueConstraintViolationErr(err error) bool {
+	switch e := err.(type) {
+	case *pq.Error:
+		return e.Code == "23505"
+	case pq.Error:
+		return e.Code == "23505"
+	case *sqlite3.Error:
+		return e.Code == sqlite3.ErrConstraint
+	case sqlite3.Error:
+		return e.Code == sqlite3.ErrConstraint
+	}
 	return false
 }
diff --git a/internal/sqlutil/postgres.go b/internal/sqlutil/unique_constraint_wasm.go
similarity index 74%
rename from internal/sqlutil/postgres.go
rename to internal/sqlutil/unique_constraint_wasm.go
index 5e656b1da..02ceb5851 100644
--- a/internal/sqlutil/postgres.go
+++ b/internal/sqlutil/unique_constraint_wasm.go
@@ -12,15 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build !wasm
-// +build !wasm
+//go:build wasm
+// +build wasm
 
 package sqlutil
 
-import "github.com/lib/pq"
+import "github.com/mattn/go-sqlite3"
 
-// IsUniqueConstraintViolationErr returns true if the error is a postgresql unique_violation error
+// IsUniqueConstraintViolationErr returns true if the error is an unique_violation error
 func IsUniqueConstraintViolationErr(err error) bool {
-	pqErr, ok := err.(*pq.Error)
-	return ok && pqErr.Code == "23505"
+	switch e := err.(type) {
+	case *sqlite3.Error:
+		return e.Code == sqlite3.ErrConstraint
+	case sqlite3.Error:
+		return e.Code == sqlite3.ErrConstraint
+	}
+	return false
 }
diff --git a/keyserver/storage/postgres/key_changes_table.go b/keyserver/storage/postgres/key_changes_table.go
index e91b048d5..c0e3429c7 100644
--- a/keyserver/storage/postgres/key_changes_table.go
+++ b/keyserver/storage/postgres/key_changes_table.go
@@ -18,8 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
-
-	"github.com/sirupsen/logrus"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -81,8 +80,7 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
-				// not a fatal error, log and continue
-				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
+				return fmt.Errorf("unable to manually insert migration '%s': %w", migrationName, err)
 			}
 			return nil
 		}
diff --git a/keyserver/storage/sqlite3/key_changes_table.go b/keyserver/storage/sqlite3/key_changes_table.go
index b68df5552..0c844d67a 100644
--- a/keyserver/storage/sqlite3/key_changes_table.go
+++ b/keyserver/storage/sqlite3/key_changes_table.go
@@ -18,8 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
-
-	"github.com/sirupsen/logrus"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -80,8 +79,7 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
-				// not a fatal error, log and continue
-				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
+				return fmt.Errorf("unable to manually insert migration '%s': %w", migrationName, err)
 			}
 			return nil
 		}
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index 26178df83..23a5f79eb 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -21,8 +21,6 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/sirupsen/logrus"
-
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
 
@@ -79,8 +77,7 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
-				// not a fatal error, log and continue
-				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
+				return fmt.Errorf("unable to manually insert migration '%s': %w", migrationName, err)
 			}
 			return nil
 		}
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index c7bc00393..01c3f879c 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -22,7 +22,6 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -87,8 +86,7 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
-				// not a fatal error, log and continue
-				logrus.WithError(err).Warnf("unable to manually insert migration '%s'", migrationName)
+				return fmt.Errorf("unable to manually insert migration '%s': %w", migrationName, err)
 			}
 			return nil
 		}

From c366ccdfcaf1ea820bef4744fda90aff4db1c67d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 13 Sep 2022 09:35:45 +0200
Subject: [PATCH 357/391] Send-to-device consumer/producer tweaks (#2713)

Some tweaks for the send-to-device consumers/producers:
- use `json.RawMessage` without marshalling it first
- try further devices (if available) if we failed to `PublishMsg` in the
producers
- some logging changes (to better debug E2EE issues)
---
 clientapi/producers/syncapi.go     | 32 ++++++++++++++----------------
 federationapi/producers/syncapi.go | 24 ++++++++++------------
 syncapi/consumers/sendtodevice.go  | 15 ++++++++------
 syncapi/syncapi_test.go            |  4 +---
 4 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go
index 5933ce1a8..2dc0c4843 100644
--- a/clientapi/producers/syncapi.go
+++ b/clientapi/producers/syncapi.go
@@ -21,12 +21,13 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/syncapi/types"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // SyncAPIProducer produces events for the sync API server to consume
@@ -61,7 +62,7 @@ func (p *SyncAPIProducer) SendReceipt(
 
 func (p *SyncAPIProducer) SendToDevice(
 	ctx context.Context, sender, userID, deviceID, eventType string,
-	message interface{},
+	message json.RawMessage,
 ) error {
 	devices := []string{}
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
@@ -89,24 +90,19 @@ func (p *SyncAPIProducer) SendToDevice(
 		devices = append(devices, deviceID)
 	}
 
-	js, err := json.Marshal(message)
-	if err != nil {
-		return err
-	}
-
 	log.WithFields(log.Fields{
 		"user_id":     userID,
 		"num_devices": len(devices),
 		"type":        eventType,
 	}).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent)
-	for _, device := range devices {
+	for i, device := range devices {
 		ote := &types.OutputSendToDeviceEvent{
 			UserID:   userID,
 			DeviceID: device,
 			SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
 				Sender:  sender,
 				Type:    eventType,
-				Content: js,
+				Content: message,
 			},
 		}
 
@@ -115,15 +111,17 @@ func (p *SyncAPIProducer) SendToDevice(
 			log.WithError(err).Error("sendToDevice failed json.Marshal")
 			return err
 		}
-		m := &nats.Msg{
-			Subject: p.TopicSendToDeviceEvent,
-			Data:    eventJSON,
-			Header:  nats.Header{},
-		}
+		m := nats.NewMsg(p.TopicSendToDeviceEvent)
+		m.Data = eventJSON
 		m.Header.Set("sender", sender)
 		m.Header.Set(jetstream.UserID, userID)
+
 		if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil {
-			log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage")
+			if i < len(devices)-1 {
+				log.WithError(err).Warn("sendToDevice failed to PublishMsg, trying further devices")
+				continue
+			}
+			log.WithError(err).Error("sendToDevice failed to PublishMsg for all devices")
 			return err
 		}
 	}
diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go
index 86c8c10a3..4abd3fbe5 100644
--- a/federationapi/producers/syncapi.go
+++ b/federationapi/producers/syncapi.go
@@ -64,7 +64,7 @@ func (p *SyncAPIProducer) SendReceipt(
 
 func (p *SyncAPIProducer) SendToDevice(
 	ctx context.Context, sender, userID, deviceID, eventType string,
-	message interface{},
+	message json.RawMessage,
 ) error {
 	devices := []string{}
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
@@ -92,24 +92,19 @@ func (p *SyncAPIProducer) SendToDevice(
 		devices = append(devices, deviceID)
 	}
 
-	js, err := json.Marshal(message)
-	if err != nil {
-		return err
-	}
-
 	log.WithFields(log.Fields{
 		"user_id":     userID,
 		"num_devices": len(devices),
 		"type":        eventType,
 	}).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent)
-	for _, device := range devices {
+	for i, device := range devices {
 		ote := &types.OutputSendToDeviceEvent{
 			UserID:   userID,
 			DeviceID: device,
 			SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
 				Sender:  sender,
 				Type:    eventType,
-				Content: js,
+				Content: message,
 			},
 		}
 
@@ -118,16 +113,17 @@ func (p *SyncAPIProducer) SendToDevice(
 			log.WithError(err).Error("sendToDevice failed json.Marshal")
 			return err
 		}
-		m := &nats.Msg{
-			Subject: p.TopicSendToDeviceEvent,
-			Data:    eventJSON,
-			Header:  nats.Header{},
-		}
+		m := nats.NewMsg(p.TopicSendToDeviceEvent)
+		m.Data = eventJSON
 		m.Header.Set("sender", sender)
 		m.Header.Set(jetstream.UserID, userID)
 
 		if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil {
-			log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage")
+			if i < len(devices)-1 {
+				log.WithError(err).Warn("sendToDevice failed to PublishMsg, trying further devices")
+				continue
+			}
+			log.WithError(err).Error("sendToDevice failed to PublishMsg for all devices")
 			return err
 		}
 	}
diff --git a/syncapi/consumers/sendtodevice.go b/syncapi/consumers/sendtodevice.go
index 89b01d7e5..7d6aae597 100644
--- a/syncapi/consumers/sendtodevice.go
+++ b/syncapi/consumers/sendtodevice.go
@@ -19,16 +19,17 @@ import (
 	"encoding/json"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/nats-io/nats.go"
-	log "github.com/sirupsen/logrus"
 )
 
 // OutputSendToDeviceEventConsumer consumes events that originated in the EDU server.
@@ -79,16 +80,18 @@ func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msgs []
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		sentry.CaptureException(err)
+		log.WithError(err).Errorf("send-to-device: failed to split user id, dropping message")
 		return true
 	}
 	if domain != s.serverName {
+		log.Tracef("ignoring send-to-device event with destination %s", domain)
 		return true
 	}
 
 	var output types.OutputSendToDeviceEvent
 	if err = json.Unmarshal(msg.Data, &output); err != nil {
 		// If the message was invalid, log it and move on to the next message in the stream
-		log.WithError(err).Errorf("output log: message parse failure")
+		log.WithError(err).Errorf("send-to-device: message parse failure")
 		sentry.CaptureException(err)
 		return true
 	}
@@ -105,7 +108,7 @@ func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msgs []
 	)
 	if err != nil {
 		sentry.CaptureException(err)
-		log.WithError(err).Errorf("failed to store send-to-device message")
+		log.WithError(err).Errorf("send-to-device: failed to store message")
 		return false
 	}
 
diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go
index c81256aa7..a4985dbf4 100644
--- a/syncapi/syncapi_test.go
+++ b/syncapi/syncapi_test.go
@@ -624,9 +624,7 @@ func testSendToDevice(t *testing.T, dbType test.DBType) {
 		// Send to-device messages of type "m.dendrite.test" with content `{"dummy":"message $counter"}`
 		for i := 0; i < tc.sendMessagesCount; i++ {
 			msgCounter++
-			msg := map[string]string{
-				"dummy": fmt.Sprintf("message %d", msgCounter),
-			}
+			msg := json.RawMessage(fmt.Sprintf(`{"dummy":"message %d"}`, msgCounter))
 			if err := producer.SendToDevice(ctx, user.ID, user.ID, alice.ID, "m.dendrite.test", msg); err != nil {
 				t.Fatalf("unable to send to device message: %v", err)
 			}

From 3e55856254f6bc918f5075d7adb85d53da92c7c8 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 13 Sep 2022 09:37:38 +0100
Subject: [PATCH 358/391] Always resolve state in `QueryStateAfterEvents`

---
 roomserver/internal/query/query.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 6dce2bc3e..d08c5c491 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -106,7 +106,7 @@ func (r *Queryer) QueryStateAfterEvents(
 		return err
 	}
 
-	if len(request.PrevEventIDs) > 1 && len(request.StateToFetch) == 0 {
+	if len(request.PrevEventIDs) > 1 {
 		var authEventIDs []string
 		for _, e := range stateEvents {
 			authEventIDs = append(authEventIDs, e.AuthEventIDs()...)

From b05e028f7d8befc045dcb62e87d09ad462bb277c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 13 Sep 2022 12:52:09 +0100
Subject: [PATCH 359/391] Tweak `LoadMembershipAtEvent` behaviour when state
 not known (#2716)

Previously `LoadMembershipAtEvent` would fail if the state before one of
the events was not known, i.e. because it was an outlier. This modifies
it so that it gracefully handles not knowing the state and returns no
memberships instead, so that history visibility doesn't freak out and
kill `/sync` requests dead.
---
 roomserver/api/query.go            | 3 ++-
 roomserver/internal/query/query.go | 9 ++++++++-
 roomserver/state/state.go          | 8 ++++++--
 syncapi/streams/stream_pdu.go      | 5 ++---
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index 32d63bb51..aa7dc4735 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -439,6 +439,7 @@ type QueryMembershipAtEventRequest struct {
 
 // QueryMembershipAtEventResponse is the response to QueryMembershipAtEventRequest.
 type QueryMembershipAtEventResponse struct {
-	// Memberships is a map from eventID to a list of events (if any).
+	// Memberships is a map from eventID to a list of events (if any). Events that
+	// do not have known state will return an empty array here.
 	Memberships map[string][]*gomatrixserverlib.HeaderedEvent `json:"memberships"`
 }
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index d08c5c491..b41a92e94 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -208,6 +208,9 @@ func (r *Queryer) QueryMembershipForUser(
 	return err
 }
 
+// QueryMembershipAtEvent returns the known memberships at a given event.
+// If the state before an event is not known, an empty list will be returned
+// for that event instead.
 func (r *Queryer) QueryMembershipAtEvent(
 	ctx context.Context,
 	request *api.QueryMembershipAtEventRequest,
@@ -237,7 +240,11 @@ func (r *Queryer) QueryMembershipAtEvent(
 	}
 
 	for _, eventID := range request.EventIDs {
-		stateEntry := stateEntries[eventID]
+		stateEntry, ok := stateEntries[eventID]
+		if !ok {
+			response.Memberships[eventID] = []*gomatrixserverlib.HeaderedEvent{}
+			continue
+		}
 		memberships, err := helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false)
 		if err != nil {
 			return fmt.Errorf("unable to get memberships at state: %w", err)
diff --git a/roomserver/state/state.go b/roomserver/state/state.go
index a40a2e9ba..cb96d83ec 100644
--- a/roomserver/state/state.go
+++ b/roomserver/state/state.go
@@ -18,6 +18,7 @@ package state
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"sort"
 	"sync"
@@ -134,11 +135,14 @@ func (v *StateResolution) LoadMembershipAtEvent(
 	for i := range eventIDs {
 		eventID := eventIDs[i]
 		snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID)
-		if err != nil {
+		if err != nil && err != sql.ErrNoRows {
 			return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %w", eventID, err)
 		}
 		if snapshotNID == 0 {
-			return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID)
+			// If we don't know a state snapshot for this event then we can't calculate
+			// memberships at the time of the event, so skip over it. This means that
+			// it isn't guaranteed that the response map will contain every single event.
+			continue
 		}
 		snapshotNIDMap[snapshotNID] = append(snapshotNIDMap[snapshotNID], eventID)
 	}
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index ffcf64df6..0ab6de886 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -388,7 +388,7 @@ func applyHistoryVisibilityFilter(
 	if err != nil {
 		// Not a fatal error, we can continue without the stateEvents,
 		// they are only needed if there are state events in the timeline.
-		logrus.WithError(err).Warnf("failed to get current room state")
+		logrus.WithError(err).Warnf("Failed to get current room state for history visibility")
 	}
 	alwaysIncludeIDs := make(map[string]struct{}, len(stateEvents))
 	for _, ev := range stateEvents {
@@ -397,7 +397,6 @@ func applyHistoryVisibilityFilter(
 	startTime := time.Now()
 	events, err := internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync")
 	if err != nil {
-
 		return nil, err
 	}
 	logrus.WithFields(logrus.Fields{
@@ -405,7 +404,7 @@ func applyHistoryVisibilityFilter(
 		"room_id":  roomID,
 		"before":   len(recentEvents),
 		"after":    len(events),
-	}).Debug("applied history visibility (sync)")
+	}).Trace("Applied history visibility (sync)")
 	return events, nil
 }
 

From 482914aef4a7d637a8c468d46904fde9f478b5d1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 13 Sep 2022 15:25:02 +0100
Subject: [PATCH 360/391] Use `AckNone` on the ephemeral room input consumer

---
 roomserver/internal/input/input.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index a8a3e0248..1cb980dc3 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -172,11 +172,10 @@ func (r *Inputer) Start() error {
 		func(m *nats.Msg) {
 			roomID := m.Header.Get(jetstream.RoomID)
 			r.startWorkerForRoom(roomID)
-			_ = m.Ack()
 		},
 		nats.HeadersOnly(),
 		nats.DeliverAll(),
-		nats.AckAll(),
+		nats.AckNone(),
 		nats.BindStream(r.InputRoomEventTopic),
 	)
 	return err

From 7f89fed1e45cc6ea65c420e32a017df634f4164f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 14 Sep 2022 09:55:50 +0100
Subject: [PATCH 361/391] Revert 482914aef4a7d637a8c468d46904fde9f478b5d1

---
 roomserver/internal/input/input.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 1cb980dc3..c47793f0a 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -172,10 +172,12 @@ func (r *Inputer) Start() error {
 		func(m *nats.Msg) {
 			roomID := m.Header.Get(jetstream.RoomID)
 			r.startWorkerForRoom(roomID)
+			_ = m.Ack()
 		},
 		nats.HeadersOnly(),
 		nats.DeliverAll(),
-		nats.AckNone(),
+		nats.AckExplicit(),
+		nats.ReplayInstant(),
 		nats.BindStream(r.InputRoomEventTopic),
 	)
 	return err

From e6960d0b15b6cb216f1d566a7a47a891348fd48f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 14 Sep 2022 14:25:25 +0100
Subject: [PATCH 362/391] Update to
 matrix-org/pinecone@608215eb1b2920f3510b56c4a36a87ed9e75779f

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index d151b2995..263a0065d 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a
-	github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534
+	github.com/matrix-org/pinecone v0.0.0-20220914131522-608215eb1b29
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
 	github.com/nats-io/nats-server/v2 v2.9.0
diff --git a/go.sum b/go.sum
index 3cad3f530..abdc11665 100644
--- a/go.sum
+++ b/go.sum
@@ -390,8 +390,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a h1:XeGBDZZsUe4kgj3myl0EiuDNVWxszJecMTrON3Wn9sI=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
-github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534 h1:XuJYAJNkdG3zj9cO0yQSvL+Sp2xogsTOuZRx7PwdtoA=
-github.com/matrix-org/pinecone v0.0.0-20220912093434-b215925d5534/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
+github.com/matrix-org/pinecone v0.0.0-20220914131522-608215eb1b29 h1:/AIaqhK1BBi2sMEVQdgZRV8H8sNloAGCgztLZhsPqD0=
+github.com/matrix-org/pinecone v0.0.0-20220914131522-608215eb1b29/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 0ea948c70548875601a544c57a588271af3adce4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 14 Sep 2022 14:26:24 +0100
Subject: [PATCH 363/391] Fix Pinecone demo build errors after Pinecone update

---
 build/gobind-pinecone/monolith.go  | 2 +-
 cmd/dendrite-demo-pinecone/main.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 91bbb6873..500403ae4 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -255,7 +255,7 @@ func (m *DendriteMonolith) Start() {
 	m.logger.SetOutput(BindLogger{})
 	logrus.SetOutput(BindLogger{})
 
-	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
+	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
 	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
 	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
 	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index ff9cb5aa1..99e597e59 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -151,7 +151,7 @@ func main() {
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck
 
-	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
+	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
 	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
 	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
 	pManager := pineconeConnections.NewConnectionManager(pRouter, nil)

From a5f8c07184921b4cfdea42c872852ca5e5e5a4ce Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 15 Sep 2022 07:26:26 +0200
Subject: [PATCH 364/391] Hopefully fix `upgrade-tests` (#2717)

Wait for events to come down `/sync` before ending the test.
---
 cmd/dendrite-upgrade-tests/tests.go | 45 +++++++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/cmd/dendrite-upgrade-tests/tests.go b/cmd/dendrite-upgrade-tests/tests.go
index ff1e09dda..5c9589df2 100644
--- a/cmd/dendrite-upgrade-tests/tests.go
+++ b/cmd/dendrite-upgrade-tests/tests.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"strings"
+	"time"
 
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -81,11 +82,14 @@ func runTests(baseURL, branchName string) error {
 			client: users[1].client, text: "4: " + branchName,
 		},
 	}
+	wantEventIDs := make(map[string]struct{}, 8)
 	for _, msg := range msgs {
-		_, err = msg.client.SendText(dmRoomID, msg.text)
+		var resp *gomatrix.RespSendEvent
+		resp, err = msg.client.SendText(dmRoomID, msg.text)
 		if err != nil {
 			return fmt.Errorf("failed to send text in dm room: %s", err)
 		}
+		wantEventIDs[resp.EventID] = struct{}{}
 	}
 
 	// attempt to create/join the shared public room
@@ -113,11 +117,48 @@ func runTests(baseURL, branchName string) error {
 	}
 	// send messages
 	for _, msg := range msgs {
-		_, err = msg.client.SendText(publicRoomID, "public "+msg.text)
+		resp, err := msg.client.SendText(publicRoomID, "public "+msg.text)
 		if err != nil {
 			return fmt.Errorf("failed to send text in public room: %s", err)
 		}
+		wantEventIDs[resp.EventID] = struct{}{}
 	}
+
+	// Sync until we have all expected messages
+	doneCh := make(chan struct{})
+	go func() {
+		syncClient := users[0].client
+		since := ""
+		for len(wantEventIDs) > 0 {
+			select {
+			case <-doneCh:
+				return
+			default:
+			}
+			syncResp, err := syncClient.SyncRequest(1000, since, "1", false, "")
+			if err != nil {
+				continue
+			}
+			for _, room := range syncResp.Rooms.Join {
+				for _, ev := range room.Timeline.Events {
+					if ev.Type != "m.room.message" {
+						continue
+					}
+					delete(wantEventIDs, ev.ID)
+				}
+			}
+			since = syncResp.NextBatch
+		}
+		close(doneCh)
+	}()
+
+	select {
+	case <-time.After(time.Second * 10):
+		close(doneCh)
+		return fmt.Errorf("failed to receive all expected messages: %+v", wantEventIDs)
+	case <-doneCh:
+	}
+
 	log.Printf("OK! rooms(public=%s, dm=%s) users(%s, %s)\n", publicRoomID, dmRoomID, users[0].userID, users[1].userID)
 	return nil
 }

From fc1d8e479b282b56dd0eb707559785dc20fc65a6 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:35:32 +0100
Subject: [PATCH 365/391] Ensure that all state event IDs are included in the
 `added` section when rewriting state (#2725)

This should hopefully fix an entire class of problems where components
downstream from the roomserver (i.e. the sync API) could just lose a
whole bunch of state after a rewrite operation like a federated join.

The root of the bug is that we set `RewritesState` in the output event
which instructs downstream components to purge their copy of any room
state, but then didn't send the entire state snapshot in
`adds_state_event_ids` so the downstream state ends up being incomplete
as a result.
---
 .../internal/input/input_latest_events.go     | 31 +++++++++++++------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 205a33e83..a223820ef 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -264,16 +264,27 @@ func (u *latestEventsUpdater) latestState() error {
 		return fmt.Errorf("roomState.CalculateAndStoreStateAfterEvents: %w", err)
 	}
 
-	// Now that we have a new state snapshot based on the latest events,
-	// we can compare that new snapshot to the previous one and see what
-	// has changed. This gives us one list of removed state events and
-	// another list of added ones. Replacing a value for a state-key tuple
-	// will result one removed (the old event) and one added (the new event).
-	u.removed, u.added, err = roomState.DifferenceBetweeenStateSnapshots(
-		ctx, u.oldStateNID, u.newStateNID,
-	)
-	if err != nil {
-		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
+	// Include information about what changed in the state transition. If the
+	// event rewrites the state (i.e. is a federated join) then we will simply
+	// include the entire state snapshot as added events, as the "RewritesState"
+	// flag in the output event signals downstream components to purge their
+	// room state first. If it doesn't rewrite the state then we will work out
+	// what the difference is between the state snapshots and send that. In all
+	// cases where a state event is being replaced, the old state event will
+	// appear in "removed" and the replacement will appear in "added".
+	if u.rewritesState {
+		u.removed = []types.StateEntry{}
+		u.added, err = roomState.LoadStateAtSnapshot(ctx, u.newStateNID)
+		if err != nil {
+			return fmt.Errorf("roomState.LoadStateAtSnapshot: %w", err)
+		}
+	} else {
+		u.removed, u.added, err = roomState.DifferenceBetweeenStateSnapshots(
+			ctx, u.oldStateNID, u.newStateNID,
+		)
+		if err != nil {
+			return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
+		}
 	}
 
 	if removed := len(u.removed) - len(u.added); !u.rewritesState && removed > 0 {

From 7bfc3074d10b2cd91b37290e39d3882119853107 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 16 Sep 2022 13:30:20 +0200
Subject: [PATCH 366/391] Fix origin on device list update EDUs

---
 federationapi/producers/syncapi.go | 4 ++--
 federationapi/routing/send.go      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go
index 4abd3fbe5..659ff1bcf 100644
--- a/federationapi/producers/syncapi.go
+++ b/federationapi/producers/syncapi.go
@@ -163,10 +163,10 @@ func (p *SyncAPIProducer) SendPresence(
 }
 
 func (p *SyncAPIProducer) SendDeviceListUpdate(
-	ctx context.Context, deviceListUpdate gomatrixserverlib.RawJSON, origin string,
+	ctx context.Context, deviceListUpdate gomatrixserverlib.RawJSON, origin gomatrixserverlib.ServerName,
 ) (err error) {
 	m := nats.NewMsg(p.TopicDeviceListUpdate)
-	m.Header.Set("origin", origin)
+	m.Header.Set("origin", string(origin))
 	m.Data = deviceListUpdate
 	log.Debugf("Sending device list update: %+v", m.Header)
 	_, err = p.JetStream.PublishMsg(m, nats.Context(ctx))
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index a9714c65a..060af676d 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -359,7 +359,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				}
 			}
 		case gomatrixserverlib.MDeviceListUpdate:
-			if err := t.producer.SendDeviceListUpdate(ctx, e.Content, e.Origin); err != nil {
+			if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil {
 				util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate")
 			}
 		case gomatrixserverlib.MReceipt:

From 99f6b6a95234f59a16806b9dc6b16cb41f038504 Mon Sep 17 00:00:00 2001
From: Tak Wai Wong <64229756+tak-hntlabs@users.noreply.github.com>
Date: Mon, 19 Sep 2022 09:39:06 -0700
Subject: [PATCH 367/391] Bug fix #2718 appservice txnid should be different
 for each batch of events (#2719)

See issue: [#2718](https://github.com/matrix-org/dendrite/issues/2718)
for more details.

The fix assumes that if the number of transaction items are different,
then the txnid should be different.

txnid := OriginalServerTS()_len(transactions)

The case that it doesn't address is if the txnid generated this way is
the same for 2 different batches of events which have the same
OriginalServerTS and the same array length.

Another option:

txnid := OriginalServerTS()_hash(transactions)

Would love to hear other ideas and ways to fix this.

### Pull Request Checklist

* [x ] I have added added tests for PR _or_ I have justified why this PR
doesn't need tests.
* [x ] Pull request includes a [sign
off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)

Signed-off-by: `Tak Wai Wong <tak@hntlabs.com>`

Co-authored-by: Tak Wai Wong <tak@hntlabs.com>
---
 appservice/consumers/roomserver.go | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index a4bcfa7d9..d44f32b38 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -22,6 +22,7 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -151,10 +152,17 @@ func (s *OutputRoomEventConsumer) onMessage(
 		return true
 	}
 
+	txnID := ""
+	// Try to get the message metadata, if we're able to, use the timestamp as the txnID
+	metadata, err := msgs[0].Metadata()
+	if err == nil {
+		txnID = strconv.Itoa(int(metadata.Timestamp.UnixNano()))
+	}
+
 	// Send event to any relevant application services. If we hit
 	// an error here, return false, so that we negatively ack.
 	log.WithField("appservice", state.ID).Debugf("Appservice worker sending %d events(s) from roomserver", len(events))
-	return s.sendEvents(ctx, state, events) == nil
+	return s.sendEvents(ctx, state, events, txnID) == nil
 }
 
 // sendEvents passes events to the appservice by using the transactions
@@ -162,6 +170,7 @@ func (s *OutputRoomEventConsumer) onMessage(
 func (s *OutputRoomEventConsumer) sendEvents(
 	ctx context.Context, state *appserviceState,
 	events []*gomatrixserverlib.HeaderedEvent,
+	txnID string,
 ) error {
 	// Create the transaction body.
 	transaction, err := json.Marshal(
@@ -173,13 +182,14 @@ func (s *OutputRoomEventConsumer) sendEvents(
 		return err
 	}
 
-	// TODO: We should probably be more intelligent and pick something not
-	// in the control of the event. A NATS timestamp header or something maybe.
-	txnID := events[0].Event.OriginServerTS()
+	// If txnID is not defined, generate one from the events.
+	if txnID == "" {
+		txnID = fmt.Sprintf("%d_%d", events[0].Event.OriginServerTS(), len(transaction))
+	}
 
 	// Send the transaction to the appservice.
 	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	address := fmt.Sprintf("%s/transactions/%d?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken))
+	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken))
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err

From cf01d29277accf42ac31358ff1407abc7bf63fea Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Sep 2022 09:47:57 +0100
Subject: [PATCH 368/391] Update contributing documentation

---
 docs/CONTRIBUTING.md | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 771af9ecf..6ba05f46f 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -11,8 +11,8 @@ possible to get started.
 
 ## Sign off
 
-We ask that everyone who contributes to the project signs off their contributions
-in accordance with the [DCO](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off).
+We require that everyone who contributes to the project signs off their contributions
+in accordance with the [Developer Certificate of Origin](https://github.com/matrix-org/matrix-spec/blob/main/CONTRIBUTING.rst#sign-off).
 In effect, this means adding a statement to your pull requests or commit messages
 along the lines of:
 
@@ -20,7 +20,18 @@ along the lines of:
 Signed-off-by: Full Name <email address>
 ```
 
-Unfortunately we can't accept contributions without it.
+Unfortunately we can't accept contributions without a sign-off.
+
+Please note that we can only accept contributions under a legally identifiable name,
+such as your name as it appears on government-issued documentation or common-law names
+(claimed by legitimate usage or repute). We cannot accept sign-offs from a pseudonym or
+alias and cannot accept anonymous contributions.
+
+If you would prefer to sign off privately instead (so as to not reveal your full
+name on a public pull request), you can do so by emailing a sign-off declaration
+and a link to your pull request directly to the [Matrix.org Foundation](https://matrix.org/foundation/)
+at `dco@matrix.org`. Once a private sign-off has been made, you will not be required
+to do so for future contributions.
 
 ## Getting up and running
 

From 47af4bff5b930c86630c7879e14f5b07b9d08497 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Sep 2022 09:52:22 +0100
Subject: [PATCH 369/391] Update database documentation

---
 docs/installation/4_database.md | 40 ++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/docs/installation/4_database.md b/docs/installation/4_database.md
index f6222a8d2..68f2d44d0 100644
--- a/docs/installation/4_database.md
+++ b/docs/installation/4_database.md
@@ -10,24 +10,6 @@ permalink: /installation/database
 Dendrite uses SQL databases to store data. Depending on the database engine being used, you
 may need to perform some manual steps outlined below.
 
-## SQLite
-
-SQLite deployments do not require manual database creation. Simply configure the database
-filenames in the Dendrite configuration file and start Dendrite. The databases will be created
-and populated automatically.
-
-Note that Dendrite **cannot share a single SQLite database across multiple components**. Each
-component must be configured with its own SQLite database filename. You will have to remove
-the `global.database` section from your Dendrite config and add it to each individual section
-instead in order to use SQLite.
-
-### Connection strings
-
-Connection strings for SQLite databases take the following forms:
-
-* Current working directory path: `file:dendrite_component.db`
-* Full specified path: `file:///path/to/dendrite_component.db`
-
 ## PostgreSQL
 
 Dendrite can automatically populate the database with the relevant tables and indexes, but
@@ -106,3 +88,25 @@ for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver use
     sudo -u postgres createdb -O dendrite dendrite_$i
 done
 ```
+
+## SQLite
+
+**WARNING:** The Dendrite SQLite backend is slower, less reliable and not recommended for
+production usage. You should use PostgreSQL instead. We may not be able to provide support if
+you run into issues with your deployment while using the SQLite backend.
+
+SQLite deployments do not require manual database creation. Simply configure the database
+filenames in the Dendrite configuration file and start Dendrite. The databases will be created
+and populated automatically.
+
+Note that Dendrite **cannot share a single SQLite database across multiple components**. Each
+component must be configured with its own SQLite database filename. You will have to remove
+the `global.database` section from your Dendrite config and add it to each individual section
+instead in order to use SQLite.
+
+### Connection strings
+
+Connection strings for SQLite databases take the following forms:
+
+* Current working directory path: `file:dendrite_component.db`
+* Full specified path: `file:///path/to/dendrite_component.db`

From e007b8038f677b6578a65cfe2cbd8a7192ae69b0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 20 Sep 2022 11:32:03 +0200
Subject: [PATCH 370/391] Mark device list as stale, if we don't have the
 requesting device (#2728)

This hopefully makes E2EE chats a little bit more reliable by re-syncing
devices if we don't have the `requesting_device_id` in our database. (As
seen in
[Synapse](https://github.com/matrix-org/synapse/blob/c52abc1cfdd9e5480cdb4a03d626fe61cacc6573/synapse/handlers/devicemessage.py#L157-L201))
---
 keyserver/api/api.go               | 10 +++++++++-
 keyserver/internal/internal.go     | 24 +++++++++++++++++++-----
 keyserver/inthttp/client.go        | 12 ++++++++++++
 keyserver/inthttp/server.go        |  6 ++++++
 syncapi/consumers/sendtodevice.go  | 25 +++++++++++++++++++++++--
 syncapi/internal/keychange_test.go |  4 ++++
 syncapi/syncapi.go                 |  5 +++--
 7 files changed, 76 insertions(+), 10 deletions(-)

diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index 9ba3988b9..c9ec59a75 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -21,9 +21,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/keyserver/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type KeyInternalAPI interface {
@@ -56,6 +57,7 @@ type UserKeyAPI interface {
 type SyncKeyAPI interface {
 	QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse) error
 	QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse) error
+	PerformMarkAsStaleIfNeeded(ctx context.Context, req *PerformMarkAsStaleRequest, res *struct{}) error
 }
 
 type FederationKeyAPI interface {
@@ -335,3 +337,9 @@ type QuerySignaturesResponse struct {
 	// The request error, if any
 	Error *KeyError
 }
+
+type PerformMarkAsStaleRequest struct {
+	UserID   string
+	Domain   gomatrixserverlib.ServerName
+	DeviceID string
+}
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 41b4d44a4..a8d1128c4 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -23,16 +23,17 @@ import (
 	"sync"
 	"time"
 
-	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/keyserver/api"
-	"github.com/matrix-org/dendrite/keyserver/producers"
-	"github.com/matrix-org/dendrite/keyserver/storage"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
+
+	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/keyserver/producers"
+	"github.com/matrix-org/dendrite/keyserver/storage"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 type KeyInternalAPI struct {
@@ -224,6 +225,19 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query
 	return nil
 }
 
+// PerformMarkAsStaleIfNeeded marks the users device list as stale, if the given deviceID is not present
+// in our database.
+func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *api.PerformMarkAsStaleRequest, res *struct{}) error {
+	knownDevices, err := a.DB.DeviceKeysForUser(ctx, req.UserID, []string{req.DeviceID}, true)
+	if err != nil {
+		return err
+	}
+	if len(knownDevices) == 0 {
+		return a.Updater.ManualUpdate(ctx, req.Domain, req.UserID)
+	}
+	return nil
+}
+
 // nolint:gocyclo
 func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) error {
 	res.DeviceKeys = make(map[string]map[string]json.RawMessage)
diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go
index 7a7131145..75d537d9c 100644
--- a/keyserver/inthttp/client.go
+++ b/keyserver/inthttp/client.go
@@ -37,6 +37,7 @@ const (
 	QueryOneTimeKeysPath              = "/keyserver/queryOneTimeKeys"
 	QueryDeviceMessagesPath           = "/keyserver/queryDeviceMessages"
 	QuerySignaturesPath               = "/keyserver/querySignatures"
+	PerformMarkAsStalePath            = "/keyserver/markAsStale"
 )
 
 // NewKeyServerClient creates a KeyInternalAPI implemented by talking to a HTTP POST API.
@@ -172,3 +173,14 @@ func (h *httpKeyInternalAPI) QuerySignatures(
 		h.httpClient, ctx, request, response,
 	)
 }
+
+func (h *httpKeyInternalAPI) PerformMarkAsStaleIfNeeded(
+	ctx context.Context,
+	request *api.PerformMarkAsStaleRequest,
+	response *struct{},
+) error {
+	return httputil.CallInternalRPCAPI(
+		"MarkAsStale", h.apiURL+PerformMarkAsStalePath,
+		h.httpClient, ctx, request, response,
+	)
+}
diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go
index 4e5f9fba4..7af0ff6e5 100644
--- a/keyserver/inthttp/server.go
+++ b/keyserver/inthttp/server.go
@@ -16,6 +16,7 @@ package inthttp
 
 import (
 	"github.com/gorilla/mux"
+
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver/api"
 )
@@ -70,4 +71,9 @@ func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) {
 		QuerySignaturesPath,
 		httputil.MakeInternalRPCAPI("KeyserverQuerySignatures", s.QuerySignatures),
 	)
+
+	internalAPIMux.Handle(
+		PerformMarkAsStalePath,
+		httputil.MakeInternalRPCAPI("KeyserverMarkAsStale", s.PerformMarkAsStaleIfNeeded),
+	)
 }
diff --git a/syncapi/consumers/sendtodevice.go b/syncapi/consumers/sendtodevice.go
index 7d6aae597..c0b432256 100644
--- a/syncapi/consumers/sendtodevice.go
+++ b/syncapi/consumers/sendtodevice.go
@@ -23,7 +23,9 @@ import (
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
 
+	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -39,6 +41,7 @@ type OutputSendToDeviceEventConsumer struct {
 	durable    string
 	topic      string
 	db         storage.Database
+	keyAPI     keyapi.SyncKeyAPI
 	serverName gomatrixserverlib.ServerName // our server name
 	stream     types.StreamProvider
 	notifier   *notifier.Notifier
@@ -51,6 +54,7 @@ func NewOutputSendToDeviceEventConsumer(
 	cfg *config.SyncAPI,
 	js nats.JetStreamContext,
 	store storage.Database,
+	keyAPI keyapi.SyncKeyAPI,
 	notifier *notifier.Notifier,
 	stream types.StreamProvider,
 ) *OutputSendToDeviceEventConsumer {
@@ -60,6 +64,7 @@ func NewOutputSendToDeviceEventConsumer(
 		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
 		durable:    cfg.Matrix.JetStream.Durable("SyncAPISendToDeviceConsumer"),
 		db:         store,
+		keyAPI:     keyAPI,
 		serverName: cfg.Matrix.ServerName,
 		notifier:   notifier,
 		stream:     stream,
@@ -96,12 +101,28 @@ func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msgs []
 		return true
 	}
 
-	util.GetLogger(context.TODO()).WithFields(log.Fields{
+	logger := util.GetLogger(context.TODO()).WithFields(log.Fields{
 		"sender":     output.Sender,
 		"user_id":    output.UserID,
 		"device_id":  output.DeviceID,
 		"event_type": output.Type,
-	}).Debugf("sync API received send-to-device event from the clientapi/federationsender")
+	})
+	logger.Debugf("sync API received send-to-device event from the clientapi/federationsender")
+
+	// Check we actually got the requesting device in our store, if we receive a room key request
+	if output.Type == "m.room_key_request" {
+		requestingDeviceID := gjson.GetBytes(output.SendToDeviceEvent.Content, "requesting_device_id").Str
+		_, senderDomain, _ := gomatrixserverlib.SplitID('@', output.Sender)
+		if requestingDeviceID != "" && senderDomain != s.serverName {
+			// Mark the requesting device as stale, if we don't know about it.
+			if err = s.keyAPI.PerformMarkAsStaleIfNeeded(ctx, &keyapi.PerformMarkAsStaleRequest{
+				UserID: output.Sender, Domain: senderDomain, DeviceID: requestingDeviceID,
+			}, &struct{}{}); err != nil {
+				logger.WithError(err).Errorf("failed to mark as stale if needed")
+				return false
+			}
+		}
+	}
 
 	streamPos, err := s.db.StoreNewSendForDeviceMessage(
 		s.ctx, output.UserID, output.DeviceID, output.SendToDeviceEvent,
diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 80d2811be..3b9c8221c 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -22,6 +22,10 @@ var (
 
 type mockKeyAPI struct{}
 
+func (k *mockKeyAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *keyapi.PerformMarkAsStaleRequest, res *struct{}) error {
+	return nil
+}
+
 func (k *mockKeyAPI) PerformUploadKeys(ctx context.Context, req *keyapi.PerformUploadKeysRequest, res *keyapi.PerformUploadKeysResponse) error {
 	return nil
 }
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index 92db18d56..68537bc45 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -17,9 +17,10 @@ package syncapi
 import (
 	"context"
 
-	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/sirupsen/logrus"
 
+	"github.com/matrix-org/dendrite/internal/caching"
+
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -126,7 +127,7 @@ func AddPublicRoutes(
 	}
 
 	sendToDeviceConsumer := consumers.NewOutputSendToDeviceEventConsumer(
-		base.ProcessContext, cfg, js, syncDB, notifier, streams.SendToDeviceStreamProvider,
+		base.ProcessContext, cfg, js, syncDB, keyAPI, notifier, streams.SendToDeviceStreamProvider,
 	)
 	if err = sendToDeviceConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start send-to-device consumer")

From 97d7cf2232e205d53770c1f5e0828b5d2d03eb9f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Sep 2022 11:25:18 +0100
Subject: [PATCH 371/391] Remove deleted state logging lines from sync API
 (they are pointless)

---
 syncapi/storage/postgres/output_room_events_table.go | 11 -----------
 syncapi/storage/sqlite3/output_room_events_table.go  | 12 ------------
 2 files changed, 23 deletions(-)

diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 9c758cac1..041f99061 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -30,7 +30,6 @@ import (
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
-	log "github.com/sirupsen/logrus"
 )
 
 const outputRoomEventsSchema = `
@@ -273,16 +272,6 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs, &historyVisibility); err != nil {
 			return nil, nil, err
 		}
-		// Sanity check for deleted state and whine if we see it. We don't need to do anything
-		// since it'll just mark the event as not being needed.
-		if len(addIDs) < len(delIDs) {
-			log.WithFields(log.Fields{
-				"since":   r.From,
-				"current": r.To,
-				"adds":    len(addIDs),
-				"dels":    len(delIDs),
-			}).Warn("StateBetween: ignoring deleted state")
-		}
 
 		// TODO: Handle redacted events
 		var ev gomatrixserverlib.HeaderedEvent
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index cd9a46d4c..1626e32ef 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -31,7 +31,6 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
-	log "github.com/sirupsen/logrus"
 )
 
 const outputRoomEventsSchema = `
@@ -228,17 +227,6 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 			return nil, nil, err
 		}
 
-		// Sanity check for deleted state and whine if we see it. We don't need to do anything
-		// since it'll just mark the event as not being needed.
-		if len(addIDs) < len(delIDs) {
-			log.WithFields(log.Fields{
-				"since":   r.From,
-				"current": r.To,
-				"adds":    len(addIDsJSON),
-				"dels":    len(delIDsJSON),
-			}).Warn("StateBetween: ignoring deleted state")
-		}
-
 		// TODO: Handle redacted events
 		var ev gomatrixserverlib.HeaderedEvent
 		if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil {

From 201ac0594336046679cf3612405e85e802a57e12 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Sep 2022 14:10:30 +0100
Subject: [PATCH 372/391] Update readme

---
 README.md | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 34a08be94..3bb9a2350 100644
--- a/README.md
+++ b/README.md
@@ -11,21 +11,20 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
   a [brand new Go test suite](https://github.com/matrix-org/complement).
 - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments.
 
-As of October 2020 (current [progress below](#progress)), Dendrite has now entered **beta** which means:
+Dendrite is **beta** software, which means:
 
 - Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
-- Dendrite has periodic semver releases. We intend to release new versions as we land significant features.
+- Dendrite has periodic releases. We intend to release new versions as we fix bugs and land significant features.
 - Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite.
-- Breaking changes will not occur on minor releases. This means you can safely upgrade Dendrite without modifying your database or config file.
 
 This does not mean:
 
 - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
 - Dendrite is feature-complete. There may be client or federation APIs that are not implemented.
-- Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
+- Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
-In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode.
+In the future, we will be able to scale up to gigantic servers (equivalent to `matrix.org`) via polylith mode.
 
 If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in:
 

From bd39748b5c8b1890950db100382d3b128d333222 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Sep 2022 15:01:19 +0100
Subject: [PATCH 373/391] Update dependencies (#2729)

This updates Dendrite dependencies.
---
 federationapi/queue/destinationqueue.go |   4 +-
 federationapi/statistics/statistics.go  |   2 +-
 go.mod                                  |  82 ++++++-----
 go.sum                                  | 176 ++++++++++++------------
 internal/sqlutil/unique_constraint.go   |   2 -
 internal/sqlutil/writer_exclusive.go    |   2 +-
 setup/base/base.go                      |   4 +-
 setup/process/process.go                |   2 +-
 8 files changed, 131 insertions(+), 143 deletions(-)

diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go
index 0d937ffaf..5cb8cae1f 100644
--- a/federationapi/queue/destinationqueue.go
+++ b/federationapi/queue/destinationqueue.go
@@ -162,7 +162,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 // requests to retry.
 func (oq *destinationQueue) wakeQueueIfNeeded() {
 	// If we are backing off then interrupt the backoff.
-	if oq.backingOff.CAS(true, false) {
+	if oq.backingOff.CompareAndSwap(true, false) {
 		oq.interruptBackoff <- true
 	}
 	// If we aren't running then wake up the queue.
@@ -242,7 +242,7 @@ func (oq *destinationQueue) getPendingFromDatabase() {
 func (oq *destinationQueue) backgroundSend() {
 	// Check if a worker is already running, and if it isn't, then
 	// mark it as started.
-	if !oq.running.CAS(false, true) {
+	if !oq.running.CompareAndSwap(false, true) {
 		return
 	}
 	destinationQueueRunning.Inc()
diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go
index b8e16a259..db6d5c735 100644
--- a/federationapi/statistics/statistics.go
+++ b/federationapi/statistics/statistics.go
@@ -114,7 +114,7 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 	// a new backoff period. Increase the failure counter and
 	// start a goroutine which will wait out the backoff and
 	// unset the backoffStarted flag when done.
-	if s.backoffStarted.CAS(false, true) {
+	if s.backoffStarted.CompareAndSwap(false, true) {
 		if s.backoffCount.Inc() >= s.statistics.FailuresUntilBlacklist {
 			s.blacklisted.Store(true)
 			if s.statistics.DB != nil {
diff --git a/go.mod b/go.mod
index 263a0065d..bd401ca65 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/matrix-org/dendrite
 
 require (
-	github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a
+	github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/MFAshby/stdemuxerhook v1.0.0
@@ -9,45 +9,45 @@ require (
 	github.com/blevesearch/bleve/v2 v2.3.4
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d
-	github.com/docker/docker v20.10.16+incompatible
+	github.com/docker/docker v20.10.18+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.13.0
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.8
+	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/kardianos/minwinsvc v1.0.0
-	github.com/lib/pq v1.10.5
+	github.com/lib/pq v1.10.7
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a
-	github.com/matrix-org/pinecone v0.0.0-20220914131522-608215eb1b29
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef
+	github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
-	github.com/mattn/go-sqlite3 v1.14.13
+	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/nats-io/nats-server/v2 v2.9.0
-	github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0
+	github.com/nats-io/nats.go v1.17.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.12.2
+	github.com/prometheus/client_golang v1.13.0
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.7.1
-	github.com/tidwall/gjson v1.14.1
-	github.com/tidwall/sjson v1.2.4
+	github.com/stretchr/testify v1.8.0
+	github.com/tidwall/gjson v1.14.3
+	github.com/tidwall/sjson v1.2.5
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c
-	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
-	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
+	go.uber.org/atomic v1.10.0
+	golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0
+	golang.org/x/image v0.0.0-20220902085622-e7cb96979f69
 	golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105
-	golang.org/x/net v0.0.0-20220728211354-c7608f3a8462
-	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
+	golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1
+	golang.org/x/term v0.0.0-20220919170432-7a66f970e087
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	nhooyr.io/websocket v1.8.7
@@ -56,12 +56,12 @@ require (
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
-	github.com/Microsoft/go-winio v0.5.1 // indirect
-	github.com/RoaringBitmap/roaring v0.9.4 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/RoaringBitmap/roaring v1.2.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.2.0 // indirect
+	github.com/bits-and-blooms/bitset v1.3.3 // indirect
 	github.com/blevesearch/bleve_index_api v1.0.3 // indirect
-	github.com/blevesearch/geo v0.1.13 // indirect
+	github.com/blevesearch/geo v0.1.14 // indirect
 	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
 	github.com/blevesearch/gtreap v0.1.1 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
@@ -76,31 +76,27 @@ require (
 	github.com/blevesearch/zapx/v14 v14.3.5 // indirect
 	github.com/blevesearch/zapx/v15 v15.3.5 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/frankban/quicktest v1.14.3 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/glog v1.0.0 // indirect
+	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
-	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/lucas-clemente/quic-go v0.28.1 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
+	github.com/juju/errors v1.0.0 // indirect
+	github.com/klauspost/compress v1.15.10 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/lucas-clemente/quic-go v0.29.0 // indirect
 	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
-	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/miekg/dns v1.1.49 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -117,21 +113,21 @@ require (
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
-	go.etcd.io/bbolt v1.3.5 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77 // indirect
+	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
-	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
+	golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 // indirect
 	golang.org/x/tools v0.1.12 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.0.3 // indirect
 )
 
diff --git a/go.sum b/go.sum
index abdc11665..b180ea46f 100644
--- a/go.sum
+++ b/go.sum
@@ -38,8 +38,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a h1:yfbnOyqPcx2gi5cFIJ2rlPz5M6rFPHT/c8FgZmFjCdc=
-github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
+github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf h1:kjPkmDHUTWUma/4tqDl208bOk3jsUEqOJA6TsMZo5Jk=
+github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk=
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ=
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -55,11 +55,12 @@ github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGh
 github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
-github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
-github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo=
 github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
+github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A=
+github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
 github.com/RyanCarrier/dijkstra v1.1.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
 github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -86,14 +87,16 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/bits-and-blooms/bitset v1.3.3 h1:R1XWiopGiXf66xygsiLpzLo67xEYvMkHw3w+rCOSAwg=
+github.com/bits-and-blooms/bitset v1.3.3/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/blevesearch/bleve/v2 v2.3.4 h1:SSb7/cwGzo85LWX1jchIsXM8ZiNNMX3shT5lROM63ew=
 github.com/blevesearch/bleve/v2 v2.3.4/go.mod h1:Ot0zYum8XQRfPcwhae8bZmNyYubynsoMjVvl1jPqL30=
 github.com/blevesearch/bleve_index_api v1.0.3 h1:DDSWaPXOZZJ2BB73ZTWjKxydAugjwywcqU+91AAqcAg=
 github.com/blevesearch/bleve_index_api v1.0.3/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
-github.com/blevesearch/geo v0.1.13 h1:RsY1vfFm81iv1g+uoCQtsOFvKAhZnpOdTOK8JRA6pqw=
 github.com/blevesearch/geo v0.1.13/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc=
+github.com/blevesearch/geo v0.1.14 h1:TTDpJN6l9ck/cUYbXSn4aCElNls0Whe44rcQKsB7EfU=
+github.com/blevesearch/geo v0.1.14/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
 github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
@@ -134,7 +137,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -159,14 +161,14 @@ github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+
 github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
-github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405crbpxvWzKovETOQ=
-github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
+github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v20.10.18+incompatible h1:SN84VYXTBNGn92T/QwIRPlum9zfemfitN7pbsp26WSc=
+github.com/docker/docker v20.10.18+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
-github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
-github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
@@ -179,9 +181,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
-github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
-github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
@@ -204,9 +204,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -229,8 +231,9 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
 github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -262,8 +265,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo=
 github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4=
 github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -278,9 +282,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -313,7 +316,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
-github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -334,12 +336,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 h1:EJHbsNpQyupmMeWTq7inn+5L/WZ7JfzCVPJ+DP9McCQ=
-github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9/go.mod h1:TRm7EVGA3mQOqSVcBySRY7a9Y1/gyVhh/WTCnc5sD4U=
-github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4 h1:NO5tuyw++EGLnz56Q8KMyDZRwJwWO8jQnj285J3FOmY=
-github.com/juju/mgo/v2 v2.0.0-20210302023703-70d5d206e208 h1:/WiCm+Vpj87e4QWuWwPD/bNE9kDrWCLvPBHOQNcG2+A=
-github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 h1:XEDzpuZb8Ma7vLja3+5hzUqVTvAqm5Y+ygvnDs5iTMM=
-github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494/go.mod h1:rUquetT0ALL48LHZhyRGvjjBH8xZaZ8dFClulKK5wK4=
+github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM=
+github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
@@ -348,14 +346,13 @@ github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloL
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo=
+github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -365,48 +362,44 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
-github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lucas-clemente/quic-go v0.28.1 h1:Uo0lvVxWg5la9gflIF9lwa39ONq85Xq2D91YNEIslzU=
+github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
+github.com/lucas-clemente/quic-go v0.29.0 h1:Vw0mGTfmWqGzh4jx/kMymsIkFK6rErFVmg+t9RLrnZE=
+github.com/lucas-clemente/quic-go v0.29.0/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
-github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
 github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
-github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
 github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
 github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM=
 github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
-github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 h1:7m/WlWcSROrcK5NxuXaxYD32BZqe/LEEnBrWcH/cOqQ=
 github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
+github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU=
+github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM=
 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
-github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a h1:XeGBDZZsUe4kgj3myl0EiuDNVWxszJecMTrON3Wn9sI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220912142654-7b96db48888a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
-github.com/matrix-org/pinecone v0.0.0-20220914131522-608215eb1b29 h1:/AIaqhK1BBi2sMEVQdgZRV8H8sNloAGCgztLZhsPqD0=
-github.com/matrix-org/pinecone v0.0.0-20220914131522-608215eb1b29/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
-github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef h1:qnDDZVkqKAB8if91AUX2VtQGCGnxLBYYt2tG972SYPk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc h1:NAf6Xec/RoYD+/UT3m2q58yZAu0pYA9C4gc/RbS89ls=
+github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
-github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
+github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
-github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
-github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
-github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -431,13 +424,12 @@ github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
 github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
 github.com/nats-io/nats-server/v2 v2.9.0 h1:DLWu+7/VgGOoChcDKytnUZPAmudpv7o/MhKmNrnH1RE=
 github.com/nats-io/nats-server/v2 v2.9.0/go.mod h1:BWKY6217RvhI+FDoOLZ2BH+hOC37xeKRBlQ1Lz7teKI=
-github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0 h1:dPUKD6Iv8M1y9MU8PK6H4a4/12yx5/CbaYWz/Z1arY8=
-github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/nats-io/nats.go v1.17.0 h1:1jp5BThsdGlN91hW0k3YEfJbfACjiOYtUiLXG0RL4IE=
+github.com/nats-io/nats.go v1.17.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
 github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
@@ -487,8 +479,9 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
-github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
+github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -498,15 +491,17 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -553,26 +548,27 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
-github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
-github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
+github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
-github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
 github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
@@ -599,16 +595,18 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
+go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -619,7 +617,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -628,8 +625,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
-golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY=
+golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -644,11 +641,13 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4=
+golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 h1:LRtI4W37N+KFebI/qV0OFiLUv4GLOWeEW5hn/KEJvxE=
-golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 h1:Lj6HJGCSn5AjxRAH2+r35Mir4icalbqku+CLUtjnvXY=
+golang.org/x/image v0.0.0-20220902085622-e7cb96979f69/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -691,7 +690,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -712,14 +710,15 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 h1:UreQrH7DbFXSi9ZFox6FNT3WBooWmdANpU+IfkT1T4I=
 golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1 h1:TWZxd/th7FbRSMret2MVQdlI8uT49QEtwZdvJrxjEHU=
+golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -728,6 +727,7 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -761,8 +761,6 @@ golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -790,6 +788,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -800,25 +799,23 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77 h1:C1tElbkWrsSkn3IRl1GCW/gETw1TywWIPgwZtXTZbYg=
-golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
+golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
-golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w=
+golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -832,8 +829,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
-golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 h1:yuLAip3bfURHClMG9VBdzPrQvCWjWiWUTBGV+/fCbUs=
+golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -852,7 +849,6 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -884,7 +880,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
@@ -986,8 +981,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -999,8 +994,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM=
 gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
-gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
-gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
+gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI=
 gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o=
@@ -1016,8 +1010,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
diff --git a/internal/sqlutil/unique_constraint.go b/internal/sqlutil/unique_constraint.go
index 4a1b7fd94..ed70f5ed2 100644
--- a/internal/sqlutil/unique_constraint.go
+++ b/internal/sqlutil/unique_constraint.go
@@ -27,8 +27,6 @@ func IsUniqueConstraintViolationErr(err error) bool {
 	switch e := err.(type) {
 	case *pq.Error:
 		return e.Code == "23505"
-	case pq.Error:
-		return e.Code == "23505"
 	case *sqlite3.Error:
 		return e.Code == sqlite3.ErrConstraint
 	case sqlite3.Error:
diff --git a/internal/sqlutil/writer_exclusive.go b/internal/sqlutil/writer_exclusive.go
index 91dd77e4d..8eff3ce55 100644
--- a/internal/sqlutil/writer_exclusive.go
+++ b/internal/sqlutil/writer_exclusive.go
@@ -57,7 +57,7 @@ func (w *ExclusiveWriter) Do(db *sql.DB, txn *sql.Tx, f func(txn *sql.Tx) error)
 // opened using the database object from the task and then this will
 // be passed as a parameter to the task function.
 func (w *ExclusiveWriter) run() {
-	if !w.running.CAS(false, true) {
+	if !w.running.CompareAndSwap(false, true) {
 		return
 	}
 	if tracingEnabled {
diff --git a/setup/base/base.go b/setup/base/base.go
index 3070bed5f..0a34523e7 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -498,7 +498,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 			logrus.Infof("Starting internal %s listener on %s", b.componentName, internalServ.Addr)
 			b.ProcessContext.ComponentStarted()
 			internalServ.RegisterOnShutdown(func() {
-				if internalShutdown.CAS(false, true) {
+				if internalShutdown.CompareAndSwap(false, true) {
 					b.ProcessContext.ComponentFinished()
 					logrus.Infof("Stopped internal HTTP listener")
 				}
@@ -526,7 +526,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 			logrus.Infof("Starting external %s listener on %s", b.componentName, externalServ.Addr)
 			b.ProcessContext.ComponentStarted()
 			externalServ.RegisterOnShutdown(func() {
-				if externalShutdown.CAS(false, true) {
+				if externalShutdown.CompareAndSwap(false, true) {
 					b.ProcessContext.ComponentFinished()
 					logrus.Infof("Stopped external HTTP listener")
 				}
diff --git a/setup/process/process.go b/setup/process/process.go
index 01eb26e22..06ef60217 100644
--- a/setup/process/process.go
+++ b/setup/process/process.go
@@ -51,7 +51,7 @@ func (b *ProcessContext) WaitForComponentsToFinish() {
 }
 
 func (b *ProcessContext) Degraded() {
-	if b.degraded.CAS(false, true) {
+	if b.degraded.CompareAndSwap(false, true) {
 		logrus.Warn("Dendrite is running in a degraded state")
 		sentry.CaptureException(fmt.Errorf("Process is running in a degraded state"))
 	}

From d8b19c857ffa3f8818ef7f16fa2cda6ba9324430 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 20 Sep 2022 17:17:44 +0100
Subject: [PATCH 374/391] HTTP connection keepalives (#2730)

Beforehand we disabled HTTP keepalives to prevent ambient system
resources from being used by excess idle connections. Now that we've
fixed some bugs in the federation API and device list updater, this
situation is now much better and we don't open so many remote
connections anyway.

Keepalives allow us to not have to handshake TLS so often (which is
quite expensive) and reusing an idle connection is much faster than
having to open a new one. This can help with response times when talking
to remote federated servers.

This PR also adds a new option to disable keepalives if needed:

```
  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
  # more quickly than opening new connections each time. Disabling keepalives will close
  # HTTP connections immediately after a successful request but may result in more CPU and
  # memory being used on TLS handshakes for each new connection instead.
  disable_http_keepalives: false
```
---
 dendrite-sample.monolith.yaml        | 7 +++++++
 dendrite-sample.polylith.yaml        | 7 +++++++
 setup/base/base.go                   | 1 +
 setup/config/config_federationapi.go | 6 ++++++
 4 files changed, 21 insertions(+)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index 195d11610..f1758f54d 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -212,6 +212,13 @@ federation_api:
   # enable this option in production as it presents a security risk!
   disable_tls_validation: false
 
+  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
+  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
+  # more quickly than opening new connections each time. Disabling keepalives will close
+  # HTTP connections immediately after a successful request but may result in more CPU and
+  # memory being used on TLS handshakes for each new connection instead.
+  disable_http_keepalives: false
+
   # Perspective keyservers to use as a backup when direct key fetches fail. This may
   # be required to satisfy key requests for servers that are no longer online when
   # joining some rooms.
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 65d60e150..97d10825f 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -219,6 +219,13 @@ federation_api:
   # enable this option in production as it presents a security risk!
   disable_tls_validation: false
 
+  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
+  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
+  # more quickly than opening new connections each time. Disabling keepalives will close
+  # HTTP connections immediately after a successful request but may result in more CPU and
+  # memory being used on TLS handshakes for each new connection instead.
+  disable_http_keepalives: false
+
   # Perspective keyservers to use as a backup when direct key fetches fail. This may
   # be required to satisfy key requests for servers that are no longer online when
   # joining some rooms.
diff --git a/setup/base/base.go b/setup/base/base.go
index 0a34523e7..0c7b222d0 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -373,6 +373,7 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli
 	opts := []gomatrixserverlib.ClientOption{
 		gomatrixserverlib.WithTimeout(time.Minute * 5),
 		gomatrixserverlib.WithSkipVerify(b.Cfg.FederationAPI.DisableTLSValidation),
+		gomatrixserverlib.WithKeepAlives(!b.Cfg.FederationAPI.DisableHTTPKeepalives),
 	}
 	if b.Cfg.Global.DNSCache.Enabled {
 		opts = append(opts, gomatrixserverlib.WithDNSCache(b.DNSCache))
diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go
index d3a4b2b67..0f853865f 100644
--- a/setup/config/config_federationapi.go
+++ b/setup/config/config_federationapi.go
@@ -22,6 +22,11 @@ type FederationAPI struct {
 	// on remote federation endpoints. This is not recommended in production!
 	DisableTLSValidation bool `yaml:"disable_tls_validation"`
 
+	// DisableHTTPKeepalives prevents Dendrite from keeping HTTP connections
+	// open for reuse for future requests. Connections will be closed quicker
+	// but we may spend more time on TLS handshakes instead.
+	DisableHTTPKeepalives bool `yaml:"disable_http_keepalives"`
+
 	// Perspective keyservers, to use as a backup when direct key fetch
 	// requests don't succeed
 	KeyPerspectives KeyPerspectives `yaml:"key_perspectives"`
@@ -39,6 +44,7 @@ func (c *FederationAPI) Defaults(opts DefaultOpts) {
 	}
 	c.FederationMaxRetries = 16
 	c.DisableTLSValidation = false
+	c.DisableHTTPKeepalives = false
 	if opts.Generate {
 		c.KeyPerspectives = KeyPerspectives{
 			{

From 61a34d7cfb840bcf599218bf7c2559b6b8f95998 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 22 Sep 2022 11:21:43 +0100
Subject: [PATCH 375/391] Bump commonmarker from 0.23.4 to 0.23.6 in /docs
 (#2731)

Bumps [commonmarker](https://github.com/gjtorikian/commonmarker) from
0.23.4 to 0.23.6.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/gjtorikian/commonmarker/releases">commonmarker's
releases</a>.</em></p>
<blockquote>
<h2>v0.23.6</h2>
<h2>What's Changed</h2>
<p>This release includes two updates from the upstream
<code>cmark-gfm</code> library, namely:</p>
<ul>
<li><a href="https://github.com/github/cmark-gfm/releases">DoS
vulnerability in autolink extension</a> per <a
href="https://github.com/github/cmark-gfm/security/advisories/GHSA-cgh3-p57x-9q7q">GHSA-cgh3-p57x-9q7q</a></li>
<li><a
href="https://github.com/github/cmark-gfm/releases/tag/0.29.0.gfm.5">Added
<code>xmpp:</code> and <code>mailto:</code> support to the autolink
extension</a></li>
</ul>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/gjtorikian/commonmarker/blob/main/CHANGELOG.md">commonmarker's
changelog</a>.</em></p>
<blockquote>
<h1>Changelog</h1>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/a8f8d76fbc8c92ddb2e539a06bd93c5f8326705e"><code>a8f8d76</code></a>
Merge pull request <a
href="https://github-redirect.dependabot.com/gjtorikian/commonmarker/issues/190">#190</a>
from anticomputer/main</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/ac916346314aef3015a713f92f7b46a8c34e98ed"><code>ac91634</code></a>
:gem: release 0.23.6</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/777fd3054be0e0ba18f2fda18ccc7eeeee82db92"><code>777fd30</code></a>
Update cmark-upstream to <a
href="https://github.com/github/cmark-gfm/commit/9d57d8a23">https://github.com/github/cmark-gfm/commit/9d57d8a23</a>...</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/7aaeb37e9780e87a9e9fbdddfe1feba1c9f360f4"><code>7aaeb37</code></a>
Merge pull request <a
href="https://github-redirect.dependabot.com/gjtorikian/commonmarker/issues/188">#188</a>
from stevenlaidlaw/update-to-0290gfm5</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/795e628a406ec67f169440ed3aa84ba9483e2700"><code>795e628</code></a>
Update cmark-upstream to <a
href="https://github.com/github/cmark-gfm/commit/0578e1e4f">https://github.com/github/cmark-gfm/commit/0578e1e4f</a>...</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/39d19d65300c5735efcc77b2a57b65c207c013e7"><code>39d19d6</code></a>
Update cmark-upstream to <a
href="https://github.com/github/cmark-gfm/commit/766f161ef">https://github.com/github/cmark-gfm/commit/766f161ef</a>...</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/63b7bf89ee1be857c5a5757d6ea678c5c759b8b9"><code>63b7bf8</code></a>
Update FUNDING.yml</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/558c7275b18a7ae16136c0fc55f444458dd8cc58"><code>558c727</code></a>
Bump to 0.23.5</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/41eee7265f501305834d80e3d045ed6c9df77de2"><code>41eee72</code></a>
lint</li>
<li><a
href="https://github.com/gjtorikian/commonmarker/commit/897e8ed07d04b902a5cc3c928852bafdab4468aa"><code>897e8ed</code></a>
Merge pull request <a
href="https://github-redirect.dependabot.com/gjtorikian/commonmarker/issues/180">#180</a>
from lumaxis/main</li>
<li>Additional commits viewable in <a
href="https://github.com/gjtorikian/commonmarker/compare/v0.23.4...v0.23.6">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commonmarker&package-manager=bundler&previous-version=0.23.4&new-version=0.23.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
- `@dependabot use these labels` will set the current labels as the
default for future PRs for this repo and language
- `@dependabot use these reviewers` will set the current reviewers as
the default for future PRs for this repo and language
- `@dependabot use these assignees` will set the current assignees as
the default for future PRs for this repo and language
- `@dependabot use this milestone` will set the current milestone as the
default for future PRs for this repo and language

You can disable automated security fix PRs for this repo from the
[Security Alerts
page](https://github.com/matrix-org/dendrite/network/alerts).

</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 docs/Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
index 88d33ae84..bc73df728 100644
--- a/docs/Gemfile.lock
+++ b/docs/Gemfile.lock
@@ -14,7 +14,7 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.4)
+    commonmarker (0.23.6)
     concurrent-ruby (1.1.10)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)

From 852d856db85abb7dacf3d323b0bcb355736bc656 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 22 Sep 2022 13:55:40 +0100
Subject: [PATCH 376/391] Update embedded NATS Server to v2.9.1-beta1 (as this
 includes a fix that prevents high CPU usage after Dendrite startup)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index bd401ca65..48f6a1a3f 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
-	github.com/nats-io/nats-server/v2 v2.9.0
+	github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5
 	github.com/nats-io/nats.go v1.17.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
diff --git a/go.sum b/go.sum
index b180ea46f..73c20e4d1 100644
--- a/go.sum
+++ b/go.sum
@@ -422,8 +422,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
 github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
-github.com/nats-io/nats-server/v2 v2.9.0 h1:DLWu+7/VgGOoChcDKytnUZPAmudpv7o/MhKmNrnH1RE=
-github.com/nats-io/nats-server/v2 v2.9.0/go.mod h1:BWKY6217RvhI+FDoOLZ2BH+hOC37xeKRBlQ1Lz7teKI=
+github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5 h1:G/YGSXcJ2bUofD8Ts49it4VNezaJLQldI6fZR+wIUts=
+github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5/go.mod h1:BWKY6217RvhI+FDoOLZ2BH+hOC37xeKRBlQ1Lz7teKI=
 github.com/nats-io/nats.go v1.17.0 h1:1jp5BThsdGlN91hW0k3YEfJbfACjiOYtUiLXG0RL4IE=
 github.com/nats-io/nats.go v1.17.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=

From 0ddfb0cad4929e38d1995d8983e818432a942c81 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 22 Sep 2022 15:53:15 +0200
Subject: [PATCH 377/391] Tweak `InsertMigration` to avoid logging (#2720)

We'd still produce logs in Postgres when trying to insert a migration we
already ran. This should stop us from creating those log entries.
---
 federationapi/federationapi_keys_test.go | 10 +++-
 federationapi/federationapi_test.go      |  8 +--
 internal/sqlutil/migrate.go              | 62 ++++++++++++++----------
 internal/sqlutil/migrate_test.go         | 42 +++++++++++++---
 4 files changed, 82 insertions(+), 40 deletions(-)

diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go
index 4469da35b..85cc43aa5 100644
--- a/federationapi/federationapi_keys_test.go
+++ b/federationapi/federationapi_keys_test.go
@@ -12,12 +12,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type server struct {
@@ -86,7 +87,12 @@ func TestMain(m *testing.M) {
 			cfg.Global.JetStream.StoragePath = config.Path(d)
 			cfg.Global.KeyID = serverKeyID
 			cfg.Global.KeyValidityPeriod = s.validity
-			cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
+			f, err := os.CreateTemp(d, "federation_keys_test*.db")
+			if err != nil {
+				return -1
+			}
+			defer f.Close()
+			cfg.FederationAPI.Database.ConnectionString = config.DataSource("file:" + f.Name())
 			s.config = &cfg.FederationAPI
 
 			// Create a transport which redirects federation requests to
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index 15f7a6840..e923143a7 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -10,6 +10,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/internal"
@@ -20,9 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
-	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
 )
 
 type fedRoomserverAPI struct {
@@ -271,7 +272,6 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
 	cfg.Global.PrivateKey = privKey
 	cfg.Global.JetStream.InMemory = true
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	keyRing := &test.NopJSONVerifier{}
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go
index b6a8b1f25..a66a75826 100644
--- a/internal/sqlutil/migrate.go
+++ b/internal/sqlutil/migrate.go
@@ -49,12 +49,13 @@ type Migration struct {
 	Down func(ctx context.Context, txn *sql.Tx) error
 }
 
-// Migrator
+// Migrator contains fields required to run migrations.
 type Migrator struct {
 	db              *sql.DB
 	migrations      []Migration
 	knownMigrations map[string]struct{}
 	mutex           *sync.Mutex
+	insertStmt      *sql.Stmt
 }
 
 // NewMigrator creates a new DB migrator.
@@ -82,35 +83,26 @@ func (m *Migrator) AddMigrations(migrations ...Migration) {
 
 // Up executes all migrations in order they were added.
 func (m *Migrator) Up(ctx context.Context) error {
-	var (
-		err             error
-		dendriteVersion = internal.VersionString()
-	)
 	// ensure there is a table for known migrations
 	executedMigrations, err := m.ExecutedMigrations(ctx)
 	if err != nil {
 		return fmt.Errorf("unable to create/get migrations: %w", err)
 	}
-
+	// ensure we close the insert statement, as it's not needed anymore
+	defer m.close()
 	return WithTransaction(m.db, func(txn *sql.Tx) error {
 		for i := range m.migrations {
-			now := time.Now().UTC().Format(time.RFC3339)
 			migration := m.migrations[i]
 			// Skip migration if it was already executed
 			if _, ok := executedMigrations[migration.Version]; ok {
 				continue
 			}
 			logrus.Debugf("Executing database migration '%s'", migration.Version)
-			err = migration.Up(ctx, txn)
-			if err != nil {
+
+			if err = migration.Up(ctx, txn); err != nil {
 				return fmt.Errorf("unable to execute migration '%s': %w", migration.Version, err)
 			}
-			_, err = txn.ExecContext(ctx, insertVersionSQL,
-				migration.Version,
-				now,
-				dendriteVersion,
-			)
-			if err != nil {
+			if err = m.insertMigration(ctx, txn, migration.Version); err != nil {
 				return fmt.Errorf("unable to insert executed migrations: %w", err)
 			}
 		}
@@ -118,6 +110,23 @@ func (m *Migrator) Up(ctx context.Context) error {
 	})
 }
 
+func (m *Migrator) insertMigration(ctx context.Context, txn *sql.Tx, migrationName string) error {
+	if m.insertStmt == nil {
+		stmt, err := m.db.Prepare(insertVersionSQL)
+		if err != nil {
+			return fmt.Errorf("unable to prepare insert statement: %w", err)
+		}
+		m.insertStmt = stmt
+	}
+	stmt := TxStmtContext(ctx, txn, m.insertStmt)
+	_, err := stmt.ExecContext(ctx,
+		migrationName,
+		time.Now().Format(time.RFC3339),
+		internal.VersionString(),
+	)
+	return err
+}
+
 // ExecutedMigrations returns a map with already executed migrations in addition to creating the
 // migrations table, if it doesn't exist.
 func (m *Migrator) ExecutedMigrations(ctx context.Context) (map[string]struct{}, error) {
@@ -146,19 +155,20 @@ func (m *Migrator) ExecutedMigrations(ctx context.Context) (map[string]struct{},
 // inserts a migration given their name to the database.
 // This should only be used when manually inserting migrations.
 func InsertMigration(ctx context.Context, db *sql.DB, migrationName string) error {
-	_, err := db.ExecContext(ctx, createDBMigrationsSQL)
+	m := NewMigrator(db)
+	defer m.close()
+	existingMigrations, err := m.ExecutedMigrations(ctx)
 	if err != nil {
-		return fmt.Errorf("unable to create db_migrations: %w", err)
+		return err
 	}
-	_, err = db.ExecContext(ctx, insertVersionSQL,
-		migrationName,
-		time.Now().Format(time.RFC3339),
-		internal.VersionString(),
-	)
-	// If the migration was already executed, we'll get a unique constraint error,
-	// return nil instead, to avoid unnecessary logging.
-	if IsUniqueConstraintViolationErr(err) {
+	if _, ok := existingMigrations[migrationName]; ok {
 		return nil
 	}
-	return err
+	return m.insertMigration(ctx, nil, migrationName)
+}
+
+func (m *Migrator) close() {
+	if m.insertStmt != nil {
+		internal.CloseAndLogIfError(context.Background(), m.insertStmt, "unable to close insert statement")
+	}
 }
diff --git a/internal/sqlutil/migrate_test.go b/internal/sqlutil/migrate_test.go
index d8bcae196..5116237a1 100644
--- a/internal/sqlutil/migrate_test.go
+++ b/internal/sqlutil/migrate_test.go
@@ -7,9 +7,10 @@ import (
 	"reflect"
 	"testing"
 
+	_ "github.com/mattn/go-sqlite3"
+
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/test"
-	_ "github.com/mattn/go-sqlite3"
 )
 
 var dummyMigrations = []sqlutil.Migration{
@@ -81,11 +82,12 @@ func Test_migrations_Up(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-				conStr, close := test.PrepareDBConnectionString(t, dbType)
-				defer close()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		conStr, close := test.PrepareDBConnectionString(t, dbType)
+		defer close()
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
 				driverName := "sqlite3"
 				if dbType == test.DBTypePostgres {
 					driverName = "postgres"
@@ -107,6 +109,30 @@ func Test_migrations_Up(t *testing.T) {
 					t.Errorf("expected: %+v, got %v", tt.wantResult, result)
 				}
 			})
-		})
-	}
+		}
+	})
+}
+
+func Test_insertMigration(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		conStr, close := test.PrepareDBConnectionString(t, dbType)
+		defer close()
+		driverName := "sqlite3"
+		if dbType == test.DBTypePostgres {
+			driverName = "postgres"
+		}
+
+		db, err := sql.Open(driverName, conStr)
+		if err != nil {
+			t.Errorf("unable to open database: %v", err)
+		}
+
+		if err := sqlutil.InsertMigration(context.Background(), db, "testing"); err != nil {
+			t.Fatalf("unable to insert migration: %s", err)
+		}
+		// Second insert should not return an error, as it was already executed.
+		if err := sqlutil.InsertMigration(context.Background(), db, "testing"); err != nil {
+			t.Fatalf("unable to insert migration: %s", err)
+		}
+	})
 }

From f40e2803276001cafd4d382c64a661d6ea740870 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 22 Sep 2022 14:54:25 +0100
Subject: [PATCH 378/391] Version 0.9.9 (#2732)

Changelog and version bump.
---
 CHANGES.md          | 21 +++++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 79d2fe2e1..657ca1920 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,26 @@
 # Changelog
 
+## Dendrite 0.9.9 (2022-09-22)
+
+### Features
+
+* Dendrite will now try to keep HTTP connections open to remote federated servers for a few minutes after a request and attempt to reuse those connections where possible
+  * This should reduce the amount of time spent on TLS handshakes and often speed up requests to remote servers
+  * This new behaviour can be disabled with the `federation_api.disable_http_keepalives` option if needed
+* A number of dependencies have been updated
+
+### Fixes
+
+* A bug where the roomserver did not correctly propagate rewritten room state to downstream components (like the federation API and sync API) has been fixed, which could cause issues when performing a federated join to a previously left room
+* Event auth now correctly parses the `join_authorised_via_users_server` field in the membership event content
+* Database migrations should no longer produce unique constraint errors at Dendrite startup
+* The `origin` of device list updates should now be populated correctly
+* Send-to-device messages will no longer be dropped if we fail to publish them to specific devices
+* The roomserver query to find state after events will now always resolve state if there are multiple prev events
+* The roomserver will now return no memberships if querying history visibility for an event which has no state snapshot
+* The device list updater will now mark a device list as stale if a requesting device ID is not known
+* Transactions sent to appservices should no longer have accidental duplicated transaction IDs (contributed by [tak-hntlabs](https://github.com/tak-hntlabs))
+
 ## Dendrite 0.9.8 (2022-09-12)
 
 ### Important
diff --git a/internal/version.go b/internal/version.go
index ef7b879c2..f9b101702 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 9
-	VersionPatch = 8
+	VersionPatch = 9
 	VersionTag   = "" // example: "rc1"
 )
 

From c53f284fdbe635e1f87dc1da13fcf14b4b16e6a3 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 22 Sep 2022 17:49:35 +0200
Subject: [PATCH 379/391] Get the `DeviceListPosition` before anything else in
 complete syncs (#2733)

This should hopefully unflake `Can query remote device keys using POST`
in Complement.
---
 syncapi/sync/requestpool.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index c2c9616e8..b2ea105ff 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -308,6 +308,12 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 		if syncReq.Since.IsEmpty() {
 			// Complete sync
 			syncReq.Response.NextBatch = types.StreamingToken{
+				// Get the current DeviceListPosition first, as the currentPosition
+				// might advance while processing other streams, resulting in flakey
+				// tests.
+				DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
+					syncReq.Context, syncReq,
+				),
 				PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
 					syncReq.Context, syncReq,
 				),
@@ -329,9 +335,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
 					syncReq.Context, syncReq,
 				),
-				DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
-				),
 				PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
 					syncReq.Context, syncReq,
 				),

From a50556dcf0e364f9fb365c5fa5b4e72adc3d57b3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 23 Sep 2022 12:59:00 +0100
Subject: [PATCH 380/391] Tweak mainline ordering (update to
 matrix-org/gomatrixserverlib@2217f6c)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 48f6a1a3f..68b5cef84 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3
 	github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
diff --git a/go.sum b/go.sum
index 73c20e4d1..216b4a5fc 100644
--- a/go.sum
+++ b/go.sum
@@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef h1:qnDDZVkqKAB8if91AUX2VtQGCGnxLBYYt2tG972SYPk=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 h1:u3FKZmXxfhv3XhD8RziBlt96QTt8eHFhg1upCloBh2g=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc h1:NAf6Xec/RoYD+/UT3m2q58yZAu0pYA9C4gc/RbS89ls=
 github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

From 14b73220032d64550cdbe88bd2ad64806d72b60e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 23 Sep 2022 15:44:21 +0100
Subject: [PATCH 381/391] Add `-dir` option to `dendrite-demo-pinecone` and
 `dendrite-demo-yggdrasil`

---
 cmd/dendrite-demo-pinecone/main.go  | 20 +++++++++++---------
 cmd/dendrite-demo-yggdrasil/main.go | 22 ++++++++++++----------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index 99e597e59..da63f9a2c 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -62,6 +63,7 @@ var (
 	instancePort   = flag.Int("port", 8008, "the port that the client API will listen on")
 	instancePeer   = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
 	instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
+	instanceDir    = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
 )
 
 // nolint:gocyclo
@@ -88,7 +90,7 @@ func main() {
 		cfg = setup.ParseFlags(true)
 		sk = cfg.Global.PrivateKey
 	} else {
-		keyfile := *instanceName + ".pem"
+		keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
 		if _, err := os.Stat(keyfile); os.IsNotExist(err) {
 			oldkeyfile := *instanceName + ".key"
 			if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
@@ -129,15 +131,15 @@ func main() {
 			Monolithic: true,
 		})
 		cfg.Global.PrivateKey = sk
-		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
-		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
+		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(*instanceDir, *instanceName)))
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
-		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
+		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.ClientAPI.RegistrationDisabled = false
 		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 		if err := cfg.Derive(); err != nil {
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 868d9826a..cd0066679 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -56,6 +57,7 @@ var (
 	instancePort   = flag.Int("port", 8008, "the port that the client API will listen on")
 	instancePeer   = flag.String("peer", "", "the static Yggdrasil peers to connect to, comma separated-list")
 	instanceListen = flag.String("listen", "tcp://:0", "the port Yggdrasil peers can connect to")
+	instanceDir    = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
 )
 
 func main() {
@@ -76,7 +78,7 @@ func main() {
 
 	cfg := &config.Dendrite{}
 
-	keyfile := *instanceName + ".pem"
+	keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
 	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
 		oldkeyfile := *instanceName + ".key"
 		if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
@@ -121,15 +123,15 @@ func main() {
 			Monolithic: true,
 		})
 		cfg.Global.PrivateKey = sk
-		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
-		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-		cfg.MSCs.MSCs = []string{"msc2836"}
-		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
+		cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName))
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.ClientAPI.RegistrationDisabled = false
 		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
 		if err := cfg.Derive(); err != nil {

From d5978d98fda8a31918973a2481e62eafcec9f80f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 23 Sep 2022 16:21:12 +0100
Subject: [PATCH 382/391] Update to
 matrix-org/pinecone@0900fceecb89ad0c14ee0ce825be638bf2a18474

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 68b5cef84..ded7f28b0 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3
-	github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc
+	github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5
diff --git a/go.sum b/go.sum
index 216b4a5fc..570e6172f 100644
--- a/go.sum
+++ b/go.sum
@@ -386,8 +386,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 h1:u3FKZmXxfhv3XhD8RziBlt96QTt8eHFhg1upCloBh2g=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
-github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc h1:NAf6Xec/RoYD+/UT3m2q58yZAu0pYA9C4gc/RbS89ls=
-github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
+github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89 h1:Ym50Fgn3GiYya4p29k3nJ5nYsalFGev3eIm3DeGNIq4=
+github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=

From 8d64c24b23518fde968020e5a093564acc46235a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 26 Sep 2022 09:33:34 +0100
Subject: [PATCH 383/391] Update documentation to state that Dendrite requires
 PostgreSQL UTF-8 encoding

---
 docs/installation/4_database.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/installation/4_database.md b/docs/installation/4_database.md
index 68f2d44d0..d64ee6615 100644
--- a/docs/installation/4_database.md
+++ b/docs/installation/4_database.md
@@ -16,6 +16,9 @@ Dendrite can automatically populate the database with the relevant tables and in
 it is not capable of creating the databases themselves. You will need to create the databases
 manually.
 
+The databases **must** be created with UTF-8 encoding configured or you will likely run into problems
+with your Dendrite deployment.
+
 At this point, you can choose to either use a single database for all Dendrite components,
 or you can run each component with its own separate database:
 
@@ -65,7 +68,7 @@ sudo -u postgres createuser -P dendrite
 Create the database itself, using the `dendrite` role from above:
 
 ```bash
-sudo -u postgres createdb -O dendrite dendrite
+sudo -u postgres createdb -O dendrite -E UTF-8 dendrite
 ```
 
 ### Multiple database creation
@@ -85,7 +88,7 @@ The following eight components require a database. In this example they will be
 
 ```bash
 for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver userapi; do
-    sudo -u postgres createdb -O dendrite dendrite_$i
+    sudo -u postgres createdb -O dendrite -E UTF-8 dendrite_$i
 done
 ```
 

From 3c416517b0045a27ec0bd03f285600b733dd5d1c Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 26 Sep 2022 10:45:35 +0200
Subject: [PATCH 384/391] Fix possible "Database is locked" issue

---
 userapi/storage/shared/storage.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index acc65212c..e32a442d0 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -777,7 +777,7 @@ func (d *Database) GetPushers(
 func (d *Database) RemovePusher(
 	ctx context.Context, appid, pushkey, localpart string,
 ) error {
-	return d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		err := d.Pushers.DeletePusher(ctx, txn, appid, pushkey, localpart)
 		if err == sql.ErrNoRows {
 			return nil
@@ -792,7 +792,7 @@ func (d *Database) RemovePusher(
 func (d *Database) RemovePushers(
 	ctx context.Context, appid, pushkey string,
 ) error {
-	return d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.Pushers.DeletePushers(ctx, txn, appid, pushkey)
 	})
 }

From 3e87096a21729fcc7e074d09ee12da56394dd15d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 26 Sep 2022 09:54:54 +0100
Subject: [PATCH 385/391] Use `TxStmt` in SQLite pusher table

---
 userapi/storage/sqlite3/pusher_table.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/userapi/storage/sqlite3/pusher_table.go b/userapi/storage/sqlite3/pusher_table.go
index d5bd1617b..dba97c3d4 100644
--- a/userapi/storage/sqlite3/pusher_table.go
+++ b/userapi/storage/sqlite3/pusher_table.go
@@ -96,7 +96,7 @@ func (s *pushersStatements) InsertPusher(
 	ctx context.Context, txn *sql.Tx, session_id int64,
 	pushkey string, pushkeyTS int64, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string,
 ) error {
-	_, err := s.insertPusherStmt.ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data)
+	_, err := sqlutil.TxStmt(txn, s.insertPusherStmt).ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data)
 	logrus.Debugf("Created pusher %d", session_id)
 	return err
 }
@@ -144,13 +144,13 @@ func (s *pushersStatements) SelectPushers(
 func (s *pushersStatements) DeletePusher(
 	ctx context.Context, txn *sql.Tx, appid, pushkey, localpart string,
 ) error {
-	_, err := s.deletePusherStmt.ExecContext(ctx, appid, pushkey, localpart)
+	_, err := sqlutil.TxStmt(txn, s.deletePusherStmt).ExecContext(ctx, appid, pushkey, localpart)
 	return err
 }
 
 func (s *pushersStatements) DeletePushers(
 	ctx context.Context, txn *sql.Tx, appid, pushkey string,
 ) error {
-	_, err := s.deletePushersByAppIdAndPushKeyStmt.ExecContext(ctx, appid, pushkey)
+	_, err := sqlutil.TxStmt(txn, s.deletePushersByAppIdAndPushKeyStmt).ExecContext(ctx, appid, pushkey)
 	return err
 }

From f022fc1397fda984245ad1611531b37480cf4f46 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 26 Sep 2022 17:35:35 +0100
Subject: [PATCH 386/391] Remove `origin` field from PDUs (#2737)

This nukes the `origin` field from PDUs as per
matrix-org/matrix-spec#998, matrix-org/gomatrixserverlib#341.
---
 federationapi/internal/perform.go             |  7 ++---
 federationapi/routing/invite.go               |  9 ++++++-
 federationapi/routing/join.go                 | 18 ++++---------
 federationapi/routing/leave.go                | 27 ++++++++++++-------
 go.mod                                        |  2 +-
 go.sum                                        |  4 +--
 roomserver/internal/input/input_events.go     | 22 ++++++++-------
 .../internal/perform/perform_backfill.go      |  4 ++-
 roomserver/internal/perform/perform_invite.go |  8 ++++--
 roomserver/internal/perform/perform_leave.go  | 15 +++++++----
 roomserver/storage/shared/storage.go          |  2 +-
 11 files changed, 69 insertions(+), 49 deletions(-)

diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 9100c8f18..84702f4ce 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -217,7 +217,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		var remoteEvent *gomatrixserverlib.Event
 		remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion)
 		if err == nil && isWellFormedMembershipEvent(
-			remoteEvent, roomID, userID, r.cfg.Matrix.ServerName,
+			remoteEvent, roomID, userID,
 		) {
 			event = remoteEvent
 		}
@@ -285,7 +285,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 
 // isWellFormedMembershipEvent returns true if the event looks like a legitimate
 // membership event.
-func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string, origin gomatrixserverlib.ServerName) bool {
+func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string) bool {
 	if membership, err := event.Membership(); err != nil {
 		return false
 	} else if membership != gomatrixserverlib.Join {
@@ -294,9 +294,6 @@ func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID
 	if event.RoomID() != roomID {
 		return false
 	}
-	if event.Origin() != origin {
-		return false
-	}
 	if !event.StateKeyEquals(userID) {
 		return false
 	}
diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go
index 4b795018c..504204504 100644
--- a/federationapi/routing/invite.go
+++ b/federationapi/routing/invite.go
@@ -148,8 +148,15 @@ func processInvite(
 			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
 		}
 	}
+	_, serverName, err := gomatrixserverlib.SplitID('@', event.Sender())
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON contains an invalid sender"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
-		ServerName:             event.Origin(),
+		ServerName:             serverName,
 		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 1a1219873..74d065e59 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -203,14 +203,6 @@ func SendJoin(
 		}
 	}
 
-	// Check that the event is from the server sending the request.
-	if event.Origin() != request.Origin() {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The join must be sent by the server it originated on"),
-		}
-	}
-
 	// Check that a state key is provided.
 	if event.StateKey() == nil || event.StateKeyEquals("") {
 		return util.JSONResponse{
@@ -228,16 +220,16 @@ func SendJoin(
 	// Check that the sender belongs to the server that is sending us
 	// the request. By this point we've already asserted that the sender
 	// and the state key are equal so we don't need to check both.
-	var domain gomatrixserverlib.ServerName
-	if _, domain, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
+	var serverName gomatrixserverlib.ServerName
+	if _, serverName, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("The sender of the join is invalid"),
 		}
-	} else if domain != request.Origin() {
+	} else if serverName != request.Origin() {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The sender of the join must belong to the origin server"),
+			JSON: jsonerror.Forbidden("The sender does not match the server that originated the request"),
 		}
 	}
 
@@ -292,7 +284,7 @@ func SendJoin(
 		}
 	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
-		ServerName:             event.Origin(),
+		ServerName:             serverName,
 		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index 8e43ce959..a67e4e28b 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -118,6 +118,7 @@ func MakeLeave(
 }
 
 // SendLeave implements the /send_leave API
+// nolint:gocyclo
 func SendLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
@@ -167,14 +168,6 @@ func SendLeave(
 		}
 	}
 
-	// Check that the event is from the server sending the request.
-	if event.Origin() != request.Origin() {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The leave must be sent by the server it originated on"),
-		}
-	}
-
 	if event.StateKey() == nil || event.StateKeyEquals("") {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -188,6 +181,22 @@ func SendLeave(
 		}
 	}
 
+	// Check that the sender belongs to the server that is sending us
+	// the request. By this point we've already asserted that the sender
+	// and the state key are equal so we don't need to check both.
+	var serverName gomatrixserverlib.ServerName
+	if _, serverName, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender of the join is invalid"),
+		}
+	} else if serverName != request.Origin() {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender does not match the server that originated the request"),
+		}
+	}
+
 	// Check if the user has already left. If so, no-op!
 	queryReq := &api.QueryLatestEventsAndStateRequest{
 		RoomID: roomID,
@@ -240,7 +249,7 @@ func SendLeave(
 		}
 	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
-		ServerName:             event.Origin(),
+		ServerName:             serverName,
 		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
diff --git a/go.mod b/go.mod
index ded7f28b0..3d99a71e1 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5
 	github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
diff --git a/go.sum b/go.sum
index 570e6172f..32edf8c11 100644
--- a/go.sum
+++ b/go.sum
@@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 h1:u3FKZmXxfhv3XhD8RziBlt96QTt8eHFhg1upCloBh2g=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89 h1:Ym50Fgn3GiYya4p29k3nJ5nYsalFGev3eIm3DeGNIq4=
 github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 29af649ad..01fd62010 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -118,6 +118,10 @@ func (r *Inputer) processRoomEvent(
 	if roomInfo == nil && !isCreateEvent {
 		return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID())
 	}
+	_, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender())
+	if err != nil {
+		return fmt.Errorf("event has invalid sender %q", input.Event.Sender())
+	}
 
 	// If we already know about this outlier and it hasn't been rejected
 	// then we won't attempt to reprocess it. If it was rejected or has now
@@ -145,7 +149,8 @@ func (r *Inputer) processRoomEvent(
 	var missingAuth, missingPrev bool
 	serverRes := &fedapi.QueryJoinedHostServerNamesInRoomResponse{}
 	if !isCreateEvent {
-		missingAuthIDs, missingPrevIDs, err := r.DB.MissingAuthPrevEvents(ctx, event)
+		var missingAuthIDs, missingPrevIDs []string
+		missingAuthIDs, missingPrevIDs, err = r.DB.MissingAuthPrevEvents(ctx, event)
 		if err != nil {
 			return fmt.Errorf("updater.MissingAuthPrevEvents: %w", err)
 		}
@@ -158,7 +163,7 @@ func (r *Inputer) processRoomEvent(
 			RoomID:      event.RoomID(),
 			ExcludeSelf: true,
 		}
-		if err := r.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, serverReq, serverRes); err != nil {
+		if err = r.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, serverReq, serverRes); err != nil {
 			return fmt.Errorf("r.FSAPI.QueryJoinedHostServerNamesInRoom: %w", err)
 		}
 		// Sort all of the servers into a map so that we can randomise
@@ -173,9 +178,9 @@ func (r *Inputer) processRoomEvent(
 			serverRes.ServerNames = append(serverRes.ServerNames, input.Origin)
 			delete(servers, input.Origin)
 		}
-		if origin := event.Origin(); origin != input.Origin {
-			serverRes.ServerNames = append(serverRes.ServerNames, origin)
-			delete(servers, origin)
+		if senderDomain != input.Origin {
+			serverRes.ServerNames = append(serverRes.ServerNames, senderDomain)
+			delete(servers, senderDomain)
 		}
 		for server := range servers {
 			serverRes.ServerNames = append(serverRes.ServerNames, server)
@@ -188,7 +193,7 @@ func (r *Inputer) processRoomEvent(
 	isRejected := false
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 	knownEvents := map[string]*types.Event{}
-	if err := r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil {
+	if err = r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil {
 		return fmt.Errorf("r.fetchAuthEvents: %w", err)
 	}
 
@@ -231,7 +236,6 @@ func (r *Inputer) processRoomEvent(
 	if input.Kind == api.KindNew {
 		// Check that the event passes authentication checks based on the
 		// current room state.
-		var err error
 		softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs)
 		if err != nil {
 			logger.WithError(err).Warn("Error authing soft-failed event")
@@ -265,7 +269,8 @@ func (r *Inputer) processRoomEvent(
 				hadEvents:  map[string]bool{},
 				haveEvents: map[string]*gomatrixserverlib.Event{},
 			}
-			if stateSnapshot, err := missingState.processEventWithMissingState(ctx, event, headered.RoomVersion); err != nil {
+			var stateSnapshot *parsedRespState
+			if stateSnapshot, err = missingState.processEventWithMissingState(ctx, event, headered.RoomVersion); err != nil {
 				// Something went wrong with retrieving the missing state, so we can't
 				// really do anything with the event other than reject it at this point.
 				isRejected = true
@@ -302,7 +307,6 @@ func (r *Inputer) processRoomEvent(
 	// burning CPU time.
 	historyVisibility := gomatrixserverlib.HistoryVisibilityShared // Default to shared.
 	if input.Kind != api.KindOutlier && rejectionErr == nil && !isRejected {
-		var err error
 		historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev)
 		if err != nil {
 			return fmt.Errorf("r.processStateBefore: %w", err)
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index 51c66415a..69a075733 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -468,7 +468,9 @@ FindSuccessor:
 	// Store the server names in a temporary map to avoid duplicates.
 	serverSet := make(map[gomatrixserverlib.ServerName]bool)
 	for _, event := range memberEvents {
-		serverSet[event.Origin()] = true
+		if _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()); err == nil {
+			serverSet[senderDomain] = true
+		}
 	}
 	var servers []gomatrixserverlib.ServerName
 	for server := range serverSet {
diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go
index 483e78c3f..3fbdf332e 100644
--- a/roomserver/internal/perform/perform_invite.go
+++ b/roomserver/internal/perform/perform_invite.go
@@ -50,6 +50,10 @@ func (r *Inviter) PerformInvite(
 	if event.StateKey() == nil {
 		return nil, fmt.Errorf("invite must be a state event")
 	}
+	_, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender())
+	if err != nil {
+		return nil, fmt.Errorf("sender %q is invalid", event.Sender())
+	}
 
 	roomID := event.RoomID()
 	targetUserID := *event.StateKey()
@@ -67,7 +71,7 @@ func (r *Inviter) PerformInvite(
 		return nil, nil
 	}
 	isTargetLocal := domain == r.Cfg.Matrix.ServerName
-	isOriginLocal := event.Origin() == r.Cfg.Matrix.ServerName
+	isOriginLocal := senderDomain == r.Cfg.Matrix.ServerName
 	if !isOriginLocal && !isTargetLocal {
 		res.Error = &api.PerformError{
 			Code: api.PerformErrorBadRequest,
@@ -235,7 +239,7 @@ func (r *Inviter) PerformInvite(
 			{
 				Kind:         api.KindNew,
 				Event:        event,
-				Origin:       event.Origin(),
+				Origin:       senderDomain,
 				SendAsServer: req.SendAsServer,
 			},
 		},
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index 036404cd2..ada3aab06 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -81,12 +81,11 @@ func (r *Leaver) performLeaveRoomByID(
 	// that.
 	isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
-		var host gomatrixserverlib.ServerName
-		_, host, err = gomatrixserverlib.SplitID('@', senderUser)
-		if err != nil {
+		_, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser)
+		if serr != nil {
 			return nil, fmt.Errorf("sender %q is invalid", senderUser)
 		}
-		if host != r.Cfg.Matrix.ServerName {
+		if senderDomain != r.Cfg.Matrix.ServerName {
 			return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID)
 		}
 		// check that this is not a "server notice room"
@@ -172,6 +171,12 @@ func (r *Leaver) performLeaveRoomByID(
 		return nil, fmt.Errorf("eventutil.BuildEvent: %w", err)
 	}
 
+	// Get the sender domain.
+	_, senderDomain, serr := gomatrixserverlib.SplitID('@', event.Sender())
+	if serr != nil {
+		return nil, fmt.Errorf("sender %q is invalid", event.Sender())
+	}
+
 	// Give our leave event to the roomserver input stream. The
 	// roomserver will process the membership change and notify
 	// downstream automatically.
@@ -180,7 +185,7 @@ func (r *Leaver) performLeaveRoomByID(
 			{
 				Kind:         api.KindNew,
 				Event:        event.Headered(buildRes.RoomVersion),
-				Origin:       event.Origin(),
+				Origin:       senderDomain,
 				SendAsServer: string(r.Cfg.Matrix.ServerName),
 			},
 		},
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 00a17e5cb..593abbea1 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -896,7 +896,7 @@ func (d *Database) handleRedactions(
 	switch {
 	case redactUser >= pl.Redact:
 		// The power level of the redaction event’s sender is greater than or equal to the redact level.
-	case redactedEvent.Origin() == redactionEvent.Origin() && redactedEvent.Sender() == redactionEvent.Sender():
+	case redactedEvent.Sender() == redactionEvent.Sender():
 		// The domain of the redaction event’s sender matches that of the original event’s sender.
 	default:
 		return nil, "", nil

From 40fec70d1336f97e31e1d23cc0576b543cb119fc Mon Sep 17 00:00:00 2001
From: networkException <github@nwex.de>
Date: Tue, 27 Sep 2022 10:39:39 +0200
Subject: [PATCH 387/391] Add pinecone demo container image (#2710)

This pull request adds the configuration and CI steps to build and
publish a container wrapping the `dendrite-demo-pinecone` command as
well as fixes a sentence structure issue in the pull request template.

As this does not touch any go source code no tests have been added

### Pull Request Checklist

<!-- Please read docs/CONTRIBUTING.md before submitting your pull
request -->

* [x] I have added tests for PR _or_ I have justified why this PR
doesn't need tests.
* [x] Pull request includes a [sign
off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)

Signed-off-by: networkException <git@nwex.de> (by private sign-off)
Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 .github/PULL_REQUEST_TEMPLATE.md      |  2 +-
 .github/workflows/docker.yml          | 60 +++++++++++++++++++++++++++
 build/docker/Dockerfile.demo-pinecone | 25 +++++++++++
 3 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 build/docker/Dockerfile.demo-pinecone

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 9bfb01667..e0b82e2aa 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -2,7 +2,7 @@
 
 <!-- Please read docs/CONTRIBUTING.md before submitting your pull request -->
 
-* [ ] I have added added tests for PR _or_ I have justified why this PR doesn't need tests.
+* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests.
 * [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)
 
 Signed-off-by: `Your Name <your@email.example.org>`
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 642587924..b4e24e52f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -137,3 +137,63 @@ jobs:
             ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
+
+  demo-pinecone:
+    name: Pinecone demo image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get release tag
+        if: github.event_name == 'release' # Only for GitHub releases
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to GitHub Containers
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build main pinecone demo image
+        if: github.ref_name == 'main'
+        id: docker_build_demo_pinecone
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          file: ./build/docker/Dockerfile.demo-pinecone
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
+
+      - name: Build release pinecone demo image
+        if: github.event_name == 'release' # Only for GitHub releases
+        id: docker_build_demo_pinecone_release
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          file: ./build/docker/Dockerfile.demo-pinecone
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest
+            ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }}
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest
+            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }}
diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone
new file mode 100644
index 000000000..a5d709c7a
--- /dev/null
+++ b/build/docker/Dockerfile.demo-pinecone
@@ -0,0 +1,25 @@
+FROM docker.io/golang:1.18-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
+
+FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+COPY --from=base /build/bin/* /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]

From 12649ccedd858cbe075271ea234b1e268f973c5a Mon Sep 17 00:00:00 2001
From: PiotrKozimor <37144818+PiotrKozimor@users.noreply.github.com>
Date: Tue, 27 Sep 2022 10:41:36 +0200
Subject: [PATCH 388/391] Improve selectRoomIDsWithAnyMembershipSQL performance
 (#2738)

Recently I have observed that dendrite spends a lot of time (~390s) in
`selectRoomIDsWithAnyMembershipSQL` query

```
dendrite_syncapi=# select total_exec_time, left(query,100) from pg_stat_statements order by total_exec_time desc limit 5 ;
  total_exec_time   |                                                 left
--------------------+------------------------------------------------------------------------------------------------------
  747826.5800519128 | SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_vis
  389130.5490339942 | SELECT DISTINCT room_id, membership FROM syncapi_current_room_state WHERE type = $2 AND state_key =
 376104.17514700035 | SELECT psd.datname, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_
   363644.164092031 | SELECT event_type_nid, event_state_key_nid, event_nid FROM roomserver_events WHERE event_nid = ANY($
  58570.48104699995 | SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND ( $2::te
(5 rows)
```

Explain analyze showed correct usage of `syncapi_room_state_unique`
index:

```
dendrite_syncapi=#
explain analyze SELECT distinct room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = '@qjfl:dendrite.stg.globekeeper.com';
                                                                               QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Unique  (cost=2749.38..2749.56 rows=24 width=52) (actual time=2.933..2.956 rows=65 loops=1)
   ->  Sort  (cost=2749.38..2749.44 rows=24 width=52) (actual time=2.932..2.937 rows=65 loops=1)
         Sort Key: room_id, membership
         Sort Method: quicksort  Memory: 34kB
         ->  Index Scan using syncapi_room_state_unique on syncapi_current_room_state  (cost=0.41..2748.83 rows=24 width=52) (actual time=0.030..2.890 rows=65 loops=1)
               Index Cond: ((type = 'm.room.member'::text) AND (state_key = '@qjfl:dendrite.stg.globekeeper.com'::text))
 Planning Time: 0.140 ms
 Execution Time: 2.990 ms
(8 rows)
```

Multi-column indexes in Postgres shall perform well for leftmost
columns, but I gave it a try and created
`syncapi_current_room_state_type_state_key_idx` index. I could observe
significant performance improvement. Execution time dropped from 2.9 ms
to 0.24 ms:

```
explain analyze SELECT distinct room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = '@qjfl:dendrite.stg.globekeeper.com';
                                                                             QUERY PLAN
--------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Unique  (cost=96.46..96.64 rows=24 width=52) (actual time=0.199..0.218 rows=65 loops=1)
   ->  Sort  (cost=96.46..96.52 rows=24 width=52) (actual time=0.199..0.202 rows=65 loops=1)
         Sort Key: room_id, membership
         Sort Method: quicksort  Memory: 34kB
         ->  Bitmap Heap Scan on syncapi_current_room_state  (cost=4.53..95.91 rows=24 width=52) (actual time=0.048..0.139 rows=65 loops=1)
               Recheck Cond: ((type = 'm.room.member'::text) AND (state_key = '@qjfl:dendrite.stg.globekeeper.com'::text))
               Heap Blocks: exact=59
               ->  Bitmap Index Scan on syncapi_current_room_state_type_state_key_idx  (cost=0.00..4.53 rows=24 width=0) (actual time=0.037..0.037 rows=65 loops=1)
                     Index Cond: ((type = 'm.room.member'::text) AND (state_key = '@qjfl:dendrite.stg.globekeeper.com'::text))
 Planning Time: 0.236 ms
 Execution Time: 0.242 ms
(11 rows)
```

Next improvement is skipping DISTINCT and rely on map assignment in
`SelectRoomIDsWithAnyMembership`. Execution time drops by almost half:

```
explain analyze SELECT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = '@qjfl:dendrite.stg.globekeeper.com';
                                                                       QUERY PLAN
--------------------------------------------------------------------------------------------------------------------------------------------------------
 Bitmap Heap Scan on syncapi_current_room_state  (cost=4.53..95.91 rows=24 width=52) (actual time=0.032..0.113 rows=65 loops=1)
   Recheck Cond: ((type = 'm.room.member'::text) AND (state_key = '@qjfl:dendrite.stg.globekeeper.com'::text))
   Heap Blocks: exact=59
   ->  Bitmap Index Scan on syncapi_current_room_state_type_state_key_idx  (cost=0.00..4.53 rows=24 width=0) (actual time=0.021..0.021 rows=65 loops=1)
         Index Cond: ((type = 'm.room.member'::text) AND (state_key = '@qjfl:dendrite.stg.globekeeper.com'::text))
 Planning Time: 0.087 ms
 Execution Time: 0.136 ms
(7 rows)
```

In our env we spend only 1s on inserting to table, so the write penalty
of creating an index should be small.
```
dendrite_syncapi=# select total_exec_time, left(query,100) from pg_stat_statements where query like '%INSERT%syncapi_current_room_state%' order by total_exec_time desc;
  total_exec_time   |                                                 left
--------------------+------------------------------------------------------------------------------------------------------
 1139.9057619999971 | INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, he
(1 row)
```

This PR does not require test modifications.

### Pull Request Checklist

<!-- Please read docs/CONTRIBUTING.md before submitting your pull
request -->

* [x] I have added added tests for PR _or_ I have justified why this PR
doesn't need tests.
* [x] Pull request includes a [sign
off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)

Signed-off-by: `Piotr Kozimor <p1996k@gmail.com>`
---
 syncapi/storage/postgres/current_room_state_table.go | 4 +++-
 syncapi/storage/sqlite3/current_room_state_table.go  | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 083d10b88..5e6daaaf8 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -62,6 +62,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_s
 CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
 -- for querying state by event IDs
 CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON syncapi_current_room_state(event_id);
+-- for improving selectRoomIDsWithAnyMembershipSQL
+CREATE INDEX IF NOT EXISTS syncapi_current_room_state_type_state_key_idx ON syncapi_current_room_state(type, state_key);
 `
 
 const upsertRoomStateSQL = "" +
@@ -80,7 +82,7 @@ const selectRoomIDsWithMembershipSQL = "" +
 	"SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
 
 const selectRoomIDsWithAnyMembershipSQL = "" +
-	"SELECT DISTINCT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1"
+	"SELECT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1"
 
 const selectCurrentStateSQL = "" +
 	"SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" +
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index b88c11f88..bd1271dd6 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -51,6 +51,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_s
 -- CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
 -- for querying state by event IDs
 CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON syncapi_current_room_state(event_id);
+-- for improving selectRoomIDsWithAnyMembershipSQL
+CREATE INDEX IF NOT EXISTS syncapi_current_room_state_type_state_key_idx ON syncapi_current_room_state(type, state_key);
 `
 
 const upsertRoomStateSQL = "" +
@@ -69,7 +71,7 @@ const selectRoomIDsWithMembershipSQL = "" +
 	"SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
 
 const selectRoomIDsWithAnyMembershipSQL = "" +
-	"SELECT DISTINCT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1"
+	"SELECT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1"
 
 const selectCurrentStateSQL = "" +
 	"SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1"

From b5bfff47d296e704fbeb33c5b57829609bed0426 Mon Sep 17 00:00:00 2001
From: Dov Alperin <DAlperin@users.noreply.github.com>
Date: Tue, 27 Sep 2022 04:42:08 -0400
Subject: [PATCH 389/391] Use /usr/bin/env bash in shebangs to make them
 universal (#2735)

Some systems (like nixos) don't have bash living at `/bin/bash` so using
`/usr/bin/env bash` we can make these scripts universal.

### Pull Request Checklist

<!-- Please read docs/CONTRIBUTING.md before submitting your pull
request -->

* [X] I have added added tests for PR _or_ I have justified why this PR
doesn't need tests.
* [x] Pull request includes a [sign
off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)

Signed-off-by: `Dov Alperin <git@dov.dev>`

Signed-off-by: `Dov Alperin <git@dov.dev>`
---
 build/docker/images-build.sh | 2 +-
 build/docker/images-pull.sh  | 2 +-
 build/docker/images-push.sh  | 2 +-
 run-sytest.sh                | 2 +-
 show-expected-fail-tests.sh  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index eaed5f6dc..c2c140685 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 cd $(git rev-parse --show-toplevel)
 
diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh
index 496e80067..f3f98ce7c 100755
--- a/build/docker/images-pull.sh
+++ b/build/docker/images-pull.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 TAG=${1:-latest}
 
diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh
index fd9b999ea..248fdee2b 100755
--- a/build/docker/images-push.sh
+++ b/build/docker/images-push.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 TAG=${1:-latest}
 
diff --git a/run-sytest.sh b/run-sytest.sh
index e23982397..4ed1c8d45 100755
--- a/run-sytest.sh
+++ b/run-sytest.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Runs SyTest either from Docker Hub, or from ../sytest. If it's run
 # locally, the Docker image is rebuilt first.
diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh
index 3ed937a0f..b7af8f648 100755
--- a/show-expected-fail-tests.sh
+++ b/show-expected-fail-tests.sh
@@ -1,4 +1,4 @@
-#! /bin/bash
+#!/usr/bin/env bash
 #
 # Parses a results.tap file from SyTest output and a file containing test names (a test whitelist)
 # and checks whether a test name that exists in the whitelist (that should pass), failed or not.

From d531202b0e877a41765f9a833f6c52b6c8eae919 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 27 Sep 2022 10:52:03 +0100
Subject: [PATCH 390/391] Build Docker images using Go 1.19 (related to #2714)

---
 build/docker/Dockerfile.demo-pinecone | 2 +-
 build/docker/Dockerfile.monolith      | 2 +-
 build/docker/Dockerfile.polylith      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone
index a5d709c7a..133c63c53 100644
--- a/build/docker/Dockerfile.demo-pinecone
+++ b/build/docker/Dockerfile.demo-pinecone
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.18-alpine AS base
+FROM docker.io/golang:1.19-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 
diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
index bb02934cd..3180e9626 100644
--- a/build/docker/Dockerfile.monolith
+++ b/build/docker/Dockerfile.monolith
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.18-alpine AS base
+FROM docker.io/golang:1.19-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
index 166ea99cb..79f8a5f23 100644
--- a/build/docker/Dockerfile.polylith
+++ b/build/docker/Dockerfile.polylith
@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.18-alpine AS base
+FROM docker.io/golang:1.19-alpine AS base
 
 RUN apk --update --no-cache add bash build-base
 

From f18bce93cc3e7e5f57ebc55d309360b7f8703553 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 27 Sep 2022 11:15:49 +0100
Subject: [PATCH 391/391] Pinecone hybrid routing (update to
 matrix-org/pinecone#67)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 3d99a71e1..b682d9bc4 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5
-	github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89
+	github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5
diff --git a/go.sum b/go.sum
index 32edf8c11..1afed73a5 100644
--- a/go.sum
+++ b/go.sum
@@ -386,8 +386,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
-github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89 h1:Ym50Fgn3GiYya4p29k3nJ5nYsalFGev3eIm3DeGNIq4=
-github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
+github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d h1:kGPJ6Rg8nn5an2CbCZrRiuTNyNzE0rRMiqm4UXJYrRs=
+github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=