From 6c67552bf9eee18f656d731adf646aa09c5d7c92 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 27 Sep 2022 15:50:22 +0100
Subject: [PATCH 01/73] Return `M_UNRECOGNIZED` for unknown CS API
 endpoints/actions (#2740)

Fixes #2739.
---
 setup/base/base.go | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/setup/base/base.go b/setup/base/base.go
index 0c7b222d0..32716c766 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -392,17 +392,26 @@ func (b *BaseDendrite) configureHTTPErrors() {
 		_, _ = w.Write([]byte(fmt.Sprintf("405 %s not allowed on this endpoint", r.Method)))
 	}
 
+	clientNotFoundHandler := func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"errcode":"M_UNRECOGNIZED","error":"Unrecognized request"}`)) // nolint:misspell
+	}
+
 	notFoundCORSHandler := httputil.WrapHandlerInCORS(http.NotFoundHandler())
 	notAllowedCORSHandler := httputil.WrapHandlerInCORS(http.HandlerFunc(notAllowedHandler))
 
 	for _, router := range []*mux.Router{
-		b.PublicClientAPIMux, b.PublicMediaAPIMux,
-		b.DendriteAdminMux, b.SynapseAdminMux,
-		b.PublicWellKnownAPIMux,
+		b.PublicMediaAPIMux, b.DendriteAdminMux,
+		b.SynapseAdminMux, b.PublicWellKnownAPIMux,
 	} {
 		router.NotFoundHandler = notFoundCORSHandler
 		router.MethodNotAllowedHandler = notAllowedCORSHandler
 	}
+
+	// Special case so that we don't upset clients on the CS API.
+	b.PublicClientAPIMux.NotFoundHandler = http.HandlerFunc(clientNotFoundHandler)
+	b.PublicClientAPIMux.MethodNotAllowedHandler = http.HandlerFunc(clientNotFoundHandler)
 }
 
 // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on

From 87be32ca2671173a4287a938932e543410a32c3a Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 27 Sep 2022 18:06:49 +0200
Subject: [PATCH 02/73] Fulltext implementation using Bleve (#2675)

Based on #2480

This actually indexes events based on their event type. They are removed
from the index if we receive a `m.room.redaction` event on the
`OutputRoomEvent` stream.
An admin endpoint is added to reindex all existing events.


Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 clientapi/routing/admin.go                    |  22 ++
 clientapi/routing/routing.go                  |  17 +-
 cmd/generate-config/main.go                   |   9 +-
 dendrite-sample.monolith.yaml                 |   9 +-
 dendrite-sample.polylith.yaml                 |   9 +-
 docs/administration/4_adminapi.md             |   5 +
 docs/installation/7_configuration.md          |   4 +-
 internal/fulltext/bleve.go                    |   3 +-
 internal/fulltext/bleve_test.go               |  10 +-
 setup/base/base.go                            |   7 +-
 setup/config/config_syncapi.go                |  10 +-
 syncapi/consumers/clientapi.go                | 109 +++++-
 syncapi/consumers/roomserver.go               |  54 +++
 syncapi/routing/context.go                    |   7 +-
 syncapi/routing/routing.go                    |  28 +-
 syncapi/routing/search.go                     | 344 ++++++++++++++++++
 syncapi/storage/interface.go                  |   1 +
 .../postgres/output_room_events_table.go      |  28 ++
 syncapi/storage/shared/syncserver.go          |   8 +
 .../sqlite3/output_room_events_table.go       |  41 +++
 syncapi/storage/tables/interface.go           |   1 +
 syncapi/syncapi.go                            |   7 +-
 22 files changed, 680 insertions(+), 53 deletions(-)
 create mode 100644 syncapi/routing/search.go

diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index 0c5f8c167..5089d7c36 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -3,15 +3,19 @@ package routing
 import (
 	"encoding/json"
 	"net/http"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
 )
 
 func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
@@ -138,3 +142,21 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 		},
 	}
 }
+
+func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse {
+	if device.AccountType != userapi.AccountTypeAdmin {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+		}
+	}
+	_, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10)
+	if err != nil {
+		logrus.WithError(err).Error("failed to publish nats message")
+		return jsonerror.InternalServerError()
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index d7a48d228..9c1f8f720 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -20,6 +20,12 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/auth"
@@ -34,11 +40,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/nats-io/nats.go"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 )
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
@@ -161,6 +162,12 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/fulltext/reindex",
+		httputil.MakeAuthAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminReindex(req, cfg, device, natsClient)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go
index c24e8153e..8b042c56e 100644
--- a/cmd/generate-config/main.go
+++ b/cmd/generate-config/main.go
@@ -5,10 +5,11 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 	"gopkg.in/yaml.v2"
+
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 func main() {
@@ -82,6 +83,12 @@ func main() {
 				EnableInbound:  true,
 				EnableOutbound: true,
 			}
+			cfg.SyncAPI.Fulltext = config.Fulltext{
+				Enabled:   true,
+				IndexPath: config.Path(filepath.Join(*dirPath, "fulltextindex")),
+				InMemory:  true,
+				Language:  "en",
+			}
 		}
 	} else {
 		var err error
diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index f1758f54d..3cad17da8 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -275,10 +275,15 @@ sync_api:
   # address of the client. This is likely required if Dendrite is running behind
   # a reverse proxy server.
   # real_ip_header: X-Real-IP
-  fulltext:
+
+  # Configuration for the fulltext search
+  search:
     enabled: false
+    # The path where the fulltext index will be created in.
     index_path: "./fulltextindex"
-    language: "en" # more possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
+    # The language most likely to be used on the server - used when indexing, to ensure the returned results match the expectations.
+    # A full list of possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
+    language: "en"
 
 # Configuration for the User API.
 user_api:
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 97d10825f..e58062fe2 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -326,10 +326,15 @@ sync_api:
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
-  fulltext:
+
+  # Configuration for the fulltext search
+  search:
     enabled: false
+    # The path where the fulltext index will be created in.
     index_path: "./fulltextindex"
-    language: "en" # more possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
+    # The language most likely to be used on the server - used when indexing, to ensure the returned results match the expectations.
+    # A full list of possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
+    language: "en"
 
   # This option controls which HTTP header to inspect to find the real remote IP
   # address of the client. This is likely required if Dendrite is running behind
diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index a34bfde1f..1712bb1bf 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -57,6 +57,11 @@ Request body format:
 Reset the password of a local user. The `localpart` is the username only, i.e. if
 the full user ID is `@alice:domain.com` then the local part is `alice`.
 
+## GET `/_dendrite/admin/fulltext/reindex`
+
+This endpoint instructs Dendrite to reindex all searchable events (`m.room.message`, `m.room.topic` and `m.room.name`). An empty JSON body will be returned immediately.
+Indexing is done in the background, the server logs every 1000 events (or below) when they are being indexed. Once reindexing is done, you'll see something along the lines `Indexed 69586 events in 53.68223182s` in your debug logs.
+
 ## POST `/_synapse/admin/v1/send_server_notice`
 
 Request body format:
diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md
index 8fbe71c40..67cd339cf 100644
--- a/docs/installation/7_configuration.md
+++ b/docs/installation/7_configuration.md
@@ -140,12 +140,12 @@ room_server:
 
 ## Fulltext search
 
-Dendrite supports experimental fulltext indexing using [Bleve](https://github.com/blevesearch/bleve), it is configured in the `sync_api` section as follows. Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, to ensure the returned results match the expections. A full list of possible languages can be found [here](https://github.com/blevesearch/bleve/tree/master/analysis/lang).
+Dendrite supports experimental fulltext indexing using [Bleve](https://github.com/blevesearch/bleve), it is configured in the `sync_api` section as follows. Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, to ensure the returned results match the expectations. A full list of possible languages can be found [here](https://github.com/blevesearch/bleve/tree/master/analysis/lang).
 
 ```yaml
 sync_api:
   # ...
-  fulltext:
+  search:
     enabled: false
     index_path: "./fulltextindex"
     language: "en"
diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go
index b07c0e51d..da8932f5c 100644
--- a/internal/fulltext/bleve.go
+++ b/internal/fulltext/bleve.go
@@ -22,8 +22,9 @@ import (
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/mapping"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/setup/config"
 )
 
 // Search contains all existing bleve.Index
diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go
index 84a282423..d16397a45 100644
--- a/internal/fulltext/bleve_test.go
+++ b/internal/fulltext/bleve_test.go
@@ -27,11 +27,11 @@ import (
 
 func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search {
 	t.Helper()
-	cfg := config.Fulltext{}
-	cfg.Defaults(config.DefaultOpts{
-		Generate:   true,
-		Monolithic: true,
-	})
+	cfg := config.Fulltext{
+		Enabled:  true,
+		InMemory: true,
+		Language: "en",
+	}
 	if tempDir != "" {
 		cfg.IndexPath = config.Path(tempDir)
 		cfg.InMemory = false
diff --git a/setup/base/base.go b/setup/base/base.go
index 32716c766..0636c7b8d 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -37,16 +37,13 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/fulltext"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushgateway"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
-
 	"github.com/gorilla/mux"
 	"github.com/kardianos/minwinsvc"
 
@@ -61,6 +58,8 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/setup/process"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp"
 )
diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go
index c890b0054..edef22c93 100644
--- a/setup/config/config_syncapi.go
+++ b/setup/config/config_syncapi.go
@@ -10,7 +10,7 @@ type SyncAPI struct {
 
 	RealIPHeader string `yaml:"real_ip_header"`
 
-	Fulltext Fulltext `yaml:"fulltext"`
+	Fulltext Fulltext `yaml:"search"`
 }
 
 func (c *SyncAPI) Defaults(opts DefaultOpts) {
@@ -52,16 +52,12 @@ func (f *Fulltext) Defaults(opts DefaultOpts) {
 	f.Enabled = false
 	f.IndexPath = "./fulltextindex"
 	f.Language = "en"
-	if opts.Generate {
-		f.Enabled = true
-		f.InMemory = true
-	}
 }
 
 func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	if !f.Enabled {
 		return
 	}
-	checkNotEmpty(configErrs, "syncapi.fulltext.index_path", string(f.IndexPath))
-	checkNotEmpty(configErrs, "syncapi.fulltext.language", f.Language)
+	checkNotEmpty(configErrs, "syncapi.search.index_path", string(f.IndexPath))
+	checkNotEmpty(configErrs, "syncapi.search.language", f.Language)
 }
diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go
index a170a6ec1..b11ed4f5e 100644
--- a/syncapi/consumers/clientapi.go
+++ b/syncapi/consumers/clientapi.go
@@ -17,14 +17,18 @@ package consumers
 import (
 	"context"
 	"encoding/json"
+	"strings"
+	"time"
 
 	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/internal/fulltext"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -35,14 +39,18 @@ import (
 
 // OutputClientDataConsumer consumes events that originated in the client API server.
 type OutputClientDataConsumer struct {
-	ctx        context.Context
-	jetstream  nats.JetStreamContext
-	durable    string
-	topic      string
-	db         storage.Database
-	stream     types.StreamProvider
-	notifier   *notifier.Notifier
-	serverName gomatrixserverlib.ServerName
+	ctx          context.Context
+	jetstream    nats.JetStreamContext
+	nats         *nats.Conn
+	durable      string
+	topic        string
+	topicReIndex string
+	db           storage.Database
+	stream       types.StreamProvider
+	notifier     *notifier.Notifier
+	serverName   gomatrixserverlib.ServerName
+	fts          *fulltext.Search
+	cfg          *config.SyncAPI
 }
 
 // NewOutputClientDataConsumer creates a new OutputClientData consumer. Call Start() to begin consuming from room servers.
@@ -50,24 +58,93 @@ func NewOutputClientDataConsumer(
 	process *process.ProcessContext,
 	cfg *config.SyncAPI,
 	js nats.JetStreamContext,
+	nats *nats.Conn,
 	store storage.Database,
 	notifier *notifier.Notifier,
 	stream types.StreamProvider,
+	fts *fulltext.Search,
 ) *OutputClientDataConsumer {
 	return &OutputClientDataConsumer{
-		ctx:        process.Context(),
-		jetstream:  js,
-		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
-		durable:    cfg.Matrix.JetStream.Durable("SyncAPIAccountDataConsumer"),
-		db:         store,
-		notifier:   notifier,
-		stream:     stream,
-		serverName: cfg.Matrix.ServerName,
+		ctx:          process.Context(),
+		jetstream:    js,
+		topic:        cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
+		topicReIndex: cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex),
+		durable:      cfg.Matrix.JetStream.Durable("SyncAPIAccountDataConsumer"),
+		nats:         nats,
+		db:           store,
+		notifier:     notifier,
+		stream:       stream,
+		serverName:   cfg.Matrix.ServerName,
+		fts:          fts,
+		cfg:          cfg,
 	}
 }
 
 // Start consuming from room servers
 func (s *OutputClientDataConsumer) Start() error {
+	_, err := s.nats.Subscribe(s.topicReIndex, func(msg *nats.Msg) {
+		if err := msg.Ack(); err != nil {
+			return
+		}
+		if !s.cfg.Fulltext.Enabled {
+			logrus.Warn("Fulltext indexing is disabled")
+			return
+		}
+		ctx := context.Background()
+		logrus.Debugf("Starting to index events")
+		var offset int
+		start := time.Now()
+		count := 0
+		var id int64 = 0
+		for {
+			evs, err := s.db.ReIndex(ctx, 1000, id)
+			if err != nil {
+				logrus.WithError(err).Errorf("unable to get events to index")
+				return
+			}
+			if len(evs) == 0 {
+				break
+			}
+			logrus.Debugf("Indexing %d events", len(evs))
+			elements := make([]fulltext.IndexElement, 0, len(evs))
+
+			for streamPos, ev := range evs {
+				id = streamPos
+				e := fulltext.IndexElement{
+					EventID:        ev.EventID(),
+					RoomID:         ev.RoomID(),
+					StreamPosition: streamPos,
+				}
+				e.SetContentType(ev.Type())
+
+				switch ev.Type() {
+				case "m.room.message":
+					e.Content = gjson.GetBytes(ev.Content(), "body").String()
+				case gomatrixserverlib.MRoomName:
+					e.Content = gjson.GetBytes(ev.Content(), "name").String()
+				case gomatrixserverlib.MRoomTopic:
+					e.Content = gjson.GetBytes(ev.Content(), "topic").String()
+				default:
+					continue
+				}
+
+				if strings.TrimSpace(e.Content) == "" {
+					continue
+				}
+				elements = append(elements, e)
+			}
+			if err = s.fts.Index(elements...); err != nil {
+				logrus.WithError(err).Error("unable to index events")
+				continue
+			}
+			offset += len(evs)
+			count += len(elements)
+		}
+		logrus.Debugf("Indexed %d events in %v", count, time.Since(start))
+	})
+	if err != nil {
+		return err
+	}
 	return jetstream.JetStreamConsumer(
 		s.ctx, s.jetstream, s.topic, s.durable, 1,
 		s.onMessage, nats.DeliverAll(), nats.ManualAck(),
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 0964ae207..3756ad75c 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -24,7 +24,9 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
 
+	"github.com/matrix-org/dendrite/internal/fulltext"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -46,6 +48,7 @@ type OutputRoomEventConsumer struct {
 	pduStream    types.StreamProvider
 	inviteStream types.StreamProvider
 	notifier     *notifier.Notifier
+	fts          *fulltext.Search
 }
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@@ -58,6 +61,7 @@ func NewOutputRoomEventConsumer(
 	pduStream types.StreamProvider,
 	inviteStream types.StreamProvider,
 	rsAPI api.SyncRoomserverAPI,
+	fts *fulltext.Search,
 ) *OutputRoomEventConsumer {
 	return &OutputRoomEventConsumer{
 		ctx:          process.Context(),
@@ -70,6 +74,7 @@ func NewOutputRoomEventConsumer(
 		pduStream:    pduStream,
 		inviteStream: inviteStream,
 		rsAPI:        rsAPI,
+		fts:          fts,
 	}
 }
 
@@ -251,6 +256,12 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 		}).Panicf("roomserver output log: write new event failure")
 		return nil
 	}
+	if err = s.writeFTS(ev, pduPos); err != nil {
+		log.WithFields(log.Fields{
+			"event_id": ev.EventID(),
+			"type":     ev.Type(),
+		}).WithError(err).Warn("failed to index fulltext element")
+	}
 
 	if pduPos, err = s.notifyJoinedPeeks(ctx, ev, pduPos); err != nil {
 		log.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos)
@@ -295,6 +306,13 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent(
 		return nil
 	}
 
+	if err = s.writeFTS(ev, pduPos); err != nil {
+		log.WithFields(log.Fields{
+			"event_id": ev.EventID(),
+			"type":     ev.Type(),
+		}).WithError(err).Warn("failed to index fulltext element")
+	}
+
 	if pduPos, err = s.notifyJoinedPeeks(ctx, ev, pduPos); err != nil {
 		log.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos)
 		return err
@@ -451,3 +469,39 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head
 	event.Event, err = event.SetUnsigned(prev)
 	return event, err
 }
+
+func (s *OutputRoomEventConsumer) writeFTS(ev *gomatrixserverlib.HeaderedEvent, pduPosition types.StreamPosition) error {
+	if !s.cfg.Fulltext.Enabled {
+		return nil
+	}
+	e := fulltext.IndexElement{
+		EventID:        ev.EventID(),
+		RoomID:         ev.RoomID(),
+		StreamPosition: int64(pduPosition),
+	}
+	e.SetContentType(ev.Type())
+
+	switch ev.Type() {
+	case "m.room.message":
+		e.Content = gjson.GetBytes(ev.Content(), "body").String()
+	case gomatrixserverlib.MRoomName:
+		e.Content = gjson.GetBytes(ev.Content(), "name").String()
+	case gomatrixserverlib.MRoomTopic:
+		e.Content = gjson.GetBytes(ev.Content(), "topic").String()
+	case gomatrixserverlib.MRoomRedaction:
+		log.Tracef("Redacting event: %s", ev.Redacts())
+		if err := s.fts.Delete(ev.Redacts()); err != nil {
+			return fmt.Errorf("failed to delete entry from fulltext index: %w", err)
+		}
+		return nil
+	default:
+		return nil
+	}
+	if e.Content != "" {
+		log.Tracef("Indexing element: %+v", e)
+		if err := s.fts.Index(e); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 13c4e9d89..1ebdfe604 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -37,11 +37,11 @@ import (
 
 type ContextRespsonse struct {
 	End          string                          `json:"end"`
-	Event        gomatrixserverlib.ClientEvent   `json:"event"`
+	Event        *gomatrixserverlib.ClientEvent  `json:"event,omitempty"`
 	EventsAfter  []gomatrixserverlib.ClientEvent `json:"events_after,omitempty"`
 	EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before,omitempty"`
 	Start        string                          `json:"start"`
-	State        []gomatrixserverlib.ClientEvent `json:"state"`
+	State        []gomatrixserverlib.ClientEvent `json:"state,omitempty"`
 }
 
 func Context(
@@ -162,8 +162,9 @@ func Context(
 	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfterFiltered, gomatrixserverlib.FormatAll)
 	newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache)
 
+	ev := gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll)
 	response := ContextRespsonse{
-		Event:        gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll),
+		Event:        &ev,
 		EventsAfter:  eventsAfterClient,
 		EventsBefore: eventsBeforeClient,
 		State:        gomatrixserverlib.HeaderedToClientEvents(newState, gomatrixserverlib.FormatAll),
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 6bc495d8d..8f84a1341 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -18,15 +18,18 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/fulltext"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/sync"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 // Setup configures the given mux with sync-server listeners
@@ -40,6 +43,7 @@ func Setup(
 	rsAPI api.SyncRoomserverAPI,
 	cfg *config.SyncAPI,
 	lazyLoadCache caching.LazyLoadCache,
+	fts *fulltext.Search,
 ) {
 	v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
 
@@ -95,4 +99,24 @@ func Setup(
 			)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/search",
+		httputil.MakeAuthAPI("search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			if !cfg.Fulltext.Enabled {
+				return util.JSONResponse{
+					Code: http.StatusNotImplemented,
+					JSON: jsonerror.Unknown("Search has been disabled by the server administrator."),
+				}
+			}
+			var nextBatch *string
+			if err := req.ParseForm(); err != nil {
+				return jsonerror.InternalServerError()
+			}
+			if req.Form.Has("next_batch") {
+				nb := req.FormValue("next_batch")
+				nextBatch = &nb
+			}
+			return Search(req, device, syncDB, fts, nextBatch)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
 }
diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go
new file mode 100644
index 000000000..341efeb14
--- /dev/null
+++ b/syncapi/routing/search.go
@@ -0,0 +1,344 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"context"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/blevesearch/bleve/v2/search"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/fulltext"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/userapi/api"
+)
+
+// nolint:gocyclo
+func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts *fulltext.Search, from *string) util.JSONResponse {
+	start := time.Now()
+	var (
+		searchReq SearchRequest
+		err       error
+		ctx       = req.Context()
+	)
+	resErr := httputil.UnmarshalJSONRequest(req, &searchReq)
+	if resErr != nil {
+		logrus.Error("failed to unmarshal search request")
+		return *resErr
+	}
+
+	nextBatch := 0
+	if from != nil && *from != "" {
+		nextBatch, err = strconv.Atoi(*from)
+		if err != nil {
+			return jsonerror.InternalServerError()
+		}
+	}
+
+	if searchReq.SearchCategories.RoomEvents.Filter.Limit == 0 {
+		searchReq.SearchCategories.RoomEvents.Filter.Limit = 5
+	}
+
+	// only search rooms the user is actually joined to
+	joinedRooms, err := syncDB.RoomIDsWithMembership(ctx, device.UserID, "join")
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	if len(joinedRooms) == 0 {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("User not joined to any rooms."),
+		}
+	}
+	joinedRoomsMap := make(map[string]struct{}, len(joinedRooms))
+	for _, roomID := range joinedRooms {
+		joinedRoomsMap[roomID] = struct{}{}
+	}
+	rooms := []string{}
+	if searchReq.SearchCategories.RoomEvents.Filter.Rooms != nil {
+		for _, roomID := range *searchReq.SearchCategories.RoomEvents.Filter.Rooms {
+			if _, ok := joinedRoomsMap[roomID]; ok {
+				rooms = append(rooms, roomID)
+			}
+		}
+	} else {
+		rooms = joinedRooms
+	}
+
+	if len(rooms) == 0 {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Unknown("User not allowed to search in this room(s)."),
+		}
+	}
+
+	orderByTime := searchReq.SearchCategories.RoomEvents.OrderBy == "recent"
+
+	result, err := fts.Search(
+		searchReq.SearchCategories.RoomEvents.SearchTerm,
+		rooms,
+		searchReq.SearchCategories.RoomEvents.Keys,
+		searchReq.SearchCategories.RoomEvents.Filter.Limit,
+		nextBatch,
+		orderByTime,
+	)
+	if err != nil {
+		logrus.WithError(err).Error("failed to search fulltext")
+		return jsonerror.InternalServerError()
+	}
+	logrus.Debugf("Search took %s", result.Took)
+
+	// From was specified but empty, return no results, only the count
+	if from != nil && *from == "" {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: SearchResponse{
+				SearchCategories: SearchCategories{
+					RoomEvents: RoomEvents{
+						Count:     int(result.Total),
+						NextBatch: nil,
+					},
+				},
+			},
+		}
+	}
+
+	results := []Result{}
+
+	wantEvents := make([]string, 0, len(result.Hits))
+	eventScore := make(map[string]*search.DocumentMatch)
+
+	for _, hit := range result.Hits {
+		wantEvents = append(wantEvents, hit.ID)
+		eventScore[hit.ID] = hit
+	}
+
+	// Filter on m.room.message, as otherwise we also get events like m.reaction
+	// which "breaks" displaying results in Element Web.
+	types := []string{"m.room.message"}
+	roomFilter := &gomatrixserverlib.RoomEventFilter{
+		Rooms: &rooms,
+		Types: &types,
+	}
+
+	evs, err := syncDB.Events(ctx, wantEvents)
+	if err != nil {
+		logrus.WithError(err).Error("failed to get events from database")
+		return jsonerror.InternalServerError()
+	}
+
+	groups := make(map[string]RoomResult)
+	knownUsersProfiles := make(map[string]ProfileInfo)
+
+	// Sort the events by depth, as the returned values aren't ordered
+	if orderByTime {
+		sort.Slice(evs, func(i, j int) bool {
+			return evs[i].Depth() > evs[j].Depth()
+		})
+	}
+
+	stateForRooms := make(map[string][]gomatrixserverlib.ClientEvent)
+	for _, event := range evs {
+		eventsBefore, eventsAfter, err := contextEvents(ctx, syncDB, event, roomFilter, searchReq)
+		if err != nil {
+			logrus.WithError(err).Error("failed to get context events")
+			return jsonerror.InternalServerError()
+		}
+		startToken, endToken, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter)
+		if err != nil {
+			logrus.WithError(err).Error("failed to get start/end")
+			return jsonerror.InternalServerError()
+		}
+
+		profileInfos := make(map[string]ProfileInfo)
+		for _, ev := range append(eventsBefore, eventsAfter...) {
+			profile, ok := knownUsersProfiles[event.Sender()]
+			if !ok {
+				stateEvent, err := syncDB.GetStateEvent(ctx, ev.RoomID(), gomatrixserverlib.MRoomMember, ev.Sender())
+				if err != nil {
+					logrus.WithError(err).WithField("user_id", event.Sender()).Warn("failed to query userprofile")
+					continue
+				}
+				if stateEvent == nil {
+					continue
+				}
+				profile = ProfileInfo{
+					AvatarURL:   gjson.GetBytes(stateEvent.Content(), "avatar_url").Str,
+					DisplayName: gjson.GetBytes(stateEvent.Content(), "displayname").Str,
+				}
+				knownUsersProfiles[event.Sender()] = profile
+			}
+			profileInfos[ev.Sender()] = profile
+		}
+
+		results = append(results, Result{
+			Context: SearchContextResponse{
+				Start:        startToken.String(),
+				End:          endToken.String(),
+				EventsAfter:  gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatSync),
+				EventsBefore: gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatSync),
+				ProfileInfo:  profileInfos,
+			},
+			Rank:   eventScore[event.EventID()].Score,
+			Result: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll),
+		})
+		roomGroup := groups[event.RoomID()]
+		roomGroup.Results = append(roomGroup.Results, event.EventID())
+		groups[event.RoomID()] = roomGroup
+		if _, ok := stateForRooms[event.RoomID()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok {
+			stateFilter := gomatrixserverlib.DefaultStateFilter()
+			state, err := syncDB.CurrentState(ctx, event.RoomID(), &stateFilter, nil)
+			if err != nil {
+				logrus.WithError(err).Error("unable to get current state")
+				return jsonerror.InternalServerError()
+			}
+			stateForRooms[event.RoomID()] = gomatrixserverlib.HeaderedToClientEvents(state, gomatrixserverlib.FormatSync)
+		}
+	}
+
+	var nextBatchResult *string = nil
+	if int(result.Total) > nextBatch+len(results) {
+		nb := strconv.Itoa(len(results) + nextBatch)
+		nextBatchResult = &nb
+	} else if int(result.Total) == nextBatch+len(results) {
+		// Sytest expects a next_batch even if we don't actually have any more results
+		nb := ""
+		nextBatchResult = &nb
+	}
+
+	res := SearchResponse{
+		SearchCategories: SearchCategories{
+			RoomEvents: RoomEvents{
+				Count:      int(result.Total),
+				Groups:     Groups{RoomID: groups},
+				Results:    results,
+				NextBatch:  nextBatchResult,
+				Highlights: strings.Split(searchReq.SearchCategories.RoomEvents.SearchTerm, " "),
+				State:      stateForRooms,
+			},
+		},
+	}
+
+	logrus.Debugf("Full search request took %v", time.Since(start))
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
+
+// contextEvents returns the events around a given eventID
+func contextEvents(
+	ctx context.Context,
+	syncDB storage.Database,
+	event *gomatrixserverlib.HeaderedEvent,
+	roomFilter *gomatrixserverlib.RoomEventFilter,
+	searchReq SearchRequest,
+) ([]*gomatrixserverlib.HeaderedEvent, []*gomatrixserverlib.HeaderedEvent, error) {
+	id, _, err := syncDB.SelectContextEvent(ctx, event.RoomID(), event.EventID())
+	if err != nil {
+		logrus.WithError(err).Error("failed to query context event")
+		return nil, nil, err
+	}
+	roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.BeforeLimit
+	eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter)
+	if err != nil {
+		logrus.WithError(err).Error("failed to query before context event")
+		return nil, nil, err
+	}
+	roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.AfterLimit
+	_, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter)
+	if err != nil {
+		logrus.WithError(err).Error("failed to query after context event")
+		return nil, nil, err
+	}
+	return eventsBefore, eventsAfter, err
+}
+
+type SearchRequest struct {
+	SearchCategories struct {
+		RoomEvents struct {
+			EventContext struct {
+				AfterLimit     int  `json:"after_limit,omitempty"`
+				BeforeLimit    int  `json:"before_limit,omitempty"`
+				IncludeProfile bool `json:"include_profile,omitempty"`
+			} `json:"event_context"`
+			Filter    gomatrixserverlib.StateFilter `json:"filter"`
+			Groupings struct {
+				GroupBy []struct {
+					Key string `json:"key"`
+				} `json:"group_by"`
+			} `json:"groupings"`
+			IncludeState bool     `json:"include_state"`
+			Keys         []string `json:"keys"`
+			OrderBy      string   `json:"order_by"`
+			SearchTerm   string   `json:"search_term"`
+		} `json:"room_events"`
+	} `json:"search_categories"`
+}
+
+type SearchResponse struct {
+	SearchCategories SearchCategories `json:"search_categories"`
+}
+type RoomResult struct {
+	NextBatch *string  `json:"next_batch,omitempty"`
+	Order     int      `json:"order"`
+	Results   []string `json:"results"`
+}
+
+type Groups struct {
+	RoomID map[string]RoomResult `json:"room_id"`
+}
+
+type Result struct {
+	Context SearchContextResponse         `json:"context"`
+	Rank    float64                       `json:"rank"`
+	Result  gomatrixserverlib.ClientEvent `json:"result"`
+}
+
+type SearchContextResponse struct {
+	End          string                          `json:"end"`
+	EventsAfter  []gomatrixserverlib.ClientEvent `json:"events_after"`
+	EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before"`
+	Start        string                          `json:"start"`
+	ProfileInfo  map[string]ProfileInfo          `json:"profile_info"`
+}
+
+type ProfileInfo struct {
+	AvatarURL   string `json:"avatar_url"`
+	DisplayName string `json:"display_name"`
+}
+
+type RoomEvents struct {
+	Count      int                                        `json:"count"`
+	Groups     Groups                                     `json:"groups"`
+	Highlights []string                                   `json:"highlights"`
+	NextBatch  *string                                    `json:"next_batch,omitempty"`
+	Results    []Result                                   `json:"results"`
+	State      map[string][]gomatrixserverlib.ClientEvent `json:"state,omitempty"`
+}
+type SearchCategories struct {
+	RoomEvents RoomEvents `json:"room_events"`
+}
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index ad3be4206..dd03365e9 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -161,6 +161,7 @@ type Database interface {
 	// returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty
 	// string as the membership.
 	SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
+	ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error)
 }
 
 type Presence interface {
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 041f99061..20a9ea428 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -166,6 +166,8 @@ const selectContextAfterEventSQL = "" +
 	" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
 	" ORDER BY id ASC LIMIT $3"
 
+const selectSearchSQL = "SELECT id, event_id, headered_event_json FROM syncapi_output_room_events WHERE id > $1 AND type = ANY($2) ORDER BY id ASC LIMIT $3"
+
 type outputRoomEventsStatements struct {
 	insertEventStmt               *sql.Stmt
 	selectEventsStmt              *sql.Stmt
@@ -180,6 +182,7 @@ type outputRoomEventsStatements struct {
 	selectContextEventStmt        *sql.Stmt
 	selectContextBeforeEventStmt  *sql.Stmt
 	selectContextAfterEventStmt   *sql.Stmt
+	selectSearchStmt              *sql.Stmt
 }
 
 func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
@@ -215,6 +218,7 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
 		{&s.selectContextEventStmt, selectContextEventSQL},
 		{&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL},
 		{&s.selectContextAfterEventStmt, selectContextAfterEventSQL},
+		{&s.selectSearchStmt, selectSearchSQL},
 	}.Prepare(db)
 }
 
@@ -632,3 +636,27 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
 	}
 	return result, rows.Err()
 }
+
+func (s *outputRoomEventsStatements) ReIndex(ctx context.Context, txn *sql.Tx, limit, afterID int64, types []string) (map[int64]gomatrixserverlib.HeaderedEvent, error) {
+	rows, err := sqlutil.TxStmt(txn, s.selectSearchStmt).QueryContext(ctx, afterID, pq.StringArray(types), limit)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed")
+
+	var eventID string
+	var id int64
+	result := make(map[int64]gomatrixserverlib.HeaderedEvent)
+	for rows.Next() {
+		var ev gomatrixserverlib.HeaderedEvent
+		var eventBytes []byte
+		if err = rows.Scan(&id, &eventID, &eventBytes); err != nil {
+			return nil, err
+		}
+		if err = ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil {
+			return nil, err
+		}
+		result[id] = ev
+	}
+	return result, rows.Err()
+}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 215bad3a8..47e3a991c 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -1093,3 +1093,11 @@ func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.Stre
 func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) {
 	return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos)
 }
+
+func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{
+		gomatrixserverlib.MRoomName,
+		gomatrixserverlib.MRoomTopic,
+		"m.room.message",
+	})
+}
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 1626e32ef..6269f4fdf 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -115,6 +115,8 @@ const selectContextAfterEventSQL = "" +
 
 // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
 
+const selectSearchSQL = "SELECT id, event_id, headered_event_json FROM syncapi_output_room_events WHERE type IN ($1) AND id > $2 LIMIT $3 ORDER BY id ASC"
+
 type outputRoomEventsStatements struct {
 	db                           *sql.DB
 	streamIDStatements           *StreamIDStatements
@@ -125,6 +127,7 @@ type outputRoomEventsStatements struct {
 	selectContextEventStmt       *sql.Stmt
 	selectContextBeforeEventStmt *sql.Stmt
 	selectContextAfterEventStmt  *sql.Stmt
+	//selectSearchStmt             *sql.Stmt - prepared at runtime
 }
 
 func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Events, error) {
@@ -157,6 +160,7 @@ func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Even
 		{&s.selectContextEventStmt, selectContextEventSQL},
 		{&s.selectContextBeforeEventStmt, selectContextBeforeEventSQL},
 		{&s.selectContextAfterEventStmt, selectContextAfterEventSQL},
+		//{&s.selectSearchStmt, selectSearchSQL}, - prepared at runtime
 	}.Prepare(db)
 }
 
@@ -628,3 +632,40 @@ func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs [
 	}
 	return
 }
+
+func (s *outputRoomEventsStatements) ReIndex(ctx context.Context, txn *sql.Tx, limit, afterID int64, types []string) (map[int64]gomatrixserverlib.HeaderedEvent, error) {
+	params := make([]interface{}, len(types))
+	for i := range types {
+		params[i] = types[i]
+	}
+	params = append(params, afterID)
+	params = append(params, limit)
+	selectSQL := strings.Replace(selectSearchSQL, "($1)", sqlutil.QueryVariadic(len(types)), 1)
+
+	stmt, err := s.db.Prepare(selectSQL)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, stmt, "selectEvents: stmt.close() failed")
+	rows, err := stmt.QueryContext(ctx, params...)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "rows.close() failed")
+
+	var eventID string
+	var id int64
+	result := make(map[int64]gomatrixserverlib.HeaderedEvent)
+	for rows.Next() {
+		var ev gomatrixserverlib.HeaderedEvent
+		var eventBytes []byte
+		if err = rows.Scan(&id, &eventID, &eventBytes); err != nil {
+			return nil, err
+		}
+		if err = ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil {
+			return nil, err
+		}
+		result[id] = ev
+	}
+	return result, rows.Err()
+}
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 9a873c2ed..2a6d6fa82 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -75,6 +75,7 @@ type Events interface {
 	SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error)
 	SelectContextBeforeEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error)
 	SelectContextAfterEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error)
+	ReIndex(ctx context.Context, txn *sql.Tx, limit, offset int64, types []string) (map[int64]gomatrixserverlib.HeaderedEvent, error)
 }
 
 // Topology keeps track of the depths and stream positions for all events.
diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go
index f5d00f367..be19310f2 100644
--- a/syncapi/syncapi.go
+++ b/syncapi/syncapi.go
@@ -88,14 +88,15 @@ func AddPublicRoutes(
 
 	roomConsumer := consumers.NewOutputRoomEventConsumer(
 		base.ProcessContext, cfg, js, syncDB, notifier, streams.PDUStreamProvider,
-		streams.InviteStreamProvider, rsAPI,
+		streams.InviteStreamProvider, rsAPI, base.Fulltext,
 	)
 	if err = roomConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start room server consumer")
 	}
 
 	clientConsumer := consumers.NewOutputClientDataConsumer(
-		base.ProcessContext, cfg, js, syncDB, notifier, streams.AccountDataStreamProvider,
+		base.ProcessContext, cfg, js, natsClient, syncDB, notifier,
+		streams.AccountDataStreamProvider, base.Fulltext,
 	)
 	if err = clientConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start client data consumer")
@@ -131,6 +132,6 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		base.PublicClientAPIMux, requestPool, syncDB, userAPI,
-		rsAPI, cfg, base.Caches,
+		rsAPI, cfg, base.Caches, base.Fulltext,
 	)
 }

From 34993717fd702db50a82858a3ad4b660f0c7feac Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 27 Sep 2022 17:10:47 +0100
Subject: [PATCH 03/73] Update search docs

---
 cmd/generate-config/main.go          |  2 +-
 dendrite-sample.monolith.yaml        | 14 +++++++++-----
 dendrite-sample.polylith.yaml        | 14 +++++++++-----
 docs/installation/7_configuration.md |  8 +++++---
 setup/config/config_syncapi.go       |  2 +-
 5 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go
index 8b042c56e..33b18c471 100644
--- a/cmd/generate-config/main.go
+++ b/cmd/generate-config/main.go
@@ -85,7 +85,7 @@ func main() {
 			}
 			cfg.SyncAPI.Fulltext = config.Fulltext{
 				Enabled:   true,
-				IndexPath: config.Path(filepath.Join(*dirPath, "fulltextindex")),
+				IndexPath: config.Path(filepath.Join(*dirPath, "searchindex")),
 				InMemory:  true,
 				Language:  "en",
 			}
diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index 3cad17da8..e41e83d7c 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -276,13 +276,17 @@ sync_api:
   # a reverse proxy server.
   # real_ip_header: X-Real-IP
 
-  # Configuration for the fulltext search
+  # Configuration for the full-text search engine.
   search:
+    # Whether or not search is enabled.
     enabled: false
-    # The path where the fulltext index will be created in.
-    index_path: "./fulltextindex"
-    # The language most likely to be used on the server - used when indexing, to ensure the returned results match the expectations.
-    # A full list of possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
+
+    # The path where the search index will be created in.
+    index_path: "./searchindex"
+
+    # The language most likely to be used on the server - used when indexing, to
+    # ensure the returned results match expectations. A full list of possible languages
+    # can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
     language: "en"
 
 # Configuration for the User API.
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index e58062fe2..0ae4cc8fb 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -327,13 +327,17 @@ sync_api:
     max_idle_conns: 2
     conn_max_lifetime: -1
 
-  # Configuration for the fulltext search
+  # Configuration for the full-text search engine.
   search:
+    # Whether or not search is enabled.
     enabled: false
-    # The path where the fulltext index will be created in.
-    index_path: "./fulltextindex"
-    # The language most likely to be used on the server - used when indexing, to ensure the returned results match the expectations.
-    # A full list of possible languages can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
+
+    # The path where the search index will be created in.
+    index_path: "./searchindex"
+
+    # The language most likely to be used on the server - used when indexing, to
+    # ensure the returned results match expectations. A full list of possible languages
+    # can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
     language: "en"
 
   # This option controls which HTTP header to inspect to find the real remote IP
diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md
index 67cd339cf..19958c92f 100644
--- a/docs/installation/7_configuration.md
+++ b/docs/installation/7_configuration.md
@@ -138,16 +138,18 @@ room_server:
     conn_max_lifetime: -1
 ```
 
-## Fulltext search
+## Full-text search
 
-Dendrite supports experimental fulltext indexing using [Bleve](https://github.com/blevesearch/bleve), it is configured in the `sync_api` section as follows. Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, to ensure the returned results match the expectations. A full list of possible languages can be found [here](https://github.com/blevesearch/bleve/tree/master/analysis/lang).
+Dendrite supports experimental full-text indexing using [Bleve](https://github.com/blevesearch/bleve). It is configured in the `sync_api` section as follows.
+
+Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, to ensure the returned results match the expectations. A full list of possible languages can be found [here](https://github.com/blevesearch/bleve/tree/master/analysis/lang).
 
 ```yaml
 sync_api:
   # ...
   search:
     enabled: false
-    index_path: "./fulltextindex"
+    index_path: "./searchindex"
     language: "en"
 ```
 
diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go
index edef22c93..a87da3732 100644
--- a/setup/config/config_syncapi.go
+++ b/setup/config/config_syncapi.go
@@ -50,7 +50,7 @@ type Fulltext struct {
 
 func (f *Fulltext) Defaults(opts DefaultOpts) {
 	f.Enabled = false
-	f.IndexPath = "./fulltextindex"
+	f.IndexPath = "./searchindex"
 	f.Language = "en"
 }
 

From 083ae01520afadfacc6f0ea4bdd501d41f0d832b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 27 Sep 2022 17:30:40 +0100
Subject: [PATCH 04/73] Promote reindexing log level

---
 syncapi/consumers/clientapi.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go
index b11ed4f5e..796cc61e1 100644
--- a/syncapi/consumers/clientapi.go
+++ b/syncapi/consumers/clientapi.go
@@ -91,7 +91,7 @@ func (s *OutputClientDataConsumer) Start() error {
 			return
 		}
 		ctx := context.Background()
-		logrus.Debugf("Starting to index events")
+		logrus.Infof("Starting to index events")
 		var offset int
 		start := time.Now()
 		count := 0
@@ -140,7 +140,7 @@ func (s *OutputClientDataConsumer) Start() error {
 			offset += len(evs)
 			count += len(elements)
 		}
-		logrus.Debugf("Indexed %d events in %v", count, time.Since(start))
+		logrus.Infof("Indexed %d events in %v", count, time.Since(start))
 	})
 	if err != nil {
 		return err

From a574ed53696c06e6be6dbe313af0caaa56a659ec Mon Sep 17 00:00:00 2001
From: texuf <texuf.eth@gmail.com>
Date: Tue, 27 Sep 2022 21:19:34 -0700
Subject: [PATCH 05/73] =?UTF-8?q?Fix=20for=20`sql:=20converting=20argument?=
 =?UTF-8?q?=20$1=20type:=20unsupported=20type=20[]interfa=E2=80=A6=20(#274?=
 =?UTF-8?q?3)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ce {}, a slice of interface` in new notifications select

The sqlite3 version was just not working, original pr here:
https://github.com/matrix-org/dendrite/pull/2688

signed off by: austin ellis <austin@hntlabs.com>

This doesn't fix the notification counts, they still only work about 1
out of every 5 times in my tests. I will stick with my other fix locally
for reliable notification delivery:
https://github.com/matrix-org/dendrite/pull/2701
---
 syncapi/storage/sqlite3/notification_data_table.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/syncapi/storage/sqlite3/notification_data_table.go b/syncapi/storage/sqlite3/notification_data_table.go
index ceff60555..a690ffad6 100644
--- a/syncapi/storage/sqlite3/notification_data_table.go
+++ b/syncapi/storage/sqlite3/notification_data_table.go
@@ -90,8 +90,8 @@ func (r *notificationDataStatements) SelectUserUnreadCountsForRooms(
 	for i := range roomIDs {
 		params[i+1] = roomIDs[i]
 	}
-	sql := strings.Replace(selectUserUnreadNotificationsForRooms, "($1)", sqlutil.QueryVariadic(len(params)), 1)
-	rows, err := r.db.QueryContext(ctx, sql, params)
+	sql := strings.Replace(selectUserUnreadNotificationsForRooms, "($2)", sqlutil.QueryVariadicOffset(len(roomIDs), 1), 1)
+	rows, err := r.db.QueryContext(ctx, sql, params...)
 	if err != nil {
 		return nil, err
 	}

From 3f9e38e80a7be356aaf1294038888df27e0697a8 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 28 Sep 2022 10:18:03 +0100
Subject: [PATCH 06/73] Consistent `*sql.Tx` usage across sync API (#2744)

This tidies up the `storage` package so that everything takes a
transaction parameter instead of something things that do and some that
don't.
---
 .../storage/postgres/account_data_table.go    |  5 ++--
 .../postgres/backwards_extremities_table.go   |  4 +--
 .../postgres/current_room_state_table.go      | 12 ++++-----
 syncapi/storage/postgres/filter_table.go      | 14 +++++-----
 syncapi/storage/postgres/invites_table.go     |  2 +-
 .../postgres/output_room_events_table.go      |  4 +--
 .../output_room_events_topology_table.go      |  8 +++---
 syncapi/storage/postgres/peeks_table.go       |  4 +--
 syncapi/storage/postgres/receipt_table.go     |  4 +--
 syncapi/storage/shared/syncserver.go          | 26 +++++++++----------
 syncapi/storage/sqlite3/account_data_table.go |  4 +--
 .../sqlite3/backwards_extremities_table.go    |  4 +--
 .../sqlite3/current_room_state_table.go       | 21 ++++++++++-----
 syncapi/storage/sqlite3/filter_table.go       | 14 +++++-----
 .../sqlite3/notification_data_table.go        |  7 ++++-
 .../sqlite3/output_room_events_table.go       |  6 ++---
 .../output_room_events_topology_table.go      |  4 +--
 syncapi/storage/sqlite3/peeks_table.go        |  4 +--
 syncapi/storage/sqlite3/receipt_table.go      |  9 +++++--
 syncapi/storage/tables/interface.go           | 20 +++++++-------
 20 files changed, 99 insertions(+), 77 deletions(-)

diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go
index e9c72058b..aa54cb08f 100644
--- a/syncapi/storage/postgres/account_data_table.go
+++ b/syncapi/storage/postgres/account_data_table.go
@@ -99,14 +99,15 @@ func (s *accountDataStatements) InsertAccountData(
 }
 
 func (s *accountDataStatements) SelectAccountDataInRange(
-	ctx context.Context,
+	ctx context.Context, txn *sql.Tx,
 	userID string,
 	r types.Range,
 	accountDataEventFilter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
 	data = make(map[string][]string)
 
-	rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(),
+	rows, err := sqlutil.TxStmt(txn, s.selectAccountDataInRangeStmt).QueryContext(
+		ctx, userID, r.Low(), r.High(),
 		pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)),
 		pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.NotTypes)),
 		accountDataEventFilter.Limit,
diff --git a/syncapi/storage/postgres/backwards_extremities_table.go b/syncapi/storage/postgres/backwards_extremities_table.go
index d4515735c..8fc92091f 100644
--- a/syncapi/storage/postgres/backwards_extremities_table.go
+++ b/syncapi/storage/postgres/backwards_extremities_table.go
@@ -79,9 +79,9 @@ func (s *backwardExtremitiesStatements) InsertsBackwardExtremity(
 }
 
 func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom(
-	ctx context.Context, roomID string,
+	ctx context.Context, txn *sql.Tx, roomID string,
 ) (bwExtrems map[string][]string, err error) {
-	rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID)
+	rows, err := sqlutil.TxStmt(txn, s.selectBackwardExtremitiesForRoomStmt).QueryContext(ctx, roomID)
 	if err != nil {
 		return
 	}
diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 5e6daaaf8..4ffd29610 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -185,9 +185,9 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
 
 // SelectJoinedUsers returns a map of room ID to a list of joined user IDs.
 func (s *currentRoomStateStatements) SelectJoinedUsers(
-	ctx context.Context,
+	ctx context.Context, txn *sql.Tx,
 ) (map[string][]string, error) {
-	rows, err := s.selectJoinedUsersStmt.QueryContext(ctx)
+	rows, err := sqlutil.TxStmt(txn, s.selectJoinedUsersStmt).QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -209,9 +209,9 @@ func (s *currentRoomStateStatements) SelectJoinedUsers(
 
 // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
 func (s *currentRoomStateStatements) SelectJoinedUsersInRoom(
-	ctx context.Context, roomIDs []string,
+	ctx context.Context, txn *sql.Tx, roomIDs []string,
 ) (map[string][]string, error) {
-	rows, err := s.selectJoinedUsersInRoomStmt.QueryContext(ctx, pq.StringArray(roomIDs))
+	rows, err := sqlutil.TxStmt(txn, s.selectJoinedUsersInRoomStmt).QueryContext(ctx, pq.StringArray(roomIDs))
 	if err != nil {
 		return nil, err
 	}
@@ -387,9 +387,9 @@ func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) {
 }
 
 func (s *currentRoomStateStatements) SelectStateEvent(
-	ctx context.Context, roomID, evType, stateKey string,
+	ctx context.Context, txn *sql.Tx, roomID, evType, stateKey string,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
-	stmt := s.selectStateEventStmt
+	stmt := sqlutil.TxStmt(txn, s.selectStateEventStmt)
 	var res []byte
 	err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res)
 	if err == sql.ErrNoRows {
diff --git a/syncapi/storage/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go
index c82ef092f..86cec3625 100644
--- a/syncapi/storage/postgres/filter_table.go
+++ b/syncapi/storage/postgres/filter_table.go
@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -73,11 +74,11 @@ func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) {
 }
 
 func (s *filterStatements) SelectFilter(
-	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
+	ctx context.Context, txn *sql.Tx, target *gomatrixserverlib.Filter, localpart string, filterID string,
 ) error {
 	// Retrieve filter from database (stored as canonical JSON)
 	var filterData []byte
-	err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
+	err := sqlutil.TxStmt(txn, s.selectFilterStmt).QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
 	if err != nil {
 		return err
 	}
@@ -90,7 +91,7 @@ func (s *filterStatements) SelectFilter(
 }
 
 func (s *filterStatements) InsertFilter(
-	ctx context.Context, filter *gomatrixserverlib.Filter, localpart string,
+	ctx context.Context, txn *sql.Tx, filter *gomatrixserverlib.Filter, localpart string,
 ) (filterID string, err error) {
 	var existingFilterID string
 
@@ -111,8 +112,9 @@ func (s *filterStatements) InsertFilter(
 	// This can result in a race condition when two clients try to insert the
 	// same filter and localpart at the same time, however this is not a
 	// problem as both calls will result in the same filterID
-	err = s.selectFilterIDByContentStmt.QueryRowContext(ctx,
-		localpart, filterJSON).Scan(&existingFilterID)
+	err = sqlutil.TxStmt(txn, s.selectFilterIDByContentStmt).QueryRowContext(
+		ctx, localpart, filterJSON,
+	).Scan(&existingFilterID)
 	if err != nil && err != sql.ErrNoRows {
 		return "", err
 	}
@@ -122,7 +124,7 @@ func (s *filterStatements) InsertFilter(
 	}
 
 	// Otherwise insert the filter and return the new ID
-	err = s.insertFilterStmt.QueryRowContext(ctx, filterJSON, localpart).
+	err = sqlutil.TxStmt(txn, s.insertFilterStmt).QueryRowContext(ctx, filterJSON, localpart).
 		Scan(&filterID)
 	return
 }
diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go
index 97001ae2c..f87ccf965 100644
--- a/syncapi/storage/postgres/invites_table.go
+++ b/syncapi/storage/postgres/invites_table.go
@@ -99,7 +99,7 @@ func (s *inviteEventsStatements) InsertInviteEvent(
 		return
 	}
 
-	err = s.insertInviteEventStmt.QueryRowContext(
+	err = sqlutil.TxStmt(txn, s.insertInviteEventStmt).QueryRowContext(
 		ctx,
 		inviteEvent.RoomID(),
 		inviteEvent.EventID(),
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index 20a9ea428..cb092150d 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -222,12 +222,12 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) {
 	}.Prepare(db)
 }
 
-func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
+func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent) error {
 	headeredJSON, err := json.Marshal(event)
 	if err != nil {
 		return err
 	}
-	_, err = s.updateEventJSONStmt.ExecContext(ctx, headeredJSON, event.EventID())
+	_, err = sqlutil.TxStmt(txn, s.updateEventJSONStmt).ExecContext(ctx, headeredJSON, event.EventID())
 	return err
 }
 
diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go
index a1fc9b2a3..6fab900eb 100644
--- a/syncapi/storage/postgres/output_room_events_topology_table.go
+++ b/syncapi/storage/postgres/output_room_events_topology_table.go
@@ -173,7 +173,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange(
 func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology(
 	ctx context.Context, txn *sql.Tx, eventID string,
 ) (pos, spos types.StreamPosition, err error) {
-	err = s.selectPositionInTopologyStmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos)
+	err = sqlutil.TxStmt(txn, s.selectPositionInTopologyStmt).QueryRowContext(ctx, eventID).Scan(&pos, &spos)
 	return
 }
 
@@ -183,9 +183,9 @@ func (s *outputRoomEventsTopologyStatements) SelectStreamToTopologicalPosition(
 	ctx context.Context, txn *sql.Tx, roomID string, streamPos types.StreamPosition, backwardOrdering bool,
 ) (topoPos types.StreamPosition, err error) {
 	if backwardOrdering {
-		err = s.selectStreamToTopologicalPositionDescStmt.QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
+		err = sqlutil.TxStmt(txn, s.selectStreamToTopologicalPositionDescStmt).QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
 	} else {
-		err = s.selectStreamToTopologicalPositionAscStmt.QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
+		err = sqlutil.TxStmt(txn, s.selectStreamToTopologicalPositionAscStmt).QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
 	}
 	return
 }
@@ -193,6 +193,6 @@ func (s *outputRoomEventsTopologyStatements) SelectStreamToTopologicalPosition(
 func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology(
 	ctx context.Context, txn *sql.Tx, roomID string,
 ) (pos types.StreamPosition, spos types.StreamPosition, err error) {
-	err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos)
+	err = sqlutil.TxStmt(txn, s.selectMaxPositionInTopologyStmt).QueryRowContext(ctx, roomID).Scan(&pos, &spos)
 	return
 }
diff --git a/syncapi/storage/postgres/peeks_table.go b/syncapi/storage/postgres/peeks_table.go
index 75eeac986..e20a4882f 100644
--- a/syncapi/storage/postgres/peeks_table.go
+++ b/syncapi/storage/postgres/peeks_table.go
@@ -152,9 +152,9 @@ func (s *peekStatements) SelectPeeksInRange(
 }
 
 func (s *peekStatements) SelectPeekingDevices(
-	ctx context.Context,
+	ctx context.Context, txn *sql.Tx,
 ) (peekingDevices map[string][]types.PeekingDevice, err error) {
-	rows, err := s.selectPeekingDevicesStmt.QueryContext(ctx)
+	rows, err := sqlutil.TxStmt(txn, s.selectPeekingDevicesStmt).QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/postgres/receipt_table.go b/syncapi/storage/postgres/receipt_table.go
index bbddaa939..327a7a372 100644
--- a/syncapi/storage/postgres/receipt_table.go
+++ b/syncapi/storage/postgres/receipt_table.go
@@ -104,9 +104,9 @@ func (r *receiptStatements) UpsertReceipt(ctx context.Context, txn *sql.Tx, room
 	return
 }
 
-func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) {
+func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, txn *sql.Tx, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) {
 	var lastPos types.StreamPosition
-	rows, err := r.selectRoomReceipts.QueryContext(ctx, pq.Array(roomIDs), streamPos)
+	rows, err := sqlutil.TxStmt(txn, r.selectRoomReceipts).QueryContext(ctx, pq.Array(roomIDs), streamPos)
 	if err != nil {
 		return 0, nil, fmt.Errorf("unable to query room receipts: %w", err)
 	}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 47e3a991c..a05e68804 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -148,7 +148,7 @@ func (d *Database) PeeksInRange(ctx context.Context, userID, deviceID string, r
 }
 
 func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) {
-	return d.Receipts.SelectRoomReceiptsAfter(ctx, roomIDs, streamPos)
+	return d.Receipts.SelectRoomReceiptsAfter(ctx, nil, roomIDs, streamPos)
 }
 
 // Events lookups a list of event by their event ID.
@@ -168,15 +168,15 @@ func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixse
 }
 
 func (d *Database) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) {
-	return d.CurrentRoomState.SelectJoinedUsers(ctx)
+	return d.CurrentRoomState.SelectJoinedUsers(ctx, nil)
 }
 
 func (d *Database) AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) {
-	return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, roomIDs)
+	return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, nil, roomIDs)
 }
 
 func (d *Database) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) {
-	return d.Peeks.SelectPeekingDevices(ctx)
+	return d.Peeks.SelectPeekingDevices(ctx, nil)
 }
 
 func (d *Database) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) {
@@ -186,7 +186,7 @@ func (d *Database) SharedUsers(ctx context.Context, userID string, otherUserIDs
 func (d *Database) GetStateEvent(
 	ctx context.Context, roomID, evType, stateKey string,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
-	return d.CurrentRoomState.SelectStateEvent(ctx, roomID, evType, stateKey)
+	return d.CurrentRoomState.SelectStateEvent(ctx, nil, roomID, evType, stateKey)
 }
 
 func (d *Database) GetStateEventsForRoom(
@@ -277,7 +277,7 @@ func (d *Database) GetAccountDataInRange(
 	ctx context.Context, userID string, r types.Range,
 	accountDataFilterPart *gomatrixserverlib.EventFilter,
 ) (map[string][]string, types.StreamPosition, error) {
-	return d.AccountData.SelectAccountDataInRange(ctx, userID, r, accountDataFilterPart)
+	return d.AccountData.SelectAccountDataInRange(ctx, nil, userID, r, accountDataFilterPart)
 }
 
 // UpsertAccountData keeps track of new or updated account data, by saving the type
@@ -484,7 +484,7 @@ func (d *Database) GetEventsInTopologicalRange(
 func (d *Database) BackwardExtremitiesForRoom(
 	ctx context.Context, roomID string,
 ) (backwardExtremities map[string][]string, err error) {
-	return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID)
+	return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, nil, roomID)
 }
 
 func (d *Database) MaxTopologicalPosition(
@@ -530,7 +530,7 @@ func (d *Database) StreamToTopologicalPosition(
 func (d *Database) GetFilter(
 	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
 ) error {
-	return d.Filter.SelectFilter(ctx, target, localpart, filterID)
+	return d.Filter.SelectFilter(ctx, nil, target, localpart, filterID)
 }
 
 func (d *Database) PutFilter(
@@ -538,8 +538,8 @@ func (d *Database) PutFilter(
 ) (string, error) {
 	var filterID string
 	var err error
-	err = d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
-		filterID, err = d.Filter.InsertFilter(ctx, filter, localpart)
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		filterID, err = d.Filter.InsertFilter(ctx, txn, filter, localpart)
 		return err
 	})
 	return filterID, err
@@ -561,8 +561,8 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda
 	}
 
 	newEvent := eventToRedact.Headered(redactedBecause.RoomVersion)
-	err = d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
-		return d.OutputEvents.UpdateEventJSON(ctx, newEvent)
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.OutputEvents.UpdateEventJSON(ctx, txn, newEvent)
 	})
 	return err
 }
@@ -1024,7 +1024,7 @@ func (d *Database) StoreReceipt(ctx context.Context, roomId, receiptType, userId
 }
 
 func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) {
-	_, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, roomIDs, streamPos)
+	_, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, nil, roomIDs, streamPos)
 	return receipts, err
 }
 
diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go
index 21a16dcd3..d8967113a 100644
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@@ -91,14 +91,14 @@ func (s *accountDataStatements) InsertAccountData(
 }
 
 func (s *accountDataStatements) SelectAccountDataInRange(
-	ctx context.Context,
+	ctx context.Context, txn *sql.Tx,
 	userID string,
 	r types.Range,
 	filter *gomatrixserverlib.EventFilter,
 ) (data map[string][]string, pos types.StreamPosition, err error) {
 	data = make(map[string][]string)
 	stmt, params, err := prepareWithFilters(
-		s.db, nil, selectAccountDataInRangeSQL,
+		s.db, txn, selectAccountDataInRangeSQL,
 		[]interface{}{
 			userID, r.Low(), r.High(),
 		},
diff --git a/syncapi/storage/sqlite3/backwards_extremities_table.go b/syncapi/storage/sqlite3/backwards_extremities_table.go
index c5674dded..3a5fd6be3 100644
--- a/syncapi/storage/sqlite3/backwards_extremities_table.go
+++ b/syncapi/storage/sqlite3/backwards_extremities_table.go
@@ -82,9 +82,9 @@ func (s *backwardExtremitiesStatements) InsertsBackwardExtremity(
 }
 
 func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom(
-	ctx context.Context, roomID string,
+	ctx context.Context, txn *sql.Tx, roomID string,
 ) (bwExtrems map[string][]string, err error) {
-	rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID)
+	rows, err := sqlutil.TxStmt(txn, s.selectBackwardExtremitiesForRoomStmt).QueryContext(ctx, roomID)
 	if err != nil {
 		return
 	}
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index bd1271dd6..ba6d8126c 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -163,9 +163,9 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (t
 
 // SelectJoinedUsers returns a map of room ID to a list of joined user IDs.
 func (s *currentRoomStateStatements) SelectJoinedUsers(
-	ctx context.Context,
+	ctx context.Context, txn *sql.Tx,
 ) (map[string][]string, error) {
-	rows, err := s.selectJoinedUsersStmt.QueryContext(ctx)
+	rows, err := sqlutil.TxStmt(txn, s.selectJoinedUsersStmt).QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -187,7 +187,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers(
 
 // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
 func (s *currentRoomStateStatements) SelectJoinedUsersInRoom(
-	ctx context.Context, roomIDs []string,
+	ctx context.Context, txn *sql.Tx, roomIDs []string,
 ) (map[string][]string, error) {
 	query := strings.Replace(selectJoinedUsersInRoomSQL, "($1)", sqlutil.QueryVariadic(len(roomIDs)), 1)
 	params := make([]interface{}, 0, len(roomIDs))
@@ -200,7 +200,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsersInRoom(
 	}
 	defer internal.CloseAndLogIfError(ctx, stmt, "SelectJoinedUsersInRoom: stmt.close() failed")
 
-	rows, err := stmt.QueryContext(ctx, params...)
+	rows, err := sqlutil.TxStmt(txn, stmt).QueryContext(ctx, params...)
 	if err != nil {
 		return nil, err
 	}
@@ -401,9 +401,9 @@ func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) {
 }
 
 func (s *currentRoomStateStatements) SelectStateEvent(
-	ctx context.Context, roomID, evType, stateKey string,
+	ctx context.Context, txn *sql.Tx, roomID, evType, stateKey string,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
-	stmt := s.selectStateEventStmt
+	stmt := sqlutil.TxStmt(txn, s.selectStateEventStmt)
 	var res []byte
 	err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res)
 	if err == sql.ErrNoRows {
@@ -429,10 +429,17 @@ func (s *currentRoomStateStatements) SelectSharedUsers(
 		params[k+1] = v
 	}
 
+	var provider sqlutil.QueryProvider
+	if txn == nil {
+		provider = s.db
+	} else {
+		provider = txn
+	}
+
 	result := make([]string, 0, len(otherUserIDs))
 	query := strings.Replace(selectSharedUsersSQL, "($2)", sqlutil.QueryVariadicOffset(len(otherUserIDs), 1), 1)
 	err := sqlutil.RunLimitedVariablesQuery(
-		ctx, query, s.db, params, sqlutil.SQLite3MaxVariables,
+		ctx, query, provider, params, sqlutil.SQLite3MaxVariables,
 		func(rows *sql.Rows) error {
 			var stateKey string
 			for rows.Next() {
diff --git a/syncapi/storage/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go
index 6081a48b1..5f1e980eb 100644
--- a/syncapi/storage/sqlite3/filter_table.go
+++ b/syncapi/storage/sqlite3/filter_table.go
@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -77,11 +78,11 @@ func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) {
 }
 
 func (s *filterStatements) SelectFilter(
-	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
+	ctx context.Context, txn *sql.Tx, target *gomatrixserverlib.Filter, localpart string, filterID string,
 ) error {
 	// Retrieve filter from database (stored as canonical JSON)
 	var filterData []byte
-	err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
+	err := sqlutil.TxStmt(txn, s.selectFilterStmt).QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
 	if err != nil {
 		return err
 	}
@@ -94,7 +95,7 @@ func (s *filterStatements) SelectFilter(
 }
 
 func (s *filterStatements) InsertFilter(
-	ctx context.Context, filter *gomatrixserverlib.Filter, localpart string,
+	ctx context.Context, txn *sql.Tx, filter *gomatrixserverlib.Filter, localpart string,
 ) (filterID string, err error) {
 	var existingFilterID string
 
@@ -115,8 +116,9 @@ func (s *filterStatements) InsertFilter(
 	// This can result in a race condition when two clients try to insert the
 	// same filter and localpart at the same time, however this is not a
 	// problem as both calls will result in the same filterID
-	err = s.selectFilterIDByContentStmt.QueryRowContext(ctx,
-		localpart, filterJSON).Scan(&existingFilterID)
+	err = sqlutil.TxStmt(txn, s.selectFilterIDByContentStmt).QueryRowContext(
+		ctx, localpart, filterJSON,
+	).Scan(&existingFilterID)
 	if err != nil && err != sql.ErrNoRows {
 		return "", err
 	}
@@ -126,7 +128,7 @@ func (s *filterStatements) InsertFilter(
 	}
 
 	// Otherwise insert the filter and return the new ID
-	res, err := s.insertFilterStmt.ExecContext(ctx, filterJSON, localpart)
+	res, err := sqlutil.TxStmt(txn, s.insertFilterStmt).ExecContext(ctx, filterJSON, localpart)
 	if err != nil {
 		return "", err
 	}
diff --git a/syncapi/storage/sqlite3/notification_data_table.go b/syncapi/storage/sqlite3/notification_data_table.go
index a690ffad6..6242898e1 100644
--- a/syncapi/storage/sqlite3/notification_data_table.go
+++ b/syncapi/storage/sqlite3/notification_data_table.go
@@ -91,7 +91,12 @@ func (r *notificationDataStatements) SelectUserUnreadCountsForRooms(
 		params[i+1] = roomIDs[i]
 	}
 	sql := strings.Replace(selectUserUnreadNotificationsForRooms, "($2)", sqlutil.QueryVariadicOffset(len(roomIDs), 1), 1)
-	rows, err := r.db.QueryContext(ctx, sql, params...)
+	prep, err := r.db.PrepareContext(ctx, sql)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, prep, "SelectUserUnreadCountsForRooms: prep.close() failed")
+	rows, err := sqlutil.TxStmt(txn, prep).QueryContext(ctx, params...)
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 6269f4fdf..165943027 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -164,12 +164,12 @@ func NewSqliteEventsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Even
 	}.Prepare(db)
 }
 
-func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
+func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent) error {
 	headeredJSON, err := json.Marshal(event)
 	if err != nil {
 		return err
 	}
-	_, err = s.updateEventJSONStmt.ExecContext(ctx, headeredJSON, event.EventID())
+	_, err = sqlutil.TxStmt(txn, s.updateEventJSONStmt).ExecContext(ctx, headeredJSON, event.EventID())
 	return err
 }
 
@@ -647,7 +647,7 @@ func (s *outputRoomEventsStatements) ReIndex(ctx context.Context, txn *sql.Tx, l
 		return nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, stmt, "selectEvents: stmt.close() failed")
-	rows, err := stmt.QueryContext(ctx, params...)
+	rows, err := sqlutil.TxStmt(txn, stmt).QueryContext(ctx, params...)
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go
index b2fb77417..81b264988 100644
--- a/syncapi/storage/sqlite3/output_room_events_topology_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go
@@ -176,9 +176,9 @@ func (s *outputRoomEventsTopologyStatements) SelectStreamToTopologicalPosition(
 	ctx context.Context, txn *sql.Tx, roomID string, streamPos types.StreamPosition, backwardOrdering bool,
 ) (topoPos types.StreamPosition, err error) {
 	if backwardOrdering {
-		err = s.selectStreamToTopologicalPositionDescStmt.QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
+		err = sqlutil.TxStmt(txn, s.selectStreamToTopologicalPositionDescStmt).QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
 	} else {
-		err = s.selectStreamToTopologicalPositionAscStmt.QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
+		err = sqlutil.TxStmt(txn, s.selectStreamToTopologicalPositionAscStmt).QueryRowContext(ctx, roomID, streamPos).Scan(&topoPos)
 	}
 	return
 }
diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go
index 5ee86448c..4ef51b103 100644
--- a/syncapi/storage/sqlite3/peeks_table.go
+++ b/syncapi/storage/sqlite3/peeks_table.go
@@ -172,9 +172,9 @@ func (s *peekStatements) SelectPeeksInRange(
 }
 
 func (s *peekStatements) SelectPeekingDevices(
-	ctx context.Context,
+	ctx context.Context, txn *sql.Tx,
 ) (peekingDevices map[string][]types.PeekingDevice, err error) {
-	rows, err := s.selectPeekingDevicesStmt.QueryContext(ctx)
+	rows, err := sqlutil.TxStmt(txn, s.selectPeekingDevicesStmt).QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
diff --git a/syncapi/storage/sqlite3/receipt_table.go b/syncapi/storage/sqlite3/receipt_table.go
index 31adb005b..a4a9b4395 100644
--- a/syncapi/storage/sqlite3/receipt_table.go
+++ b/syncapi/storage/sqlite3/receipt_table.go
@@ -108,7 +108,7 @@ func (r *receiptStatements) UpsertReceipt(ctx context.Context, txn *sql.Tx, room
 }
 
 // SelectRoomReceiptsAfter select all receipts for a given room after a specific timestamp
-func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) {
+func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, txn *sql.Tx, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) {
 	selectSQL := strings.Replace(selectRoomReceipts, "($2)", sqlutil.QueryVariadicOffset(len(roomIDs), 1), 1)
 	var lastPos types.StreamPosition
 	params := make([]interface{}, len(roomIDs)+1)
@@ -116,7 +116,12 @@ func (r *receiptStatements) SelectRoomReceiptsAfter(ctx context.Context, roomIDs
 	for k, v := range roomIDs {
 		params[k+1] = v
 	}
-	rows, err := r.db.QueryContext(ctx, selectSQL, params...)
+	prep, err := r.db.Prepare(selectSQL)
+	if err != nil {
+		return 0, nil, fmt.Errorf("unable to prepare statement: %w", err)
+	}
+	defer internal.CloseAndLogIfError(ctx, prep, "SelectRoomReceiptsAfter: prep.close() failed")
+	rows, err := sqlutil.TxStmt(txn, prep).QueryContext(ctx, params...)
 	if err != nil {
 		return 0, nil, fmt.Errorf("unable to query room receipts: %w", err)
 	}
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 2a6d6fa82..89cb537af 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -28,7 +28,7 @@ import (
 type AccountData interface {
 	InsertAccountData(ctx context.Context, txn *sql.Tx, userID, roomID, dataType string) (pos types.StreamPosition, err error)
 	// SelectAccountDataInRange returns a map of room ID to a list of `dataType`.
-	SelectAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, pos types.StreamPosition, err error)
+	SelectAccountDataInRange(ctx context.Context, txn *sql.Tx, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, pos types.StreamPosition, err error)
 	SelectMaxAccountDataID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 }
 
@@ -46,7 +46,7 @@ type Peeks interface {
 	DeletePeek(ctx context.Context, txn *sql.Tx, roomID, userID, deviceID string) (streamPos types.StreamPosition, err error)
 	DeletePeeks(ctx context.Context, txn *sql.Tx, roomID, userID string) (streamPos types.StreamPosition, err error)
 	SelectPeeksInRange(ctxt context.Context, txn *sql.Tx, userID, deviceID string, r types.Range) (peeks []types.Peek, err error)
-	SelectPeekingDevices(ctxt context.Context) (peekingDevices map[string][]types.PeekingDevice, err error)
+	SelectPeekingDevices(ctxt context.Context, txn *sql.Tx) (peekingDevices map[string][]types.PeekingDevice, err error)
 	SelectMaxPeekID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 }
 
@@ -68,7 +68,7 @@ type Events interface {
 	// SelectEarlyEvents returns the earliest events in the given room.
 	SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter) ([]types.StreamEvent, error)
 	SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool) ([]types.StreamEvent, error)
-	UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error
+	UpdateEventJSON(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent) error
 	// DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely.
 	DeleteEventsForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error)
 
@@ -98,7 +98,7 @@ type Topology interface {
 }
 
 type CurrentRoomState interface {
-	SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
+	SelectStateEvent(ctx context.Context, txn *sql.Tx, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
 	SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error)
 	UpsertRoomState(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition) error
 	DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error
@@ -110,9 +110,9 @@ type CurrentRoomState interface {
 	// SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user.
 	SelectRoomIDsWithAnyMembership(ctx context.Context, txn *sql.Tx, userID string) (map[string]string, error)
 	// SelectJoinedUsers returns a map of room ID to a list of joined user IDs.
-	SelectJoinedUsers(ctx context.Context) (map[string][]string, error)
+	SelectJoinedUsers(ctx context.Context, txn *sql.Tx) (map[string][]string, error)
 	// SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room.
-	SelectJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error)
+	SelectJoinedUsersInRoom(ctx context.Context, txn *sql.Tx, roomIDs []string) (map[string][]string, error)
 	// SelectSharedUsers returns a subset of otherUserIDs that share a room with userID.
 	SelectSharedUsers(ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string) ([]string, error)
 }
@@ -142,7 +142,7 @@ type BackwardsExtremities interface {
 	// InsertsBackwardExtremity inserts a new backwards extremity.
 	InsertsBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string) (err error)
 	// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room, as a map of event_id to list of prev_event_ids.
-	SelectBackwardExtremitiesForRoom(ctx context.Context, roomID string) (bwExtrems map[string][]string, err error)
+	SelectBackwardExtremitiesForRoom(ctx context.Context, txn *sql.Tx, roomID string) (bwExtrems map[string][]string, err error)
 	// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed.
 	DeleteBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, knownEventID string) (err error)
 }
@@ -172,13 +172,13 @@ type SendToDevice interface {
 }
 
 type Filter interface {
-	SelectFilter(ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string) error
-	InsertFilter(ctx context.Context, filter *gomatrixserverlib.Filter, localpart string) (filterID string, err error)
+	SelectFilter(ctx context.Context, txn *sql.Tx, target *gomatrixserverlib.Filter, localpart string, filterID string) error
+	InsertFilter(ctx context.Context, txn *sql.Tx, filter *gomatrixserverlib.Filter, localpart string) (filterID string, err error)
 }
 
 type Receipts interface {
 	UpsertReceipt(ctx context.Context, txn *sql.Tx, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error)
-	SelectRoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error)
+	SelectRoomReceiptsAfter(ctx context.Context, txn *sql.Tx, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error)
 	SelectMaxReceiptID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 }
 

From 68d6eb0a6fe911f2e7598bcd8d4f35a3c4e7c1d9 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 29 Sep 2022 13:06:55 +0100
Subject: [PATCH 07/73] P2P demo fixes

---
 build/gobind-pinecone/monolith.go | 59 ++++++++++++++++++-------------
 go.mod                            |  2 +-
 go.sum                            |  4 +--
 3 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 500403ae4..241adcf73 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -28,6 +28,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -66,6 +67,7 @@ const (
 	PeerTypeRemote    = pineconeRouter.PeerTypeRemote
 	PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
 	PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
+	PeerTypeBonjour   = pineconeRouter.PeerTypeBonjour
 )
 
 type DendriteMonolith struct {
@@ -82,6 +84,10 @@ type DendriteMonolith struct {
 	userAPI           userapiAPI.UserInternalAPI
 }
 
+func (m *DendriteMonolith) PublicKey() string {
+	return m.PineconeRouter.PublicKey().String()
+}
+
 func (m *DendriteMonolith) BaseURL() string {
 	return fmt.Sprintf("http://%s", m.listener.Addr().String())
 }
@@ -134,32 +140,21 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error)
 	go func() {
 		conduit.portMutex.Lock()
 		defer conduit.portMutex.Unlock()
-	loop:
-		for i := 1; i <= 10; i++ {
-			logrus.Errorf("Attempting authenticated connect (attempt %d)", i)
-			var err error
-			conduit.port, err = m.PineconeRouter.Connect(
-				l,
-				pineconeRouter.ConnectionZone(zone),
-				pineconeRouter.ConnectionPeerType(peertype),
-			)
-			switch err {
-			case io.ErrClosedPipe:
-				logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i)
-				return
-			case io.EOF:
-				logrus.Errorf("Authenticated connect failed due to EOF (attempt %d)", i)
-				break loop
-			case nil:
-				logrus.Errorf("Authenticated connect succeeded, connected to port %d (attempt %d)", conduit.port, i)
-				return
-			default:
-				logrus.WithError(err).Errorf("Authenticated connect failed (attempt %d)", i)
-				time.Sleep(time.Second)
-			}
+
+		logrus.Errorf("Attempting authenticated connect")
+		var err error
+		if conduit.port, err = m.PineconeRouter.Connect(
+			l,
+			pineconeRouter.ConnectionZone(zone),
+			pineconeRouter.ConnectionPeerType(peertype),
+		); err != nil {
+			logrus.Errorf("Authenticated connect failed: %s", err)
+			_ = l.Close()
+			_ = r.Close()
+			_ = conduit.Close()
+			return
 		}
-		_ = l.Close()
-		_ = r.Close()
+		logrus.Infof("Authenticated connect succeeded (port %d)", conduit.port)
 	}()
 	return conduit, nil
 }
@@ -395,6 +390,7 @@ func (m *DendriteMonolith) Stop() {
 const MaxFrameSize = types.MaxFrameSize
 
 type Conduit struct {
+	closed    atomic.Bool
 	conn      net.Conn
 	port      types.SwitchPortID
 	portMutex sync.Mutex
@@ -407,10 +403,16 @@ func (c *Conduit) Port() int {
 }
 
 func (c *Conduit) Read(b []byte) (int, error) {
+	if c.closed.Load() {
+		return 0, io.EOF
+	}
 	return c.conn.Read(b)
 }
 
 func (c *Conduit) ReadCopy() ([]byte, error) {
+	if c.closed.Load() {
+		return nil, io.EOF
+	}
 	var buf [65535 * 2]byte
 	n, err := c.conn.Read(buf[:])
 	if err != nil {
@@ -420,9 +422,16 @@ func (c *Conduit) ReadCopy() ([]byte, error) {
 }
 
 func (c *Conduit) Write(b []byte) (int, error) {
+	if c.closed.Load() {
+		return 0, io.EOF
+	}
 	return c.conn.Write(b)
 }
 
 func (c *Conduit) Close() error {
+	if c.closed.Load() {
+		return io.ErrClosedPipe
+	}
+	c.closed.Store(true)
 	return c.conn.Close()
 }
diff --git a/go.mod b/go.mod
index b682d9bc4..b0a41c558 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5
-	github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d
+	github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5
diff --git a/go.sum b/go.sum
index 1afed73a5..b524efc71 100644
--- a/go.sum
+++ b/go.sum
@@ -386,8 +386,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
-github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d h1:kGPJ6Rg8nn5an2CbCZrRiuTNyNzE0rRMiqm4UXJYrRs=
-github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
+github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE=
+github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=

From 6f3dbee5f491118a9105415399712bad87d5811c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 29 Sep 2022 16:08:00 +0100
Subject: [PATCH 08/73] Fix Go 1.18

---
 build/gobind-pinecone/monolith.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 241adcf73..5b368a998 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -28,9 +28,10 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
+	"go.uber.org/atomic"
+
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/clientapi/userutil"

From e1bf709eb3e66b5af9f99b9a3d406e3d64ccee44 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 29 Sep 2022 16:08:41 +0100
Subject: [PATCH 09/73] Improve device list update parsing (update to
 matrix-org/gomatrixserverlib#342)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index b0a41c558..99774889c 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e
 	github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
diff --git a/go.sum b/go.sum
index b524efc71..f64eb1ce8 100644
--- a/go.sum
+++ b/go.sum
@@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e h1:no8QJnxgvBSDKeFivBPZMCBYdXbUYeAHvzqP9oehz4k=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE=
 github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

From 1b65c97ad17f95bfffa620a8c5b007d0318c0291 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 29 Sep 2022 16:18:42 +0100
Subject: [PATCH 10/73] Remove `Suppressing send-to-device` log line

The behaviour is correct and I have no idea why we are logging it.
---
 federationapi/consumers/sendtodevice.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go
index e44bad723..ffc1d8894 100644
--- a/federationapi/consumers/sendtodevice.go
+++ b/federationapi/consumers/sendtodevice.go
@@ -80,7 +80,6 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 		return true
 	}
 	if originServerName != t.ServerName {
-		log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere")
 		return true
 	}
 	// Extract the send-to-device event from msg.

From b28bbadeb02978634e7d91844568a6b09b79e53b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 29 Sep 2022 16:55:03 +0100
Subject: [PATCH 11/73] Update to matrix-org/gomatrixserverlib@377b320

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 99774889c..b805a531f 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1
 	github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
diff --git a/go.sum b/go.sum
index f64eb1ce8..4e3b6b356 100644
--- a/go.sum
+++ b/go.sum
@@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e h1:no8QJnxgvBSDKeFivBPZMCBYdXbUYeAHvzqP9oehz4k=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 h1:kS4jgi/DshFGJArjF14b1u42fVWC+8LMVdSVLdQYq3E=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE=
 github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

From 90f1985bf30ac9f79ef5a0c7ae8406aca904aaad Mon Sep 17 00:00:00 2001
From: devonh <devon.dmytro@gmail.com>
Date: Thu, 29 Sep 2022 16:05:16 +0000
Subject: [PATCH 12/73] Add network interface registration to pinecone demo
 (#2750)

### Pull Request Checklist

<!-- Please read docs/CONTRIBUTING.md before submitting your pull
request -->

No additional tests have been added as this simply extends the API for
the embedded pinecone demo.

Signed-off-by: `Devon Hudson <devonhudson@librem.one>`
---
 build/gobind-pinecone/monolith.go | 14 ++++++++++++++
 go.mod                            |  2 +-
 go.sum                            |  4 ++--
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 5b368a998..b0d3fa285 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -101,6 +101,20 @@ func (m *DendriteMonolith) SessionCount() int {
 	return len(m.PineconeQUIC.Protocol("matrix").Sessions())
 }
 
+func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) {
+	m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{
+		Name: name,
+		Index: index,
+		Mtu: mtu,
+		Up: up,
+		Broadcast: broadcast,
+		Loopback: loopback,
+		PointToPoint: pointToPoint,
+		Multicast: multicast,
+		Addrs: addrs,
+	})
+}
+
 func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 	if enabled {
 		m.PineconeMulticast.Start()
diff --git a/go.mod b/go.mod
index b805a531f..bd79680ed 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1
-	github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd
+	github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5
diff --git a/go.sum b/go.sum
index 4e3b6b356..742a52dcb 100644
--- a/go.sum
+++ b/go.sum
@@ -386,8 +386,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 h1:kS4jgi/DshFGJArjF14b1u42fVWC+8LMVdSVLdQYq3E=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
-github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE=
-github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
+github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk=
+github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=

From e45ba35e979f2f8af1e75f5f16158973b790436b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 29 Sep 2022 20:05:05 +0100
Subject: [PATCH 13/73] Enable knocking on room versions 8 and 9 (update to
 matrix-org/gomatrixserverlib@91d455c)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index bd79680ed..0985b986c 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621
 	github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
diff --git a/go.sum b/go.sum
index 742a52dcb..c3db3e09e 100644
--- a/go.sum
+++ b/go.sum
@@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 h1:kS4jgi/DshFGJArjF14b1u42fVWC+8LMVdSVLdQYq3E=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 h1:a8IaoSPDxevkgXnOUrtIW9AqVNvXBJAG0gtnX687S7g=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk=
 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

From 9005e5b4a87c0bdd4f19b1dbf46f3093ac07e727 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 30 Sep 2022 10:32:31 +0200
Subject: [PATCH 14/73] Add `/_dendrite/admin/refreshDevices/{userID}` (#2746)

Allows to immediately query `/devices/{userID}` over federation to
(hopefully) resolve E2EE issues.
---
 clientapi/routing/admin.go        | 55 ++++++++++++++++++++++++-------
 clientapi/routing/routing.go      |  8 ++++-
 docs/administration/4_adminapi.md |  5 +++
 keyserver/api/api.go              |  1 +
 keyserver/internal/internal.go    | 13 ++++++--
 5 files changed, 66 insertions(+), 16 deletions(-)

diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go
index 5089d7c36..89c269f1a 100644
--- a/clientapi/routing/admin.go
+++ b/clientapi/routing/admin.go
@@ -2,20 +2,23 @@ package routing
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/keyserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
@@ -144,12 +147,6 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 }
 
 func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse {
-	if device.AccountType != userapi.AccountTypeAdmin {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-		}
-	}
 	_, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10)
 	if err != nil {
 		logrus.WithError(err).Error("failed to publish nats message")
@@ -160,3 +157,37 @@ func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Devi
 		JSON: struct{}{},
 	}
 }
+
+func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.ClientKeyAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	userID := vars["userID"]
+
+	_, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return util.MessageResponse(http.StatusBadRequest, err.Error())
+	}
+	if domain == cfg.Matrix.ServerName {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidParam("Can not mark local device list as stale"),
+		}
+	}
+
+	err = keyAPI.PerformMarkAsStaleIfNeeded(req.Context(), &api.PerformMarkAsStaleRequest{
+		UserID: userID,
+		Domain: domain,
+	}, &struct{}{})
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown(fmt.Sprintf("Failed to mark device list as stale: %s", err)),
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 9c1f8f720..7d1c434c4 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -163,11 +163,17 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/fulltext/reindex",
-		httputil.MakeAuthAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeAdminAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return AdminReindex(req, cfg, device, natsClient)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/refreshDevices/{userID}",
+		httputil.MakeAdminAPI("admin_refresh_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminMarkAsStale(req, cfg, keyAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md
index 1712bb1bf..56e19a8b4 100644
--- a/docs/administration/4_adminapi.md
+++ b/docs/administration/4_adminapi.md
@@ -62,6 +62,11 @@ the full user ID is `@alice:domain.com` then the local part is `alice`.
 This endpoint instructs Dendrite to reindex all searchable events (`m.room.message`, `m.room.topic` and `m.room.name`). An empty JSON body will be returned immediately.
 Indexing is done in the background, the server logs every 1000 events (or below) when they are being indexed. Once reindexing is done, you'll see something along the lines `Indexed 69586 events in 53.68223182s` in your debug logs.
 
+## POST `/_dendrite/admin/refreshDevices/{userID}`
+
+This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a federated server. An empty JSON body will be returned on success, updating all locally stored user devices/keys. This can be used to possibly resolve E2EE issues, where the remote user can't decrypt messages.
+
+
 ## POST `/_synapse/admin/v1/send_server_notice`
 
 Request body format:
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index c9ec59a75..14fced3e8 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -45,6 +45,7 @@ type ClientKeyAPI interface {
 	PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse) error
 	// PerformClaimKeys claims one-time keys for use in pre-key messages
 	PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error
+	PerformMarkAsStaleIfNeeded(ctx context.Context, req *PerformMarkAsStaleRequest, res *struct{}) error
 }
 
 // API functions required by the userapi
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index a8d1128c4..6309066d7 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -228,14 +228,21 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query
 // PerformMarkAsStaleIfNeeded marks the users device list as stale, if the given deviceID is not present
 // in our database.
 func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *api.PerformMarkAsStaleRequest, res *struct{}) error {
-	knownDevices, err := a.DB.DeviceKeysForUser(ctx, req.UserID, []string{req.DeviceID}, true)
+	knownDevices, err := a.DB.DeviceKeysForUser(ctx, req.UserID, []string{}, true)
 	if err != nil {
 		return err
 	}
 	if len(knownDevices) == 0 {
-		return a.Updater.ManualUpdate(ctx, req.Domain, req.UserID)
+		return fmt.Errorf("unknown user %s", req.UserID)
 	}
-	return nil
+
+	for i := range knownDevices {
+		if knownDevices[i].DeviceID == req.DeviceID {
+			return nil // we already know about this device
+		}
+	}
+
+	return a.Updater.ManualUpdate(ctx, req.Domain, req.UserID)
 }
 
 // nolint:gocyclo

From 8a82f100460dc5ca7bd39ae2345c251d6622c494 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 09:41:16 +0100
Subject: [PATCH 15/73] Allow more time for device list updates (#2749)

This updates the device list updater so that it has a context
per-request, rather than a global 30 seconds for the entire server. This
could mean that talking to a slow remote server or requesting a lot of
user IDs was pretty much guaranteed to fail.

It also uses the process context to allow correct cancellation when
Dendrite wants to shut down cleanly.
---
 keyserver/internal/device_list_update.go      | 204 ++++++++++--------
 keyserver/internal/device_list_update_test.go |   7 +-
 keyserver/keyserver.go                        |   2 +-
 3 files changed, 117 insertions(+), 96 deletions(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 525f8a99d..fcfcd092d 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -17,6 +17,7 @@ package internal
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"net"
@@ -31,6 +32,7 @@ import (
 
 	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/setup/process"
 )
 
 var (
@@ -45,6 +47,9 @@ var (
 	)
 )
 
+const defaultWaitTime = time.Minute
+const requestTimeout = time.Second * 30
+
 func init() {
 	prometheus.MustRegister(
 		deviceListUpdateCount,
@@ -80,6 +85,7 @@ func init() {
 // In the event that the query fails, a lock is acquired and the server name along with the time to wait before retrying is
 // set in a map. A restarter goroutine periodically probes this map and injects servers which are ready to be retried.
 type DeviceListUpdater struct {
+	process *process.ProcessContext
 	// A map from user_id to a mutex. Used when we are missing prev IDs so we don't make more than 1
 	// request to the remote server and race.
 	// TODO: Put in an LRU cache to bound growth
@@ -131,10 +137,12 @@ type KeyChangeProducer interface {
 
 // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale.
 func NewDeviceListUpdater(
-	db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer,
+	process *process.ProcessContext, db DeviceListUpdaterDatabase,
+	api DeviceListUpdaterAPI, producer KeyChangeProducer,
 	fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int,
 ) *DeviceListUpdater {
 	return &DeviceListUpdater{
+		process:        process,
 		userIDToMutex:  make(map[string]*sync.Mutex),
 		mu:             &sync.Mutex{},
 		db:             db,
@@ -234,7 +242,7 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib.
 		"prev_ids":       event.PrevID,
 		"display_name":   event.DeviceDisplayName,
 		"deleted":        event.Deleted,
-	}).Info("DeviceListUpdater.Update")
+	}).Trace("DeviceListUpdater.Update")
 
 	// if we haven't missed anything update the database and notify users
 	if exists || event.Deleted {
@@ -378,111 +386,123 @@ func (u *DeviceListUpdater) worker(ch chan gomatrixserverlib.ServerName) {
 }
 
 func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerName) (time.Duration, bool) {
-	deviceListUpdateCount.WithLabelValues(string(serverName)).Inc()
-	requestTimeout := time.Second * 30 // max amount of time we want to spend on each request
-	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
-	defer cancel()
+	ctx := u.process.Context()
 	logger := util.GetLogger(ctx).WithField("server_name", serverName)
-	waitTime := 2 * time.Second
-	// fetch stale device lists
+	deviceListUpdateCount.WithLabelValues(string(serverName)).Inc()
+
+	waitTime := defaultWaitTime // How long should we wait to try again?
+	successCount := 0           // How many user requests failed?
+
 	userIDs, err := u.db.StaleDeviceLists(ctx, []gomatrixserverlib.ServerName{serverName})
 	if err != nil {
 		logger.WithError(err).Error("Failed to load stale device lists")
 		return waitTime, true
 	}
-	failCount := 0
 
-userLoop:
+	defer func() {
+		for _, userID := range userIDs {
+			// always clear the channel to unblock Update calls regardless of success/failure
+			u.clearChannel(userID)
+		}
+	}()
+
 	for _, userID := range userIDs {
-		if ctx.Err() != nil {
-			// we've timed out, give up and go to the back of the queue to let another server be processed.
-			failCount += 1
-			waitTime = time.Minute * 10
+		userWait, err := u.processServerUser(ctx, serverName, userID)
+		if err != nil {
+			if userWait > waitTime {
+				waitTime = userWait
+			}
 			break
 		}
-		res, err := u.fedClient.GetUserDevices(ctx, serverName, userID)
-		if err != nil {
-			failCount += 1
-			select {
-			case <-ctx.Done():
-				// we've timed out, give up and go to the back of the queue to let another server be processed.
-				waitTime = time.Minute * 10
-				break userLoop
-			default:
-			}
-			switch e := err.(type) {
-			case *fedsenderapi.FederationClientError:
-				if e.RetryAfter > 0 {
-					waitTime = e.RetryAfter
-				} else if e.Blacklisted {
-					waitTime = time.Hour * 8
-					break userLoop
-				} else if e.Code >= 300 {
-					// We didn't get a real FederationClientError (e.g. in polylith mode, where gomatrix.HTTPError
-					// are "converted" to FederationClientError), but we probably shouldn't hit them every $waitTime seconds.
-					waitTime = time.Hour
-					break userLoop
-				}
-			case net.Error:
-				// Use the default waitTime, if it's a timeout.
-				// It probably doesn't make sense to try further users.
-				if !e.Timeout() {
-					waitTime = time.Minute * 10
-					logger.WithError(e).Error("GetUserDevices returned net.Error")
-					break userLoop
-				}
-			case gomatrix.HTTPError:
-				// The remote server returned an error, give it some time to recover.
-				// This is to avoid spamming remote servers, which may not be Matrix servers anymore.
-				if e.Code >= 300 {
-					waitTime = time.Hour
-					logger.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError")
-					break userLoop
-				}
-			default:
-				// Something else failed
-				waitTime = time.Minute * 10
-				logger.WithError(err).WithField("user_id", userID).Debugf("GetUserDevices returned unknown error type: %T", err)
-				break userLoop
-			}
-			continue
-		}
-		if res.MasterKey != nil || res.SelfSigningKey != nil {
-			uploadReq := &api.PerformUploadDeviceKeysRequest{
-				UserID: userID,
-			}
-			uploadRes := &api.PerformUploadDeviceKeysResponse{}
-			if res.MasterKey != nil {
-				if err = sanityCheckKey(*res.MasterKey, userID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err == nil {
-					uploadReq.MasterKey = *res.MasterKey
-				}
-			}
-			if res.SelfSigningKey != nil {
-				if err = sanityCheckKey(*res.SelfSigningKey, userID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err == nil {
-					uploadReq.SelfSigningKey = *res.SelfSigningKey
-				}
-			}
-			_ = u.api.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes)
-		}
-		err = u.updateDeviceList(&res)
-		if err != nil {
-			logger.WithError(err).WithField("user_id", userID).Error("Fetched device list but failed to store/emit it")
-			failCount += 1
-		}
+		successCount++
 	}
-	if failCount > 0 {
+
+	allUsersSucceeded := successCount == len(userIDs)
+	if !allUsersSucceeded {
 		logger.WithFields(logrus.Fields{
-			"total":    len(userIDs),
-			"failed":   failCount,
-			"skipped":  len(userIDs) - failCount,
-			"waittime": waitTime,
+			"total":     len(userIDs),
+			"succeeded": successCount,
+			"failed":    len(userIDs) - successCount,
+			"wait_time": waitTime,
 		}).Warn("Failed to query device keys for some users")
 	}
-	for _, userID := range userIDs {
-		// always clear the channel to unblock Update calls regardless of success/failure
-		u.clearChannel(userID)
+	return waitTime, !allUsersSucceeded
+}
+
+func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName gomatrixserverlib.ServerName, userID string) (time.Duration, error) {
+	ctx, cancel := context.WithTimeout(ctx, requestTimeout)
+	defer cancel()
+	logger := util.GetLogger(ctx).WithFields(logrus.Fields{
+		"server_name": serverName,
+		"user_id":     userID,
+	})
+
+	res, err := u.fedClient.GetUserDevices(ctx, serverName, userID)
+	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return time.Minute * 10, err
+		}
+		switch e := err.(type) {
+		case *json.UnmarshalTypeError, *json.SyntaxError:
+			logger.WithError(err).Debugf("Device list update for %q contained invalid JSON", userID)
+			return defaultWaitTime, nil
+		case *fedsenderapi.FederationClientError:
+			if e.RetryAfter > 0 {
+				return e.RetryAfter, err
+			} else if e.Blacklisted {
+				return time.Hour * 8, err
+			} else if e.Code >= 300 {
+				// We didn't get a real FederationClientError (e.g. in polylith mode, where gomatrix.HTTPError
+				// are "converted" to FederationClientError), but we probably shouldn't hit them every $waitTime seconds.
+				return time.Hour, err
+			}
+		case net.Error:
+			// Use the default waitTime, if it's a timeout.
+			// It probably doesn't make sense to try further users.
+			if !e.Timeout() {
+				logger.WithError(e).Debug("GetUserDevices returned net.Error")
+				return time.Minute * 10, err
+			}
+		case gomatrix.HTTPError:
+			// The remote server returned an error, give it some time to recover.
+			// This is to avoid spamming remote servers, which may not be Matrix servers anymore.
+			if e.Code >= 300 {
+				logger.WithError(e).Debug("GetUserDevices returned gomatrix.HTTPError")
+				return time.Hour, err
+			}
+		default:
+			// Something else failed
+			logger.WithError(err).Debugf("GetUserDevices returned unknown error type: %T", err)
+			return time.Minute * 10, err
+		}
 	}
-	return waitTime, failCount > 0
+	if res.UserID != userID {
+		logger.WithError(err).Debugf("User ID %q in device list update response doesn't match expected %q", res.UserID, userID)
+		return defaultWaitTime, nil
+	}
+	if res.MasterKey != nil || res.SelfSigningKey != nil {
+		uploadReq := &api.PerformUploadDeviceKeysRequest{
+			UserID: userID,
+		}
+		uploadRes := &api.PerformUploadDeviceKeysResponse{}
+		if res.MasterKey != nil {
+			if err = sanityCheckKey(*res.MasterKey, userID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err == nil {
+				uploadReq.MasterKey = *res.MasterKey
+			}
+		}
+		if res.SelfSigningKey != nil {
+			if err = sanityCheckKey(*res.SelfSigningKey, userID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err == nil {
+				uploadReq.SelfSigningKey = *res.SelfSigningKey
+			}
+		}
+		_ = u.api.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes)
+	}
+	err = u.updateDeviceList(&res)
+	if err != nil {
+		logger.WithError(err).Error("Fetched device list but failed to store/emit it")
+		return defaultWaitTime, err
+	}
+	return defaultWaitTime, nil
 }
 
 func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevices) error {
diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go
index 0520a9e66..28a13a0a0 100644
--- a/keyserver/internal/device_list_update_test.go
+++ b/keyserver/internal/device_list_update_test.go
@@ -30,6 +30,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/setup/process"
 )
 
 var (
@@ -146,7 +147,7 @@ func TestUpdateHavePrevID(t *testing.T) {
 	}
 	ap := &mockDeviceListUpdaterAPI{}
 	producer := &mockKeyChangeProducer{}
-	updater := NewDeviceListUpdater(db, ap, producer, nil, 1)
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1)
 	event := gomatrixserverlib.DeviceListUpdateEvent{
 		DeviceDisplayName: "Foo Bar",
 		Deleted:           false,
@@ -218,7 +219,7 @@ func TestUpdateNoPrevID(t *testing.T) {
 			`)),
 		}, nil
 	})
-	updater := NewDeviceListUpdater(db, ap, producer, fedClient, 2)
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2)
 	if err := updater.Start(); err != nil {
 		t.Fatalf("failed to start updater: %s", err)
 	}
@@ -287,7 +288,7 @@ func TestDebounce(t *testing.T) {
 		close(incomingFedReq)
 		return <-fedCh, nil
 	})
-	updater := NewDeviceListUpdater(db, ap, producer, fedClient, 1)
+	updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1)
 	if err := updater.Start(); err != nil {
 		t.Fatalf("failed to start updater: %s", err)
 	}
diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go
index 5124b777e..9ae4f9ca3 100644
--- a/keyserver/keyserver.go
+++ b/keyserver/keyserver.go
@@ -58,7 +58,7 @@ func NewInternalAPI(
 		FedClient:  fedClient,
 		Producer:   keyChangeProducer,
 	}
-	updater := internal.NewDeviceListUpdater(db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable
+	updater := internal.NewDeviceListUpdater(base.ProcessContext, db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable
 	ap.Updater = updater
 	go func() {
 		if err := updater.Start(); err != nil {

From 6348486a1365c7469a498101f5035a9b6bd16d22 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 12:48:10 +0100
Subject: [PATCH 16/73] Transactional isolation for `/sync` (#2745)

This should transactional snapshot isolation for `/sync` etc requests.

For now we don't use repeatable read due to some odd test failures with
invites.
---
 syncapi/consumers/clientapi.go                |    5 +-
 syncapi/consumers/keychange.go                |    5 +-
 syncapi/consumers/presence.go                 |    5 +-
 syncapi/consumers/receipts.go                 |    5 +-
 syncapi/consumers/roomserver.go               |   19 +-
 syncapi/consumers/sendtodevice.go             |    5 +-
 syncapi/consumers/typing.go                   |    5 +-
 syncapi/consumers/userapi.go                  |    5 +-
 syncapi/internal/history_visibility.go        |    2 +-
 syncapi/notifier/notifier.go                  |   19 +-
 syncapi/routing/context.go                    |   30 +-
 syncapi/routing/messages.go                   |   42 +-
 syncapi/routing/search.go                     |   24 +-
 syncapi/storage/interface.go                  |  125 +-
 syncapi/storage/postgres/invites_table.go     |   22 +-
 syncapi/storage/shared/storage_consumer.go    |  586 +++++++++
 syncapi/storage/shared/storage_sync.go        |  574 +++++++++
 syncapi/storage/shared/syncserver.go          | 1103 -----------------
 .../sqlite3/current_room_state_table.go       |    8 +-
 syncapi/storage/sqlite3/invites_table.go      |   23 +-
 syncapi/storage/sqlite3/syncserver.go         |   14 +
 syncapi/storage/storage_test.go               |  197 +--
 syncapi/storage/tables/interface.go           |    2 +-
 syncapi/streams/stream_accountdata.go         |   17 +-
 syncapi/streams/stream_devicelist.go          |    7 +-
 syncapi/streams/stream_invite.go              |   19 +-
 syncapi/streams/stream_notificationdata.go    |   20 +-
 syncapi/streams/stream_pdu.go                 |  141 +--
 syncapi/streams/stream_presence.go            |   22 +-
 syncapi/streams/stream_receipt.go             |   20 +-
 syncapi/streams/stream_sendtodevice.go        |   20 +-
 syncapi/streams/stream_typing.go              |    7 +-
 syncapi/streams/streamprovider.go             |   28 +
 syncapi/streams/streams.go                    |   77 +-
 syncapi/streams/template_stream.go            |   10 +-
 syncapi/sync/requestpool.go                   |   53 +-
 syncapi/types/provider.go                     |   20 -
 37 files changed, 1759 insertions(+), 1527 deletions(-)
 create mode 100644 syncapi/storage/shared/storage_consumer.go
 create mode 100644 syncapi/storage/shared/storage_sync.go
 delete mode 100644 syncapi/storage/shared/syncserver.go
 create mode 100644 syncapi/streams/streamprovider.go

diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go
index 796cc61e1..735f6718c 100644
--- a/syncapi/consumers/clientapi.go
+++ b/syncapi/consumers/clientapi.go
@@ -34,6 +34,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
@@ -46,7 +47,7 @@ type OutputClientDataConsumer struct {
 	topic        string
 	topicReIndex string
 	db           storage.Database
-	stream       types.StreamProvider
+	stream       streams.StreamProvider
 	notifier     *notifier.Notifier
 	serverName   gomatrixserverlib.ServerName
 	fts          *fulltext.Search
@@ -61,7 +62,7 @@ func NewOutputClientDataConsumer(
 	nats *nats.Conn,
 	store storage.Database,
 	notifier *notifier.Notifier,
-	stream types.StreamProvider,
+	stream streams.StreamProvider,
 	fts *fulltext.Search,
 ) *OutputClientDataConsumer {
 	return &OutputClientDataConsumer{
diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go
index c42e71971..dc7d9e207 100644
--- a/syncapi/consumers/keychange.go
+++ b/syncapi/consumers/keychange.go
@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
@@ -40,7 +41,7 @@ type OutputKeyChangeEventConsumer struct {
 	topic      string
 	db         storage.Database
 	notifier   *notifier.Notifier
-	stream     types.StreamProvider
+	stream     streams.StreamProvider
 	serverName gomatrixserverlib.ServerName // our server name
 	rsAPI      roomserverAPI.SyncRoomserverAPI
 }
@@ -55,7 +56,7 @@ func NewOutputKeyChangeEventConsumer(
 	rsAPI roomserverAPI.SyncRoomserverAPI,
 	store storage.Database,
 	notifier *notifier.Notifier,
-	stream types.StreamProvider,
+	stream streams.StreamProvider,
 ) *OutputKeyChangeEventConsumer {
 	s := &OutputKeyChangeEventConsumer{
 		ctx:        process.Context(),
diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go
index 61bdc13de..145059c2d 100644
--- a/syncapi/consumers/presence.go
+++ b/syncapi/consumers/presence.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -39,7 +40,7 @@ type PresenceConsumer struct {
 	requestTopic  string
 	presenceTopic string
 	db            storage.Database
-	stream        types.StreamProvider
+	stream        streams.StreamProvider
 	notifier      *notifier.Notifier
 	deviceAPI     api.SyncUserAPI
 	cfg           *config.SyncAPI
@@ -54,7 +55,7 @@ func NewPresenceConsumer(
 	nats *nats.Conn,
 	db storage.Database,
 	notifier *notifier.Notifier,
-	stream types.StreamProvider,
+	stream streams.StreamProvider,
 	deviceAPI api.SyncUserAPI,
 ) *PresenceConsumer {
 	return &PresenceConsumer{
diff --git a/syncapi/consumers/receipts.go b/syncapi/consumers/receipts.go
index 4379dd134..8aaa65730 100644
--- a/syncapi/consumers/receipts.go
+++ b/syncapi/consumers/receipts.go
@@ -28,6 +28,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
@@ -38,7 +39,7 @@ type OutputReceiptEventConsumer struct {
 	durable    string
 	topic      string
 	db         storage.Database
-	stream     types.StreamProvider
+	stream     streams.StreamProvider
 	notifier   *notifier.Notifier
 	serverName gomatrixserverlib.ServerName
 }
@@ -51,7 +52,7 @@ func NewOutputReceiptEventConsumer(
 	js nats.JetStreamContext,
 	store storage.Database,
 	notifier *notifier.Notifier,
-	stream types.StreamProvider,
+	stream streams.StreamProvider,
 ) *OutputReceiptEventConsumer {
 	return &OutputReceiptEventConsumer{
 		ctx:        process.Context(),
diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index 3756ad75c..e5e8fe293 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -33,6 +33,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
@@ -45,8 +46,8 @@ type OutputRoomEventConsumer struct {
 	durable      string
 	topic        string
 	db           storage.Database
-	pduStream    types.StreamProvider
-	inviteStream types.StreamProvider
+	pduStream    streams.StreamProvider
+	inviteStream streams.StreamProvider
 	notifier     *notifier.Notifier
 	fts          *fulltext.Search
 }
@@ -58,8 +59,8 @@ func NewOutputRoomEventConsumer(
 	js nats.JetStreamContext,
 	store storage.Database,
 	notifier *notifier.Notifier,
-	pduStream types.StreamProvider,
-	inviteStream types.StreamProvider,
+	pduStream streams.StreamProvider,
+	inviteStream streams.StreamProvider,
 	rsAPI api.SyncRoomserverAPI,
 	fts *fulltext.Search,
 ) *OutputRoomEventConsumer {
@@ -449,8 +450,14 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head
 	}
 	stateKey := *event.StateKey()
 
-	prevEvent, err := s.db.GetStateEvent(
-		context.TODO(), event.RoomID(), event.Type(), stateKey,
+	snapshot, err := s.db.NewDatabaseSnapshot(s.ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer snapshot.Rollback() // nolint:errcheck
+
+	prevEvent, err := snapshot.GetStateEvent(
+		s.ctx, event.RoomID(), event.Type(), stateKey,
 	)
 	if err != nil {
 		return event, err
diff --git a/syncapi/consumers/sendtodevice.go b/syncapi/consumers/sendtodevice.go
index c0b432256..49d84cca3 100644
--- a/syncapi/consumers/sendtodevice.go
+++ b/syncapi/consumers/sendtodevice.go
@@ -31,6 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
@@ -43,7 +44,7 @@ type OutputSendToDeviceEventConsumer struct {
 	db         storage.Database
 	keyAPI     keyapi.SyncKeyAPI
 	serverName gomatrixserverlib.ServerName // our server name
-	stream     types.StreamProvider
+	stream     streams.StreamProvider
 	notifier   *notifier.Notifier
 }
 
@@ -56,7 +57,7 @@ func NewOutputSendToDeviceEventConsumer(
 	store storage.Database,
 	keyAPI keyapi.SyncKeyAPI,
 	notifier *notifier.Notifier,
-	stream types.StreamProvider,
+	stream streams.StreamProvider,
 ) *OutputSendToDeviceEventConsumer {
 	return &OutputSendToDeviceEventConsumer{
 		ctx:        process.Context(),
diff --git a/syncapi/consumers/typing.go b/syncapi/consumers/typing.go
index 88db80f8c..67a26239d 100644
--- a/syncapi/consumers/typing.go
+++ b/syncapi/consumers/typing.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
@@ -36,7 +37,7 @@ type OutputTypingEventConsumer struct {
 	durable   string
 	topic     string
 	eduCache  *caching.EDUCache
-	stream    types.StreamProvider
+	stream    streams.StreamProvider
 	notifier  *notifier.Notifier
 }
 
@@ -48,7 +49,7 @@ func NewOutputTypingEventConsumer(
 	js nats.JetStreamContext,
 	eduCache *caching.EDUCache,
 	notifier *notifier.Notifier,
-	stream types.StreamProvider,
+	stream streams.StreamProvider,
 ) *OutputTypingEventConsumer {
 	return &OutputTypingEventConsumer{
 		ctx:       process.Context(),
diff --git a/syncapi/consumers/userapi.go b/syncapi/consumers/userapi.go
index c9b96f788..3c73dc1fc 100644
--- a/syncapi/consumers/userapi.go
+++ b/syncapi/consumers/userapi.go
@@ -28,6 +28,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/streams"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
@@ -40,7 +41,7 @@ type OutputNotificationDataConsumer struct {
 	topic     string
 	db        storage.Database
 	notifier  *notifier.Notifier
-	stream    types.StreamProvider
+	stream    streams.StreamProvider
 }
 
 // NewOutputNotificationDataConsumer creates a new consumer. Call
@@ -51,7 +52,7 @@ func NewOutputNotificationDataConsumer(
 	js nats.JetStreamContext,
 	store storage.Database,
 	notifier *notifier.Notifier,
-	stream types.StreamProvider,
+	stream streams.StreamProvider,
 ) *OutputNotificationDataConsumer {
 	s := &OutputNotificationDataConsumer{
 		ctx:       process.Context(),
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
index e73c004e5..bbfe19f4c 100644
--- a/syncapi/internal/history_visibility.go
+++ b/syncapi/internal/history_visibility.go
@@ -100,7 +100,7 @@ func (ev eventVisibility) allowed() (allowed bool) {
 // Returns the filtered events and an error, if any.
 func ApplyHistoryVisibilityFilter(
 	ctx context.Context,
-	syncDB storage.Database,
+	syncDB storage.DatabaseTransaction,
 	rsAPI api.SyncRoomserverAPI,
 	events []*gomatrixserverlib.HeaderedEvent,
 	alwaysIncludeEventIDs map[string]struct{},
diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index 87f0d86d7..a8e5bf9ad 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -318,13 +318,20 @@ func (n *Notifier) GetListener(req types.SyncRequest) UserDeviceStreamListener {
 func (n *Notifier) Load(ctx context.Context, db storage.Database) error {
 	n.lock.Lock()
 	defer n.lock.Unlock()
-	roomToUsers, err := db.AllJoinedUsersInRooms(ctx)
+
+	snapshot, err := db.NewDatabaseSnapshot(ctx)
+	if err != nil {
+		return err
+	}
+	defer snapshot.Rollback() // nolint:errcheck
+
+	roomToUsers, err := snapshot.AllJoinedUsersInRooms(ctx)
 	if err != nil {
 		return err
 	}
 	n.setUsersJoinedToRooms(roomToUsers)
 
-	roomToPeekingDevices, err := db.AllPeekingDevicesInRooms(ctx)
+	roomToPeekingDevices, err := snapshot.AllPeekingDevicesInRooms(ctx)
 	if err != nil {
 		return err
 	}
@@ -338,7 +345,13 @@ func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs [
 	n.lock.Lock()
 	defer n.lock.Unlock()
 
-	roomToUsers, err := db.AllJoinedUsersInRoom(ctx, roomIDs)
+	snapshot, err := db.NewDatabaseSnapshot(ctx)
+	if err != nil {
+		return err
+	}
+	defer snapshot.Rollback() // nolint:errcheck
+
+	roomToUsers, err := snapshot.AllJoinedUsersInRoom(ctx, roomIDs)
 	if err != nil {
 		return err
 	}
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 1ebdfe604..1ce34b85a 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -51,6 +51,12 @@ func Context(
 	roomID, eventID string,
 	lazyLoadCache caching.LazyLoadCache,
 ) util.JSONResponse {
+	snapshot, err := syncDB.NewDatabaseSnapshot(req.Context())
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	defer snapshot.Rollback() // nolint:errcheck
+
 	filter, err := parseRoomEventFilter(req)
 	if err != nil {
 		errMsg := ""
@@ -97,7 +103,7 @@ func Context(
 		ContainsURL:             filter.ContainsURL,
 	}
 
-	id, requestedEvent, err := syncDB.SelectContextEvent(ctx, roomID, eventID)
+	id, requestedEvent, err := snapshot.SelectContextEvent(ctx, roomID, eventID)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return util.JSONResponse{
@@ -111,7 +117,7 @@ func Context(
 
 	// verify the user is allowed to see the context for this room/event
 	startTime := time.Now()
-	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, []*gomatrixserverlib.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context")
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*gomatrixserverlib.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context")
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 		return jsonerror.InternalServerError()
@@ -127,20 +133,20 @@ func Context(
 		}
 	}
 
-	eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, roomID, filter)
+	eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, roomID, filter)
 	if err != nil && err != sql.ErrNoRows {
 		logrus.WithError(err).Error("unable to fetch before events")
 		return jsonerror.InternalServerError()
 	}
 
-	_, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, roomID, filter)
+	_, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, roomID, filter)
 	if err != nil && err != sql.ErrNoRows {
 		logrus.WithError(err).Error("unable to fetch after events")
 		return jsonerror.InternalServerError()
 	}
 
 	startTime = time.Now()
-	eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, syncDB, rsAPI, eventsBefore, eventsAfter, device.UserID)
+	eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, device.UserID)
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 		return jsonerror.InternalServerError()
@@ -152,7 +158,7 @@ func Context(
 	}).Debug("applied history visibility (context eventsBefore/eventsAfter)")
 
 	// TODO: Get the actual state at the last event returned by SelectContextAfterEvent
-	state, err := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
+	state, err := snapshot.CurrentState(ctx, roomID, &stateFilter, nil)
 	if err != nil {
 		logrus.WithError(err).Error("unable to fetch current room state")
 		return jsonerror.InternalServerError()
@@ -173,7 +179,7 @@ func Context(
 	if len(response.State) > filter.Limit {
 		response.State = response.State[len(response.State)-filter.Limit:]
 	}
-	start, end, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter)
+	start, end, err := getStartEnd(ctx, snapshot, eventsBefore, eventsAfter)
 	if err == nil {
 		response.End = end.String()
 		response.Start = start.String()
@@ -188,7 +194,7 @@ func Context(
 // by combining the events before and after the context event. Returns the filtered events,
 // and an error, if any.
 func applyHistoryVisibilityOnContextEvents(
-	ctx context.Context, syncDB storage.Database, rsAPI roomserver.SyncRoomserverAPI,
+	ctx context.Context, snapshot storage.DatabaseTransaction, rsAPI roomserver.SyncRoomserverAPI,
 	eventsBefore, eventsAfter []*gomatrixserverlib.HeaderedEvent,
 	userID string,
 ) (filteredBefore, filteredAfter []*gomatrixserverlib.HeaderedEvent, err error) {
@@ -205,7 +211,7 @@ func applyHistoryVisibilityOnContextEvents(
 	}
 
 	allEvents := append(eventsBefore, eventsAfter...)
-	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, allEvents, nil, userID, "context")
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, allEvents, nil, userID, "context")
 	if err != nil {
 		return nil, nil, err
 	}
@@ -222,15 +228,15 @@ func applyHistoryVisibilityOnContextEvents(
 	return filteredBefore, filteredAfter, nil
 }
 
-func getStartEnd(ctx context.Context, syncDB storage.Database, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) {
+func getStartEnd(ctx context.Context, snapshot storage.DatabaseTransaction, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) {
 	if len(startEvents) > 0 {
-		start, err = syncDB.EventPositionInTopology(ctx, startEvents[0].EventID())
+		start, err = snapshot.EventPositionInTopology(ctx, startEvents[0].EventID())
 		if err != nil {
 			return
 		}
 	}
 	if len(endEvents) > 0 {
-		end, err = syncDB.EventPositionInTopology(ctx, endEvents[0].EventID())
+		end, err = snapshot.EventPositionInTopology(ctx, endEvents[0].EventID())
 	}
 	return
 }
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 03614302c..8f3ed3f5b 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -27,6 +27,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/internal"
@@ -39,6 +40,7 @@ import (
 type messagesReq struct {
 	ctx              context.Context
 	db               storage.Database
+	snapshot         storage.DatabaseTransaction
 	rsAPI            api.SyncRoomserverAPI
 	cfg              *config.SyncAPI
 	roomID           string
@@ -70,6 +72,16 @@ func OnIncomingMessagesRequest(
 ) util.JSONResponse {
 	var err error
 
+	// NewDatabaseTransaction is used here instead of NewDatabaseSnapshot as we
+	// expect to be able to write to the database in response to a /messages
+	// request that requires backfilling from the roomserver or federation.
+	snapshot, err := db.NewDatabaseTransaction(req.Context())
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
+
 	// check if the user has already forgotten about this room
 	isForgotten, roomExists, err := checkIsRoomForgotten(req.Context(), roomID, device.UserID, rsAPI)
 	if err != nil {
@@ -132,7 +144,7 @@ func OnIncomingMessagesRequest(
 			}
 		} else {
 			fromStream = &streamToken
-			from, err = db.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, backwardOrdering)
+			from, err = snapshot.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, backwardOrdering)
 			if err != nil {
 				logrus.WithError(err).Errorf("Failed to get topological position for streaming token %v", streamToken)
 				return jsonerror.InternalServerError()
@@ -154,7 +166,7 @@ func OnIncomingMessagesRequest(
 					JSON: jsonerror.InvalidArgumentValue("Invalid to parameter: " + err.Error()),
 				}
 			} else {
-				to, err = db.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, !backwardOrdering)
+				to, err = snapshot.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, !backwardOrdering)
 				if err != nil {
 					logrus.WithError(err).Errorf("Failed to get topological position for streaming token %v", streamToken)
 					return jsonerror.InternalServerError()
@@ -165,7 +177,7 @@ func OnIncomingMessagesRequest(
 		// If "to" isn't provided, it defaults to either the earliest stream
 		// position (if we're going backward) or to the latest one (if we're
 		// going forward).
-		to, err = setToDefault(req.Context(), db, backwardOrdering, roomID)
+		to, err = setToDefault(req.Context(), snapshot, backwardOrdering, roomID)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("setToDefault failed")
 			return jsonerror.InternalServerError()
@@ -186,6 +198,7 @@ func OnIncomingMessagesRequest(
 	mReq := messagesReq{
 		ctx:              req.Context(),
 		db:               db,
+		snapshot:         snapshot,
 		rsAPI:            rsAPI,
 		cfg:              cfg,
 		roomID:           roomID,
@@ -217,7 +230,7 @@ func OnIncomingMessagesRequest(
 		Start: start.String(),
 		End:   end.String(),
 	}
-	res.applyLazyLoadMembers(req.Context(), db, roomID, device, filter.LazyLoadMembers, lazyLoadCache)
+	res.applyLazyLoadMembers(req.Context(), snapshot, roomID, device, filter.LazyLoadMembers, lazyLoadCache)
 
 	// If we didn't return any events, set the end to an empty string, so it will be omitted
 	// in the response JSON.
@@ -229,6 +242,7 @@ func OnIncomingMessagesRequest(
 	}
 
 	// Respond with the events.
+	succeeded = true
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: res,
@@ -239,7 +253,7 @@ func OnIncomingMessagesRequest(
 // LazyLoadMembers enabled.
 func (m *messagesResp) applyLazyLoadMembers(
 	ctx context.Context,
-	db storage.Database,
+	db storage.DatabaseTransaction,
 	roomID string,
 	device *userapi.Device,
 	lazyLoad bool,
@@ -292,7 +306,7 @@ func (r *messagesReq) retrieveEvents() (
 	end types.TopologyToken, err error,
 ) {
 	// Retrieve the events from the local database.
-	streamEvents, err := r.db.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering)
+	streamEvents, err := r.snapshot.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering)
 	if err != nil {
 		err = fmt.Errorf("GetEventsInRange: %w", err)
 		return
@@ -348,7 +362,7 @@ func (r *messagesReq) retrieveEvents() (
 
 	// Apply room history visibility filter
 	startTime := time.Now()
-	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.db, r.rsAPI, events, nil, r.device.UserID, "messages")
+	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.device.UserID, "messages")
 	logrus.WithFields(logrus.Fields{
 		"duration":      time.Since(startTime),
 		"room_id":       r.roomID,
@@ -366,7 +380,7 @@ func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (st
 			// else to go. This seems to fix Element iOS from looping on /messages endlessly.
 			end = types.TopologyToken{}
 		} else {
-			end, err = r.db.EventPositionInTopology(
+			end, err = r.snapshot.EventPositionInTopology(
 				r.ctx, events[0].EventID(),
 			)
 			// A stream/topological position is a cursor located between two events.
@@ -378,7 +392,7 @@ func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (st
 		}
 	} else {
 		start = *r.from
-		end, err = r.db.EventPositionInTopology(
+		end, err = r.snapshot.EventPositionInTopology(
 			r.ctx, events[len(events)-1].EventID(),
 		)
 	}
@@ -399,7 +413,7 @@ func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (st
 func (r *messagesReq) handleEmptyEventsSlice() (
 	events []*gomatrixserverlib.HeaderedEvent, err error,
 ) {
-	backwardExtremities, err := r.db.BackwardExtremitiesForRoom(r.ctx, r.roomID)
+	backwardExtremities, err := r.snapshot.BackwardExtremitiesForRoom(r.ctx, r.roomID)
 
 	// Check if we have backward extremities for this room.
 	if len(backwardExtremities) > 0 {
@@ -443,7 +457,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent
 	}
 
 	// Check if the slice contains a backward extremity.
-	backwardExtremities, err := r.db.BackwardExtremitiesForRoom(r.ctx, r.roomID)
+	backwardExtremities, err := r.snapshot.BackwardExtremitiesForRoom(r.ctx, r.roomID)
 	if err != nil {
 		return
 	}
@@ -463,7 +477,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent
 	}
 
 	// Append the events ve previously retrieved locally.
-	events = append(events, r.db.StreamEventsToEvents(nil, streamEvents)...)
+	events = append(events, r.snapshot.StreamEventsToEvents(nil, streamEvents)...)
 	sort.Sort(eventsByDepth(events))
 
 	return
@@ -553,7 +567,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 // Returns an error if there was an issue with retrieving the latest position
 // from the database
 func setToDefault(
-	ctx context.Context, db storage.Database, backwardOrdering bool,
+	ctx context.Context, snapshot storage.DatabaseTransaction, backwardOrdering bool,
 	roomID string,
 ) (to types.TopologyToken, err error) {
 	if backwardOrdering {
@@ -561,7 +575,7 @@ func setToDefault(
 		// this is because Database.GetEventsInTopologicalRange is exclusive of the lower-bound.
 		to = types.TopologyToken{}
 	} else {
-		to, err = db.MaxTopologicalPosition(ctx, roomID)
+		to, err = snapshot.MaxTopologicalPosition(ctx, roomID)
 	}
 
 	return
diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go
index 341efeb14..bac534a2c 100644
--- a/syncapi/routing/search.go
+++ b/syncapi/routing/search.go
@@ -61,8 +61,14 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 		searchReq.SearchCategories.RoomEvents.Filter.Limit = 5
 	}
 
+	snapshot, err := syncDB.NewDatabaseSnapshot(req.Context())
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	defer snapshot.Rollback() // nolint:errcheck
+
 	// only search rooms the user is actually joined to
-	joinedRooms, err := syncDB.RoomIDsWithMembership(ctx, device.UserID, "join")
+	joinedRooms, err := snapshot.RoomIDsWithMembership(ctx, device.UserID, "join")
 	if err != nil {
 		return jsonerror.InternalServerError()
 	}
@@ -161,12 +167,12 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 
 	stateForRooms := make(map[string][]gomatrixserverlib.ClientEvent)
 	for _, event := range evs {
-		eventsBefore, eventsAfter, err := contextEvents(ctx, syncDB, event, roomFilter, searchReq)
+		eventsBefore, eventsAfter, err := contextEvents(ctx, snapshot, event, roomFilter, searchReq)
 		if err != nil {
 			logrus.WithError(err).Error("failed to get context events")
 			return jsonerror.InternalServerError()
 		}
-		startToken, endToken, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter)
+		startToken, endToken, err := getStartEnd(ctx, snapshot, eventsBefore, eventsAfter)
 		if err != nil {
 			logrus.WithError(err).Error("failed to get start/end")
 			return jsonerror.InternalServerError()
@@ -176,7 +182,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 		for _, ev := range append(eventsBefore, eventsAfter...) {
 			profile, ok := knownUsersProfiles[event.Sender()]
 			if !ok {
-				stateEvent, err := syncDB.GetStateEvent(ctx, ev.RoomID(), gomatrixserverlib.MRoomMember, ev.Sender())
+				stateEvent, err := snapshot.GetStateEvent(ctx, ev.RoomID(), gomatrixserverlib.MRoomMember, ev.Sender())
 				if err != nil {
 					logrus.WithError(err).WithField("user_id", event.Sender()).Warn("failed to query userprofile")
 					continue
@@ -209,7 +215,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 		groups[event.RoomID()] = roomGroup
 		if _, ok := stateForRooms[event.RoomID()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok {
 			stateFilter := gomatrixserverlib.DefaultStateFilter()
-			state, err := syncDB.CurrentState(ctx, event.RoomID(), &stateFilter, nil)
+			state, err := snapshot.CurrentState(ctx, event.RoomID(), &stateFilter, nil)
 			if err != nil {
 				logrus.WithError(err).Error("unable to get current state")
 				return jsonerror.InternalServerError()
@@ -252,24 +258,24 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 // contextEvents returns the events around a given eventID
 func contextEvents(
 	ctx context.Context,
-	syncDB storage.Database,
+	snapshot storage.DatabaseTransaction,
 	event *gomatrixserverlib.HeaderedEvent,
 	roomFilter *gomatrixserverlib.RoomEventFilter,
 	searchReq SearchRequest,
 ) ([]*gomatrixserverlib.HeaderedEvent, []*gomatrixserverlib.HeaderedEvent, error) {
-	id, _, err := syncDB.SelectContextEvent(ctx, event.RoomID(), event.EventID())
+	id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID(), event.EventID())
 	if err != nil {
 		logrus.WithError(err).Error("failed to query context event")
 		return nil, nil, err
 	}
 	roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.BeforeLimit
-	eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter)
+	eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter)
 	if err != nil {
 		logrus.WithError(err).Error("failed to query before context event")
 		return nil, nil, err
 	}
 	roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.AfterLimit
-	_, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter)
+	_, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter)
 	if err != nil {
 		logrus.WithError(err).Error("failed to query after context event")
 		return nil, nil, err
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index dd03365e9..3732e43fb 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -17,19 +17,17 @@ package storage
 import (
 	"context"
 
-	"github.com/matrix-org/dendrite/internal/eventutil"
-
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage/shared"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
-type Database interface {
-	Presence
+type DatabaseTransaction interface {
 	SharedUsers
-	Notifications
 
 	MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error)
 	MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error)
@@ -37,6 +35,7 @@ type Database interface {
 	MaxStreamPositionForAccountData(ctx context.Context) (types.StreamPosition, error)
 	MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error)
 	MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error)
+	MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
 
 	CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error)
 	GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error)
@@ -44,23 +43,77 @@ type Database interface {
 	RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error)
 	MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error)
 	GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error)
-
 	RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error)
-
 	GetBackwardTopologyPos(ctx context.Context, events []types.StreamEvent) (types.TopologyToken, error)
 	PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error)
-
-	InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error)
+	InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error)
 	PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error)
 	RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error)
-
 	// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs.
 	AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error)
 	// AllJoinedUsersInRoom returns a map of room ID to a list of all joined user IDs for a given room.
 	AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error)
-
 	// AllPeekingDevicesInRooms returns a map of room ID to a list of all peeking devices.
 	AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error)
+	// Events lookups a list of event by their event ID.
+	// Returns a list of events matching the requested IDs found in the database.
+	// If an event is not found in the database then it will be omitted from the list.
+	// Returns an error if there was a problem talking with the database.
+	// Does not include any transaction IDs in the returned events.
+	Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error)
+	// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key
+	// If no event could be found, returns nil
+	// If there was an issue during the retrieval, returns an error
+	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
+	// GetStateEventsForRoom fetches the state events for a given room.
+	// Returns an empty slice if no state events could be found for this room.
+	// Returns an error if there was an issue with the retrieval.
+	GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error)
+	// GetAccountDataInRange returns all account data for a given user inserted or
+	// updated between two given positions
+	// Returns a map following the format data[roomID] = []dataTypes
+	// If no data is retrieved, returns an empty map
+	// If there was an issue with the retrieval, returns an error
+	GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, types.StreamPosition, error)
+	// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last.
+	GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error)
+	// EventPositionInTopology returns the depth and stream position of the given event.
+	EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error)
+	// BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events.
+	BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities map[string][]string, err error)
+	// MaxTopologicalPosition returns the highest topological position for a given room.
+	MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error)
+	// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and
+	// matches the streamevent.transactionID device then the transaction ID gets
+	// added to the unsigned section of the output event.
+	StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent
+	// SendToDeviceUpdatesForSync returns a list of send-to-device updates. It returns the
+	// relevant events within the given ranges for the supplied user ID and device ID.
+	SendToDeviceUpdatesForSync(ctx context.Context, userID, deviceID string, from, to types.StreamPosition) (pos types.StreamPosition, events []types.SendToDeviceEvent, err error)
+	// GetRoomReceipts gets all receipts for a given roomID
+	GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error)
+	SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error)
+	SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error)
+	SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error)
+	StreamToTopologicalPosition(ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool) (types.TopologyToken, error)
+	IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error)
+	// SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found
+	// returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty
+	// string as the membership.
+	SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
+	// getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms
+	GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error)
+	GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
+	PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
+}
+
+type Database interface {
+	Presence
+	Notifications
+
+	NewDatabaseSnapshot(ctx context.Context) (*shared.DatabaseTransaction, error)
+	NewDatabaseTransaction(ctx context.Context) (*shared.DatabaseTransaction, error)
+
 	// Events lookups a list of event by their event ID.
 	// Returns a list of events matching the requested IDs found in the database.
 	// If an event is not found in the database then it will be omitted from the list.
@@ -77,20 +130,6 @@ type Database interface {
 	// PurgeRoomState completely purges room state from the sync API. This is done when
 	// receiving an output event that completely resets the state.
 	PurgeRoomState(ctx context.Context, roomID string) error
-	// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key
-	// If no event could be found, returns nil
-	// If there was an issue during the retrieval, returns an error
-	GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
-	// GetStateEventsForRoom fetches the state events for a given room.
-	// Returns an empty slice if no state events could be found for this room.
-	// Returns an error if there was an issue with the retrieval.
-	GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error)
-	// GetAccountDataInRange returns all account data for a given user inserted or
-	// updated between two given positions
-	// Returns a map following the format data[roomID] = []dataTypes
-	// If no data is retrieved, returns an empty map
-	// If there was an issue with the retrieval, returns an error
-	GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, types.StreamPosition, error)
 	// UpsertAccountData keeps track of new or updated account data, by saving the type
 	// of the new/updated data, and the user ID and room ID the data is related to (empty)
 	// room ID means the data isn't specific to any room)
@@ -114,21 +153,6 @@ type Database interface {
 	// DeletePeek deletes all peeks for a given room by a given user
 	// Returns an error if there was a problem communicating with the database.
 	DeletePeeks(ctx context.Context, RoomID, UserID string) (types.StreamPosition, error)
-	// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last.
-	GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error)
-	// EventPositionInTopology returns the depth and stream position of the given event.
-	EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error)
-	// BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events.
-	BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities map[string][]string, err error)
-	// MaxTopologicalPosition returns the highest topological position for a given room.
-	MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error)
-	// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and
-	// matches the streamevent.transactionID device then the transaction ID gets
-	// added to the unsigned section of the output event.
-	StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent
-	// SendToDeviceUpdatesForSync returns a list of send-to-device updates. It returns the
-	// relevant events within the given ranges for the supplied user ID and device ID.
-	SendToDeviceUpdatesForSync(ctx context.Context, userID, deviceID string, from, to types.StreamPosition) (pos types.StreamPosition, events []types.SendToDeviceEvent, err error)
 	// StoreNewSendForDeviceMessage stores a new send-to-device event for a user's device.
 	StoreNewSendForDeviceMessage(ctx context.Context, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent) (types.StreamPosition, error)
 	// CleanSendToDeviceUpdates removes all send-to-device messages BEFORE the specified
@@ -146,29 +170,13 @@ type Database interface {
 	RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *gomatrixserverlib.HeaderedEvent) error
 	// StoreReceipt stores new receipt events
 	StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error)
-	// GetRoomReceipts gets all receipts for a given roomID
-	GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error)
-
-	SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error)
-	SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error)
-	SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error)
-
-	StreamToTopologicalPosition(ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool) (types.TopologyToken, error)
-
-	IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error)
 	UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error
-	// SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found
-	// returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty
-	// string as the membership.
-	SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error)
 	ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error)
 }
 
 type Presence interface {
-	UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error)
 	GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
-	PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
-	MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
+	UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error)
 }
 
 type SharedUsers interface {
@@ -179,7 +187,4 @@ type SharedUsers interface {
 type Notifications interface {
 	// UpsertRoomUnreadNotificationCounts updates the notification statistics about a (user, room) key.
 	UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (types.StreamPosition, error)
-
-	// getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms
-	GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error)
 }
diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go
index f87ccf965..aada70d5e 100644
--- a/syncapi/storage/postgres/invites_table.go
+++ b/syncapi/storage/postgres/invites_table.go
@@ -55,7 +55,7 @@ const deleteInviteEventSQL = "" +
 	"UPDATE syncapi_invite_events SET deleted=TRUE, id=nextval('syncapi_stream_id') WHERE event_id = $1 AND deleted=FALSE RETURNING id"
 
 const selectInviteEventsInRangeSQL = "" +
-	"SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" +
+	"SELECT id, room_id, headered_event_json, deleted FROM syncapi_invite_events" +
 	" WHERE target_user_id = $1 AND id > $2 AND id <= $3" +
 	" ORDER BY id DESC"
 
@@ -121,23 +121,28 @@ func (s *inviteEventsStatements) DeleteInviteEvent(
 // active invites for the target user ID in the supplied range.
 func (s *inviteEventsStatements) SelectInviteEventsInRange(
 	ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range,
-) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) {
+) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) {
+	var lastPos types.StreamPosition
 	stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt)
 	rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High())
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, lastPos, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed")
 	result := map[string]*gomatrixserverlib.HeaderedEvent{}
 	retired := map[string]*gomatrixserverlib.HeaderedEvent{}
 	for rows.Next() {
 		var (
+			id        types.StreamPosition
 			roomID    string
 			eventJSON []byte
 			deleted   bool
 		)
-		if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil {
-			return nil, nil, err
+		if err = rows.Scan(&id, &roomID, &eventJSON, &deleted); err != nil {
+			return nil, nil, lastPos, err
+		}
+		if id > lastPos {
+			lastPos = id
 		}
 
 		// if we have seen this room before, it has a higher stream position and hence takes priority
@@ -150,7 +155,7 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange(
 
 		var event *gomatrixserverlib.HeaderedEvent
 		if err := json.Unmarshal(eventJSON, &event); err != nil {
-			return nil, nil, err
+			return nil, nil, lastPos, err
 		}
 
 		if deleted {
@@ -159,7 +164,10 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange(
 			result[roomID] = event
 		}
 	}
-	return result, retired, rows.Err()
+	if lastPos == 0 {
+		lastPos = r.To
+	}
+	return result, retired, lastPos, rows.Err()
 }
 
 func (s *inviteEventsStatements) SelectMaxInviteID(
diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go
new file mode 100644
index 000000000..fb3b295e9
--- /dev/null
+++ b/syncapi/storage/shared/storage_consumer.go
@@ -0,0 +1,586 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+
+	"github.com/tidwall/gjson"
+
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+// Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite
+// For now this contains the shared functions
+type Database struct {
+	DB                  *sql.DB
+	Writer              sqlutil.Writer
+	Invites             tables.Invites
+	Peeks               tables.Peeks
+	AccountData         tables.AccountData
+	OutputEvents        tables.Events
+	Topology            tables.Topology
+	CurrentRoomState    tables.CurrentRoomState
+	BackwardExtremities tables.BackwardsExtremities
+	SendToDevice        tables.SendToDevice
+	Filter              tables.Filter
+	Receipts            tables.Receipts
+	Memberships         tables.Memberships
+	NotificationData    tables.NotificationData
+	Ignores             tables.Ignores
+	Presence            tables.Presence
+}
+
+func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) {
+	return d.NewDatabaseTransaction(ctx)
+
+	/*
+		TODO: Repeatable read is probably the right thing to do here,
+		but it seems to cause some problems with the invite tests, so
+		need to investigate that further.
+
+		txn, err := d.DB.BeginTx(ctx, &sql.TxOptions{
+			// Set the isolation level so that we see a snapshot of the database.
+			// In PostgreSQL repeatable read transactions will see a snapshot taken
+			// at the first query, and since the transaction is read-only it can't
+			// run into any serialisation errors.
+			// https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ
+			Isolation: sql.LevelRepeatableRead,
+			ReadOnly:  true,
+		})
+		if err != nil {
+			return nil, err
+		}
+		return &DatabaseTransaction{
+			Database: d,
+			txn:      txn,
+		}, nil
+	*/
+}
+
+func (d *Database) NewDatabaseTransaction(ctx context.Context) (*DatabaseTransaction, error) {
+	txn, err := d.DB.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &DatabaseTransaction{
+		Database: d,
+		txn:      txn,
+	}, nil
+}
+
+func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs, nil, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// We don't include a device here as we only include transaction IDs in
+	// incremental syncs.
+	return d.StreamEventsToEvents(nil, streamEvents), nil
+}
+
+// AddInviteEvent stores a new invite event for a user.
+// If the invite was successfully stored this returns the stream ID it was stored at.
+// Returns an error if there was a problem communicating with the database.
+func (d *Database) AddInviteEvent(
+	ctx context.Context, inviteEvent *gomatrixserverlib.HeaderedEvent,
+) (sp types.StreamPosition, err error) {
+	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent)
+		return err
+	})
+	return
+}
+
+// RetireInviteEvent removes an old invite event from the database.
+// Returns an error if there was a problem communicating with the database.
+func (d *Database) RetireInviteEvent(
+	ctx context.Context, inviteEventID string,
+) (sp types.StreamPosition, err error) {
+	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		sp, err = d.Invites.DeleteInviteEvent(ctx, txn, inviteEventID)
+		return err
+	})
+	return
+}
+
+// AddPeek tracks the fact that a user has started peeking.
+// If the peek was successfully stored this returns the stream ID it was stored at.
+// Returns an error if there was a problem communicating with the database.
+func (d *Database) AddPeek(
+	ctx context.Context, roomID, userID, deviceID string,
+) (sp types.StreamPosition, err error) {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		sp, err = d.Peeks.InsertPeek(ctx, txn, roomID, userID, deviceID)
+		return err
+	})
+	return
+}
+
+// DeletePeek tracks the fact that a user has stopped peeking from the specified
+// device. If the peeks was successfully deleted this returns the stream ID it was
+// stored at. Returns an error if there was a problem communicating with the database.
+func (d *Database) DeletePeek(
+	ctx context.Context, roomID, userID, deviceID string,
+) (sp types.StreamPosition, err error) {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		sp, err = d.Peeks.DeletePeek(ctx, txn, roomID, userID, deviceID)
+		return err
+	})
+	if err == sql.ErrNoRows {
+		sp = 0
+		err = nil
+	}
+	return
+}
+
+// DeletePeeks tracks the fact that a user has stopped peeking from all devices
+// If the peeks was successfully deleted this returns the stream ID it was stored at.
+// Returns an error if there was a problem communicating with the database.
+func (d *Database) DeletePeeks(
+	ctx context.Context, roomID, userID string,
+) (sp types.StreamPosition, err error) {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		sp, err = d.Peeks.DeletePeeks(ctx, txn, roomID, userID)
+		return err
+	})
+	if err == sql.ErrNoRows {
+		sp = 0
+		err = nil
+	}
+	return
+}
+
+// UpsertAccountData keeps track of new or updated account data, by saving the type
+// of the new/updated data, and the user ID and room ID the data is related to (empty)
+// room ID means the data isn't specific to any room)
+// If no data with the given type, user ID and room ID exists in the database,
+// creates a new row, else update the existing one
+// Returns an error if there was an issue with the upsert
+func (d *Database) UpsertAccountData(
+	ctx context.Context, userID, roomID, dataType string,
+) (sp types.StreamPosition, err error) {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType)
+		return err
+	})
+	return
+}
+
+func (d *Database) StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent {
+	out := make([]*gomatrixserverlib.HeaderedEvent, len(in))
+	for i := 0; i < len(in); i++ {
+		out[i] = in[i].HeaderedEvent
+		if device != nil && in[i].TransactionID != nil {
+			if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID {
+				err := out[i].SetUnsignedField(
+					"transaction_id", in[i].TransactionID.TransactionID,
+				)
+				if err != nil {
+					logrus.WithFields(logrus.Fields{
+						"event_id": out[i].EventID(),
+					}).WithError(err).Warnf("Failed to add transaction ID to event")
+				}
+			}
+		}
+	}
+	return out
+}
+
+// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of
+// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table
+// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such.
+// This function should always be called within a sqlutil.Writer for safety in SQLite.
+func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error {
+	if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil {
+		return err
+	}
+
+	// Check if we have all of the event's previous events. If an event is
+	// missing, add it to the room's backward extremities.
+	prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs(), nil, false)
+	if err != nil {
+		return err
+	}
+	var found bool
+	for _, eID := range ev.PrevEventIDs() {
+		found = false
+		for _, prevEv := range prevEvents {
+			if eID == prevEv.EventID() {
+				found = true
+			}
+		}
+
+		// If the event is missing, consider it a backward extremity.
+		if !found {
+			if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (d *Database) PurgeRoomState(
+	ctx context.Context, roomID string,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		// If the event is a create event then we'll delete all of the existing
+		// data for the room. The only reason that a create event would be replayed
+		// to us in this way is if we're about to receive the entire room state.
+		if err := d.CurrentRoomState.DeleteRoomStateForRoom(ctx, txn, roomID); err != nil {
+			return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateForRoom: %w", err)
+		}
+		return nil
+	})
+}
+
+func (d *Database) WriteEvent(
+	ctx context.Context,
+	ev *gomatrixserverlib.HeaderedEvent,
+	addStateEvents []*gomatrixserverlib.HeaderedEvent,
+	addStateEventIDs, removeStateEventIDs []string,
+	transactionID *api.TransactionID, excludeFromSync bool,
+	historyVisibility gomatrixserverlib.HistoryVisibility,
+) (pduPosition types.StreamPosition, returnErr error) {
+	returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		var err error
+		ev.Visibility = historyVisibility
+		pos, err := d.OutputEvents.InsertEvent(
+			ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, historyVisibility,
+		)
+		if err != nil {
+			return fmt.Errorf("d.OutputEvents.InsertEvent: %w", err)
+		}
+		pduPosition = pos
+		var topoPosition types.StreamPosition
+		if topoPosition, err = d.Topology.InsertEventInTopology(ctx, txn, ev, pos); err != nil {
+			return fmt.Errorf("d.Topology.InsertEventInTopology: %w", err)
+		}
+
+		if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil {
+			return fmt.Errorf("d.handleBackwardExtremities: %w", err)
+		}
+
+		if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 {
+			// Nothing to do, the event may have just been a message event.
+			return nil
+		}
+		for i := range addStateEvents {
+			addStateEvents[i].Visibility = historyVisibility
+		}
+		return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition)
+	})
+
+	return pduPosition, returnErr
+}
+
+// This function should always be called within a sqlutil.Writer for safety in SQLite.
+func (d *Database) updateRoomState(
+	ctx context.Context, txn *sql.Tx,
+	removedEventIDs []string,
+	addedEvents []*gomatrixserverlib.HeaderedEvent,
+	pduPosition types.StreamPosition,
+	topoPosition types.StreamPosition,
+) error {
+	// remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add.
+	for _, eventID := range removedEventIDs {
+		if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil {
+			return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateByEventID: %w", err)
+		}
+	}
+
+	for _, event := range addedEvents {
+		if event.StateKey() == nil {
+			// ignore non state events
+			continue
+		}
+		var membership *string
+		if event.Type() == "m.room.member" {
+			value, err := event.Membership()
+			if err != nil {
+				return fmt.Errorf("event.Membership: %w", err)
+			}
+			membership = &value
+			if err = d.Memberships.UpsertMembership(ctx, txn, event, pduPosition, topoPosition); err != nil {
+				return fmt.Errorf("d.Memberships.UpsertMembership: %w", err)
+			}
+		}
+
+		if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil {
+			return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (d *Database) GetFilter(
+	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
+) error {
+	return d.Filter.SelectFilter(ctx, nil, target, localpart, filterID)
+}
+
+func (d *Database) PutFilter(
+	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
+) (string, error) {
+	var filterID string
+	var err error
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		filterID, err = d.Filter.InsertFilter(ctx, txn, filter, localpart)
+		return err
+	})
+	return filterID, err
+}
+
+func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *gomatrixserverlib.HeaderedEvent) error {
+	redactedEvents, err := d.Events(ctx, []string{redactedEventID})
+	if err != nil {
+		return err
+	}
+	if len(redactedEvents) == 0 {
+		logrus.WithField("event_id", redactedEventID).WithField("redaction_event", redactedBecause.EventID()).Warnf("missing redacted event for redaction")
+		return nil
+	}
+	eventToRedact := redactedEvents[0].Unwrap()
+	redactionEvent := redactedBecause.Unwrap()
+	if err = eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil {
+		return err
+	}
+
+	newEvent := eventToRedact.Headered(redactedBecause.RoomVersion)
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.OutputEvents.UpdateEventJSON(ctx, txn, newEvent)
+	})
+	return err
+}
+
+// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database.
+// Returns a map of room ID to list of events.
+func (d *Database) fetchStateEvents(
+	ctx context.Context, txn *sql.Tx,
+	roomIDToEventIDSet map[string]map[string]bool,
+	eventIDToEvent map[string]types.StreamEvent,
+) (map[string][]types.StreamEvent, error) {
+	stateBetween := make(map[string][]types.StreamEvent)
+	missingEvents := make(map[string][]string)
+	for roomID, ids := range roomIDToEventIDSet {
+		events := stateBetween[roomID]
+		for id, need := range ids {
+			if !need {
+				continue // deleted state
+			}
+			e, ok := eventIDToEvent[id]
+			if ok {
+				events = append(events, e)
+			} else {
+				m := missingEvents[roomID]
+				m = append(m, id)
+				missingEvents[roomID] = m
+			}
+		}
+		stateBetween[roomID] = events
+	}
+
+	if len(missingEvents) > 0 {
+		// This happens when add_state_ids has an event ID which is not in the provided range.
+		// We need to explicitly fetch them.
+		allMissingEventIDs := []string{}
+		for _, missingEvIDs := range missingEvents {
+			allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...)
+		}
+		evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs)
+		if err != nil {
+			return nil, err
+		}
+		// we know we got them all otherwise an error would've been returned, so just loop the events
+		for _, ev := range evs {
+			roomID := ev.RoomID()
+			stateBetween[roomID] = append(stateBetween[roomID], ev)
+		}
+	}
+	return stateBetween, nil
+}
+
+func (d *Database) fetchMissingStateEvents(
+	ctx context.Context, txn *sql.Tx, eventIDs []string,
+) ([]types.StreamEvent, error) {
+	// Fetch from the events table first so we pick up the stream ID for the
+	// event.
+	events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs, nil, false)
+	if err != nil {
+		return nil, err
+	}
+
+	have := map[string]bool{}
+	for _, event := range events {
+		have[event.EventID()] = true
+	}
+	var missing []string
+	for _, eventID := range eventIDs {
+		if !have[eventID] {
+			missing = append(missing, eventID)
+		}
+	}
+	if len(missing) == 0 {
+		return events, nil
+	}
+
+	// If they are missing from the events table then they should be state
+	// events that we received from outside the main event stream.
+	// These should be in the room state table.
+	stateEvents, err := d.CurrentRoomState.SelectEventsWithEventIDs(ctx, txn, missing)
+
+	if err != nil {
+		return nil, err
+	}
+	if len(stateEvents) != len(missing) {
+		logrus.WithContext(ctx).Warnf("Failed to map all event IDs to events (got %d, wanted %d)", len(stateEvents), len(missing))
+
+		// TODO: Why is this happening? It's probably the roomserver. Uncomment
+		// this error again when we work out what it is and fix it, otherwise we
+		// just end up returning lots of 500s to the client and that breaks
+		// pretty much everything, rather than just sending what we have.
+		//return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing))
+	}
+	events = append(events, stateEvents...)
+	return events, nil
+}
+
+func (d *Database) StoreNewSendForDeviceMessage(
+	ctx context.Context, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent,
+) (newPos types.StreamPosition, err error) {
+	j, err := json.Marshal(event)
+	if err != nil {
+		return 0, err
+	}
+	// Delegate the database write task to the SendToDeviceWriter. It'll guarantee
+	// that we don't lock the table for writes in more than one place.
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		newPos, err = d.SendToDevice.InsertSendToDeviceMessage(
+			ctx, txn, userID, deviceID, string(j),
+		)
+		return err
+	})
+	if err != nil {
+		return 0, err
+	}
+	return newPos, nil
+}
+
+func (d *Database) CleanSendToDeviceUpdates(
+	ctx context.Context,
+	userID, deviceID string, before types.StreamPosition,
+) (err error) {
+	if err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.SendToDevice.DeleteSendToDeviceMessages(ctx, txn, userID, deviceID, before)
+	}); err != nil {
+		logrus.WithError(err).Errorf("Failed to clean up old send-to-device messages for user %q device %q", userID, deviceID)
+		return err
+	}
+	return nil
+}
+
+// getMembershipFromEvent returns the value of content.membership iff the event is a state event
+// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned.
+func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) (string, string) {
+	if ev.Type() != "m.room.member" || !ev.StateKeyEquals(userID) {
+		return "", ""
+	}
+	membership, err := ev.Membership()
+	if err != nil {
+		return "", ""
+	}
+	prevMembership := gjson.GetBytes(ev.Unsigned(), "prev_content.membership").Str
+	return membership, prevMembership
+}
+
+// StoreReceipt stores user receipts
+func (d *Database) StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		pos, err = d.Receipts.UpsertReceipt(ctx, txn, roomId, receiptType, userId, eventId, timestamp)
+		return err
+	})
+	return
+}
+
+func (d *Database) UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		pos, err = d.NotificationData.UpsertRoomUnreadCounts(ctx, txn, userID, roomID, notificationCount, highlightCount)
+		return err
+	})
+	return
+}
+
+func (d *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) {
+	return d.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID)
+}
+
+func (d *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	return d.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter)
+}
+func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) {
+	return d.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter)
+}
+
+func (d *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) {
+	return d.Ignores.SelectIgnores(ctx, nil, userID)
+}
+
+func (d *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.Ignores.UpsertIgnores(ctx, txn, userID, ignores)
+	})
+}
+
+func (d *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
+	var pos types.StreamPosition
+	var err error
+	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		pos, err = d.Presence.UpsertPresence(ctx, txn, userID, statusMsg, presence, lastActiveTS, fromSync)
+		return nil
+	})
+	return pos, err
+}
+
+func (d *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
+	return d.Presence.GetPresenceForUser(ctx, nil, userID)
+}
+
+func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) {
+	return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos)
+}
+
+func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) {
+	return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{
+		gomatrixserverlib.MRoomName,
+		gomatrixserverlib.MRoomTopic,
+		"m.room.message",
+	})
+}
diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go
new file mode 100644
index 000000000..a19135a69
--- /dev/null
+++ b/syncapi/storage/shared/storage_sync.go
@@ -0,0 +1,574 @@
+package shared
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type DatabaseTransaction struct {
+	*Database
+	txn *sql.Tx
+}
+
+func (d *DatabaseTransaction) Commit() error {
+	if d.txn == nil {
+		return nil
+	}
+	return d.txn.Commit()
+}
+
+func (d *DatabaseTransaction) Rollback() error {
+	if d.txn == nil {
+		return nil
+	}
+	return d.txn.Rollback()
+}
+
+func (d *DatabaseTransaction) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) {
+	id, err := d.OutputEvents.SelectMaxEventID(ctx, d.txn)
+	if err != nil {
+		return 0, fmt.Errorf("d.OutputEvents.SelectMaxEventID: %w", err)
+	}
+	return types.StreamPosition(id), nil
+}
+
+func (d *DatabaseTransaction) MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error) {
+	id, err := d.Receipts.SelectMaxReceiptID(ctx, d.txn)
+	if err != nil {
+		return 0, fmt.Errorf("d.Receipts.SelectMaxReceiptID: %w", err)
+	}
+	return types.StreamPosition(id), nil
+}
+
+func (d *DatabaseTransaction) MaxStreamPositionForInvites(ctx context.Context) (types.StreamPosition, error) {
+	id, err := d.Invites.SelectMaxInviteID(ctx, d.txn)
+	if err != nil {
+		return 0, fmt.Errorf("d.Invites.SelectMaxInviteID: %w", err)
+	}
+	return types.StreamPosition(id), nil
+}
+
+func (d *DatabaseTransaction) MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error) {
+	id, err := d.SendToDevice.SelectMaxSendToDeviceMessageID(ctx, d.txn)
+	if err != nil {
+		return 0, fmt.Errorf("d.SendToDevice.SelectMaxSendToDeviceMessageID: %w", err)
+	}
+	return types.StreamPosition(id), nil
+}
+
+func (d *DatabaseTransaction) MaxStreamPositionForAccountData(ctx context.Context) (types.StreamPosition, error) {
+	id, err := d.AccountData.SelectMaxAccountDataID(ctx, d.txn)
+	if err != nil {
+		return 0, fmt.Errorf("d.Invites.SelectMaxAccountDataID: %w", err)
+	}
+	return types.StreamPosition(id), nil
+}
+
+func (d *DatabaseTransaction) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) {
+	id, err := d.NotificationData.SelectMaxID(ctx, d.txn)
+	if err != nil {
+		return 0, fmt.Errorf("d.NotificationData.SelectMaxID: %w", err)
+	}
+	return types.StreamPosition(id), nil
+}
+
+func (d *DatabaseTransaction) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	return d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilterPart, excludeEventIDs)
+}
+
+func (d *DatabaseTransaction) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) {
+	return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, d.txn, userID, membership)
+}
+
+func (d *DatabaseTransaction) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) {
+	return d.Memberships.SelectMembershipCount(ctx, d.txn, roomID, membership, pos)
+}
+
+func (d *DatabaseTransaction) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) {
+	return d.Memberships.SelectHeroes(ctx, d.txn, roomID, userID, memberships)
+}
+
+func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) {
+	return d.OutputEvents.SelectRecentEvents(ctx, d.txn, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents)
+}
+
+func (d *DatabaseTransaction) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) {
+	return d.Topology.SelectPositionInTopology(ctx, d.txn, eventID)
+}
+
+func (d *DatabaseTransaction) InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) {
+	return d.Invites.SelectInviteEventsInRange(ctx, d.txn, targetUserID, r)
+}
+
+func (d *DatabaseTransaction) PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error) {
+	return d.Peeks.SelectPeeksInRange(ctx, d.txn, userID, deviceID, r)
+}
+
+func (d *DatabaseTransaction) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) {
+	return d.Receipts.SelectRoomReceiptsAfter(ctx, d.txn, roomIDs, streamPos)
+}
+
+// Events lookups a list of event by their event ID.
+// Returns a list of events matching the requested IDs found in the database.
+// If an event is not found in the database then it will be omitted from the list.
+// Returns an error if there was a problem talking with the database.
+// Does not include any transaction IDs in the returned events.
+func (d *DatabaseTransaction) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	streamEvents, err := d.OutputEvents.SelectEvents(ctx, d.txn, eventIDs, nil, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// We don't include a device here as we only include transaction IDs in
+	// incremental syncs.
+	return d.StreamEventsToEvents(nil, streamEvents), nil
+}
+
+func (d *DatabaseTransaction) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) {
+	return d.CurrentRoomState.SelectJoinedUsers(ctx, d.txn)
+}
+
+func (d *DatabaseTransaction) AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) {
+	return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, d.txn, roomIDs)
+}
+
+func (d *DatabaseTransaction) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) {
+	return d.Peeks.SelectPeekingDevices(ctx, d.txn)
+}
+
+func (d *DatabaseTransaction) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) {
+	return d.CurrentRoomState.SelectSharedUsers(ctx, d.txn, userID, otherUserIDs)
+}
+
+func (d *DatabaseTransaction) GetStateEvent(
+	ctx context.Context, roomID, evType, stateKey string,
+) (*gomatrixserverlib.HeaderedEvent, error) {
+	return d.CurrentRoomState.SelectStateEvent(ctx, d.txn, roomID, evType, stateKey)
+}
+
+func (d *DatabaseTransaction) GetStateEventsForRoom(
+	ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter,
+) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) {
+	stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilter, nil)
+	return
+}
+
+// GetAccountDataInRange returns all account data for a given user inserted or
+// updated between two given positions
+// Returns a map following the format data[roomID] = []dataTypes
+// If no data is retrieved, returns an empty map
+// If there was an issue with the retrieval, returns an error
+func (d *DatabaseTransaction) GetAccountDataInRange(
+	ctx context.Context, userID string, r types.Range,
+	accountDataFilterPart *gomatrixserverlib.EventFilter,
+) (map[string][]string, types.StreamPosition, error) {
+	return d.AccountData.SelectAccountDataInRange(ctx, d.txn, userID, r, accountDataFilterPart)
+}
+
+func (d *DatabaseTransaction) GetEventsInTopologicalRange(
+	ctx context.Context,
+	from, to *types.TopologyToken,
+	roomID string,
+	filter *gomatrixserverlib.RoomEventFilter,
+	backwardOrdering bool,
+) (events []types.StreamEvent, err error) {
+	var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition
+	if backwardOrdering {
+		// Backward ordering means the 'from' token has a higher depth than the 'to' token
+		minDepth = to.Depth
+		maxDepth = from.Depth
+		// for cases where we have say 5 events with the same depth, the TopologyToken needs to
+		// know which of the 5 the client has seen. This is done by using the PDU position.
+		// Events with the same maxDepth but less than this PDU position will be returned.
+		maxStreamPosForMaxDepth = from.PDUPosition
+	} else {
+		// Forward ordering means the 'from' token has a lower depth than the 'to' token.
+		minDepth = from.Depth
+		maxDepth = to.Depth
+	}
+
+	// Select the event IDs from the defined range.
+	var eIDs []string
+	eIDs, err = d.Topology.SelectEventIDsInRange(
+		ctx, d.txn, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, filter.Limit, !backwardOrdering,
+	)
+	if err != nil {
+		return
+	}
+
+	// Retrieve the events' contents using their IDs.
+	events, err = d.OutputEvents.SelectEvents(ctx, d.txn, eIDs, filter, true)
+	return
+}
+
+func (d *DatabaseTransaction) BackwardExtremitiesForRoom(
+	ctx context.Context, roomID string,
+) (backwardExtremities map[string][]string, err error) {
+	return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, d.txn, roomID)
+}
+
+func (d *DatabaseTransaction) MaxTopologicalPosition(
+	ctx context.Context, roomID string,
+) (types.TopologyToken, error) {
+	depth, streamPos, err := d.Topology.SelectMaxPositionInTopology(ctx, d.txn, roomID)
+	if err != nil {
+		return types.TopologyToken{}, err
+	}
+	return types.TopologyToken{Depth: depth, PDUPosition: streamPos}, nil
+}
+
+func (d *DatabaseTransaction) EventPositionInTopology(
+	ctx context.Context, eventID string,
+) (types.TopologyToken, error) {
+	depth, stream, err := d.Topology.SelectPositionInTopology(ctx, d.txn, eventID)
+	if err != nil {
+		return types.TopologyToken{}, err
+	}
+	return types.TopologyToken{Depth: depth, PDUPosition: stream}, nil
+}
+
+func (d *DatabaseTransaction) StreamToTopologicalPosition(
+	ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool,
+) (types.TopologyToken, error) {
+	topoPos, err := d.Topology.SelectStreamToTopologicalPosition(ctx, d.txn, roomID, streamPos, backwardOrdering)
+	switch {
+	case err == sql.ErrNoRows && backwardOrdering: // no events in range, going backward
+		return types.TopologyToken{PDUPosition: streamPos}, nil
+	case err == sql.ErrNoRows && !backwardOrdering: // no events in range, going forward
+		topoPos, streamPos, err = d.Topology.SelectMaxPositionInTopology(ctx, d.txn, roomID)
+		if err != nil {
+			return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectMaxPositionInTopology: %w", err)
+		}
+		return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil
+	case err != nil: // some other error happened
+		return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectStreamToTopologicalPosition: %w", err)
+	default:
+		return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil
+	}
+}
+
+// GetBackwardTopologyPos retrieves the backward topology position, i.e. the position of the
+// oldest event in the room's topology.
+func (d *DatabaseTransaction) GetBackwardTopologyPos(
+	ctx context.Context,
+	events []types.StreamEvent,
+) (types.TopologyToken, error) {
+	zeroToken := types.TopologyToken{}
+	if len(events) == 0 {
+		return zeroToken, nil
+	}
+	pos, spos, err := d.Topology.SelectPositionInTopology(ctx, d.txn, events[0].EventID())
+	if err != nil {
+		return zeroToken, err
+	}
+	tok := types.TopologyToken{Depth: pos, PDUPosition: spos}
+	tok.Decrement()
+	return tok, nil
+}
+
+// GetStateDeltas returns the state deltas between fromPos and toPos,
+// exclusive of oldPos, inclusive of newPos, for the rooms in which
+// the user has new membership events.
+// A list of joined room IDs is also returned in case the caller needs it.
+func (d *DatabaseTransaction) GetStateDeltas(
+	ctx context.Context, device *userapi.Device,
+	r types.Range, userID string,
+	stateFilter *gomatrixserverlib.StateFilter,
+) (deltas []types.StateDelta, joinedRoomsIDs []string, err error) {
+	// Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821
+	// - Get membership list changes for this user in this sync response
+	// - For each room which has membership list changes:
+	//     * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO).
+	//       If it is, then we need to send the full room state down (and 'limited' is always true).
+	//     * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block.
+	//     * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block.
+	// - Get all CURRENTLY joined rooms, and add them to 'joined' block.
+
+	// Look up all memberships for the user. We only care about rooms that a
+	// user has ever interacted with — joined to, kicked/banned from, left.
+	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, d.txn, userID)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
+		return nil, nil, err
+	}
+
+	allRoomIDs := make([]string, 0, len(memberships))
+	joinedRoomIDs := make([]string, 0, len(memberships))
+	for roomID, membership := range memberships {
+		allRoomIDs = append(allRoomIDs, roomID)
+		if membership == gomatrixserverlib.Join {
+			joinedRoomIDs = append(joinedRoomIDs, roomID)
+		}
+	}
+
+	// get all the state events ever (i.e. for all available rooms) between these two positions
+	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
+		return nil, nil, err
+	}
+	state, err := d.fetchStateEvents(ctx, d.txn, stateNeeded, eventMap)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
+		return nil, nil, err
+	}
+
+	// find out which rooms this user is peeking, if any.
+	// We do this before joins so any peeks get overwritten
+	peeks, err := d.Peeks.SelectPeeksInRange(ctx, d.txn, userID, device.ID, r)
+	if err != nil && err != sql.ErrNoRows {
+		return nil, nil, err
+	}
+
+	// add peek blocks
+	for _, peek := range peeks {
+		if peek.New {
+			// send full room state down instead of a delta
+			var s []types.StreamEvent
+			s, err = d.currentStateStreamEventsForRoom(ctx, peek.RoomID, stateFilter)
+			if err != nil {
+				if err == sql.ErrNoRows {
+					continue
+				}
+				return nil, nil, err
+			}
+			state[peek.RoomID] = s
+		}
+		if !peek.Deleted {
+			deltas = append(deltas, types.StateDelta{
+				Membership:  gomatrixserverlib.Peek,
+				StateEvents: d.StreamEventsToEvents(device, state[peek.RoomID]),
+				RoomID:      peek.RoomID,
+			})
+		}
+	}
+
+	// handle newly joined rooms and non-joined rooms
+	newlyJoinedRooms := make(map[string]bool, len(state))
+	for roomID, stateStreamEvents := range state {
+		for _, ev := range stateStreamEvents {
+			if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" {
+				if membership == gomatrixserverlib.Join && prevMembership != membership {
+					// send full room state down instead of a delta
+					var s []types.StreamEvent
+					s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter)
+					if err != nil {
+						if err == sql.ErrNoRows {
+							continue
+						}
+						return nil, nil, err
+					}
+					state[roomID] = s
+					newlyJoinedRooms[roomID] = true
+					continue // we'll add this room in when we do joined rooms
+				}
+
+				deltas = append(deltas, types.StateDelta{
+					Membership:    membership,
+					MembershipPos: ev.StreamPosition,
+					StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
+					RoomID:        roomID,
+				})
+				break
+			}
+		}
+	}
+
+	// Add in currently joined rooms
+	for _, joinedRoomID := range joinedRoomIDs {
+		deltas = append(deltas, types.StateDelta{
+			Membership:  gomatrixserverlib.Join,
+			StateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]),
+			RoomID:      joinedRoomID,
+			NewlyJoined: newlyJoinedRooms[joinedRoomID],
+		})
+	}
+
+	return deltas, joinedRoomIDs, nil
+}
+
+// GetStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync
+// requests with full_state=true.
+// Fetches full state for all joined rooms and uses selectStateInRange to get
+// updates for other rooms.
+func (d *DatabaseTransaction) GetStateDeltasForFullStateSync(
+	ctx context.Context, device *userapi.Device,
+	r types.Range, userID string,
+	stateFilter *gomatrixserverlib.StateFilter,
+) ([]types.StateDelta, []string, error) {
+	// Look up all memberships for the user. We only care about rooms that a
+	// user has ever interacted with — joined to, kicked/banned from, left.
+	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, d.txn, userID)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
+		return nil, nil, err
+	}
+
+	allRoomIDs := make([]string, 0, len(memberships))
+	joinedRoomIDs := make([]string, 0, len(memberships))
+	for roomID, membership := range memberships {
+		allRoomIDs = append(allRoomIDs, roomID)
+		if membership == gomatrixserverlib.Join {
+			joinedRoomIDs = append(joinedRoomIDs, roomID)
+		}
+	}
+
+	// Use a reasonable initial capacity
+	deltas := make(map[string]types.StateDelta)
+
+	peeks, err := d.Peeks.SelectPeeksInRange(ctx, d.txn, userID, device.ID, r)
+	if err != nil && err != sql.ErrNoRows {
+		return nil, nil, err
+	}
+
+	// Add full states for all peeking rooms
+	for _, peek := range peeks {
+		if !peek.Deleted {
+			s, stateErr := d.currentStateStreamEventsForRoom(ctx, peek.RoomID, stateFilter)
+			if stateErr != nil {
+				if stateErr == sql.ErrNoRows {
+					continue
+				}
+				return nil, nil, stateErr
+			}
+			deltas[peek.RoomID] = types.StateDelta{
+				Membership:  gomatrixserverlib.Peek,
+				StateEvents: d.StreamEventsToEvents(device, s),
+				RoomID:      peek.RoomID,
+			}
+		}
+	}
+
+	// Get all the state events ever between these two positions
+	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
+		return nil, nil, err
+	}
+	state, err := d.fetchStateEvents(ctx, d.txn, stateNeeded, eventMap)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, nil
+		}
+		return nil, nil, err
+	}
+
+	for roomID, stateStreamEvents := range state {
+		for _, ev := range stateStreamEvents {
+			if membership, _ := getMembershipFromEvent(ev.Event, userID); membership != "" {
+				if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above.
+					deltas[roomID] = types.StateDelta{
+						Membership:    membership,
+						MembershipPos: ev.StreamPosition,
+						StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
+						RoomID:        roomID,
+					}
+				}
+
+				break
+			}
+		}
+	}
+
+	// Add full states for all joined rooms
+	for _, joinedRoomID := range joinedRoomIDs {
+		s, stateErr := d.currentStateStreamEventsForRoom(ctx, joinedRoomID, stateFilter)
+		if stateErr != nil {
+			if stateErr == sql.ErrNoRows {
+				continue
+			}
+			return nil, nil, stateErr
+		}
+		deltas[joinedRoomID] = types.StateDelta{
+			Membership:  gomatrixserverlib.Join,
+			StateEvents: d.StreamEventsToEvents(device, s),
+			RoomID:      joinedRoomID,
+		}
+	}
+
+	// Create a response array.
+	result := make([]types.StateDelta, len(deltas))
+	i := 0
+	for _, delta := range deltas {
+		result[i] = delta
+		i++
+	}
+
+	return result, joinedRoomIDs, nil
+}
+
+func (d *DatabaseTransaction) currentStateStreamEventsForRoom(
+	ctx context.Context, roomID string,
+	stateFilter *gomatrixserverlib.StateFilter,
+) ([]types.StreamEvent, error) {
+	allState, err := d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilter, nil)
+	if err != nil {
+		return nil, err
+	}
+	s := make([]types.StreamEvent, len(allState))
+	for i := 0; i < len(s); i++ {
+		s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0}
+	}
+	return s, nil
+}
+
+func (d *DatabaseTransaction) SendToDeviceUpdatesForSync(
+	ctx context.Context,
+	userID, deviceID string,
+	from, to types.StreamPosition,
+) (types.StreamPosition, []types.SendToDeviceEvent, error) {
+	// First of all, get our send-to-device updates for this user.
+	lastPos, events, err := d.SendToDevice.SelectSendToDeviceMessages(ctx, d.txn, userID, deviceID, from, to)
+	if err != nil {
+		return from, nil, fmt.Errorf("d.SendToDevice.SelectSendToDeviceMessages: %w", err)
+	}
+	// If there's nothing to do then stop here.
+	if len(events) == 0 {
+		return to, nil, nil
+	}
+	return lastPos, events, nil
+}
+
+func (d *DatabaseTransaction) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) {
+	_, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, d.txn, roomIDs, streamPos)
+	return receipts, err
+}
+
+func (d *DatabaseTransaction) GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, rooms map[string]string) (map[string]*eventutil.NotificationData, error) {
+	roomIDs := make([]string, 0, len(rooms))
+	for roomID, membership := range rooms {
+		if membership != gomatrixserverlib.Join {
+			continue
+		}
+		roomIDs = append(roomIDs, roomID)
+	}
+	return d.NotificationData.SelectUserUnreadCountsForRooms(ctx, d.txn, userID, roomIDs)
+}
+
+func (d *DatabaseTransaction) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
+	return d.Presence.GetPresenceForUser(ctx, d.txn, userID)
+}
+
+func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
+	return d.Presence.GetPresenceAfter(ctx, d.txn, after, filter)
+}
+
+func (d *DatabaseTransaction) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
+	return d.Presence.GetMaxPresenceID(ctx, d.txn)
+}
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
deleted file mode 100644
index a05e68804..000000000
--- a/syncapi/storage/shared/syncserver.go
+++ /dev/null
@@ -1,1103 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package shared
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"fmt"
-
-	"github.com/tidwall/gjson"
-
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
-
-	"github.com/matrix-org/dendrite/internal/eventutil"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/syncapi/storage/tables"
-	"github.com/matrix-org/dendrite/syncapi/types"
-)
-
-// Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite
-// For now this contains the shared functions
-type Database struct {
-	DB                  *sql.DB
-	Writer              sqlutil.Writer
-	Invites             tables.Invites
-	Peeks               tables.Peeks
-	AccountData         tables.AccountData
-	OutputEvents        tables.Events
-	Topology            tables.Topology
-	CurrentRoomState    tables.CurrentRoomState
-	BackwardExtremities tables.BackwardsExtremities
-	SendToDevice        tables.SendToDevice
-	Filter              tables.Filter
-	Receipts            tables.Receipts
-	Memberships         tables.Memberships
-	NotificationData    tables.NotificationData
-	Ignores             tables.Ignores
-	Presence            tables.Presence
-}
-
-func (d *Database) readOnlySnapshot(ctx context.Context) (*sql.Tx, error) {
-	return d.DB.BeginTx(ctx, &sql.TxOptions{
-		// Set the isolation level so that we see a snapshot of the database.
-		// In PostgreSQL repeatable read transactions will see a snapshot taken
-		// at the first query, and since the transaction is read-only it can't
-		// run into any serialisation errors.
-		// https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ
-		Isolation: sql.LevelRepeatableRead,
-		ReadOnly:  true,
-	})
-}
-
-func (d *Database) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) {
-	id, err := d.OutputEvents.SelectMaxEventID(ctx, nil)
-	if err != nil {
-		return 0, fmt.Errorf("d.OutputEvents.SelectMaxEventID: %w", err)
-	}
-	return types.StreamPosition(id), nil
-}
-
-func (d *Database) MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error) {
-	id, err := d.Receipts.SelectMaxReceiptID(ctx, nil)
-	if err != nil {
-		return 0, fmt.Errorf("d.Receipts.SelectMaxReceiptID: %w", err)
-	}
-	return types.StreamPosition(id), nil
-}
-
-func (d *Database) MaxStreamPositionForInvites(ctx context.Context) (types.StreamPosition, error) {
-	id, err := d.Invites.SelectMaxInviteID(ctx, nil)
-	if err != nil {
-		return 0, fmt.Errorf("d.Invites.SelectMaxInviteID: %w", err)
-	}
-	return types.StreamPosition(id), nil
-}
-
-func (d *Database) MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error) {
-	id, err := d.SendToDevice.SelectMaxSendToDeviceMessageID(ctx, nil)
-	if err != nil {
-		return 0, fmt.Errorf("d.SendToDevice.SelectMaxSendToDeviceMessageID: %w", err)
-	}
-	return types.StreamPosition(id), nil
-}
-
-func (d *Database) MaxStreamPositionForAccountData(ctx context.Context) (types.StreamPosition, error) {
-	id, err := d.AccountData.SelectMaxAccountDataID(ctx, nil)
-	if err != nil {
-		return 0, fmt.Errorf("d.Invites.SelectMaxAccountDataID: %w", err)
-	}
-	return types.StreamPosition(id), nil
-}
-
-func (d *Database) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) {
-	id, err := d.NotificationData.SelectMaxID(ctx, nil)
-	if err != nil {
-		return 0, fmt.Errorf("d.NotificationData.SelectMaxID: %w", err)
-	}
-	return types.StreamPosition(id), nil
-}
-
-func (d *Database) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) {
-	return d.CurrentRoomState.SelectCurrentState(ctx, nil, roomID, stateFilterPart, excludeEventIDs)
-}
-
-func (d *Database) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) {
-	return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership)
-}
-
-func (d *Database) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) {
-	return d.Memberships.SelectMembershipCount(ctx, nil, roomID, membership, pos)
-}
-
-func (d *Database) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) {
-	return d.Memberships.SelectHeroes(ctx, nil, roomID, userID, memberships)
-}
-
-func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) {
-	return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents)
-}
-
-func (d *Database) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) {
-	return d.Topology.SelectPositionInTopology(ctx, nil, eventID)
-}
-
-func (d *Database) InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) {
-	return d.Invites.SelectInviteEventsInRange(ctx, nil, targetUserID, r)
-}
-
-func (d *Database) PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error) {
-	return d.Peeks.SelectPeeksInRange(ctx, nil, userID, deviceID, r)
-}
-
-func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) {
-	return d.Receipts.SelectRoomReceiptsAfter(ctx, nil, roomIDs, streamPos)
-}
-
-// Events lookups a list of event by their event ID.
-// Returns a list of events matching the requested IDs found in the database.
-// If an event is not found in the database then it will be omitted from the list.
-// Returns an error if there was a problem talking with the database.
-// Does not include any transaction IDs in the returned events.
-func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) {
-	streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs, nil, false)
-	if err != nil {
-		return nil, err
-	}
-
-	// We don't include a device here as we only include transaction IDs in
-	// incremental syncs.
-	return d.StreamEventsToEvents(nil, streamEvents), nil
-}
-
-func (d *Database) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) {
-	return d.CurrentRoomState.SelectJoinedUsers(ctx, nil)
-}
-
-func (d *Database) AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) {
-	return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, nil, roomIDs)
-}
-
-func (d *Database) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) {
-	return d.Peeks.SelectPeekingDevices(ctx, nil)
-}
-
-func (d *Database) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) {
-	return d.CurrentRoomState.SelectSharedUsers(ctx, nil, userID, otherUserIDs)
-}
-
-func (d *Database) GetStateEvent(
-	ctx context.Context, roomID, evType, stateKey string,
-) (*gomatrixserverlib.HeaderedEvent, error) {
-	return d.CurrentRoomState.SelectStateEvent(ctx, nil, roomID, evType, stateKey)
-}
-
-func (d *Database) GetStateEventsForRoom(
-	ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter,
-) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) {
-	stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, nil, roomID, stateFilter, nil)
-	return
-}
-
-// AddInviteEvent stores a new invite event for a user.
-// If the invite was successfully stored this returns the stream ID it was stored at.
-// Returns an error if there was a problem communicating with the database.
-func (d *Database) AddInviteEvent(
-	ctx context.Context, inviteEvent *gomatrixserverlib.HeaderedEvent,
-) (sp types.StreamPosition, err error) {
-	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent)
-		return err
-	})
-	return
-}
-
-// RetireInviteEvent removes an old invite event from the database.
-// Returns an error if there was a problem communicating with the database.
-func (d *Database) RetireInviteEvent(
-	ctx context.Context, inviteEventID string,
-) (sp types.StreamPosition, err error) {
-	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		sp, err = d.Invites.DeleteInviteEvent(ctx, txn, inviteEventID)
-		return err
-	})
-	return
-}
-
-// AddPeek tracks the fact that a user has started peeking.
-// If the peek was successfully stored this returns the stream ID it was stored at.
-// Returns an error if there was a problem communicating with the database.
-func (d *Database) AddPeek(
-	ctx context.Context, roomID, userID, deviceID string,
-) (sp types.StreamPosition, err error) {
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		sp, err = d.Peeks.InsertPeek(ctx, txn, roomID, userID, deviceID)
-		return err
-	})
-	return
-}
-
-// DeletePeek tracks the fact that a user has stopped peeking from the specified
-// device. If the peeks was successfully deleted this returns the stream ID it was
-// stored at. Returns an error if there was a problem communicating with the database.
-func (d *Database) DeletePeek(
-	ctx context.Context, roomID, userID, deviceID string,
-) (sp types.StreamPosition, err error) {
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		sp, err = d.Peeks.DeletePeek(ctx, txn, roomID, userID, deviceID)
-		return err
-	})
-	if err == sql.ErrNoRows {
-		sp = 0
-		err = nil
-	}
-	return
-}
-
-// DeletePeeks tracks the fact that a user has stopped peeking from all devices
-// If the peeks was successfully deleted this returns the stream ID it was stored at.
-// Returns an error if there was a problem communicating with the database.
-func (d *Database) DeletePeeks(
-	ctx context.Context, roomID, userID string,
-) (sp types.StreamPosition, err error) {
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		sp, err = d.Peeks.DeletePeeks(ctx, txn, roomID, userID)
-		return err
-	})
-	if err == sql.ErrNoRows {
-		sp = 0
-		err = nil
-	}
-	return
-}
-
-// GetAccountDataInRange returns all account data for a given user inserted or
-// updated between two given positions
-// Returns a map following the format data[roomID] = []dataTypes
-// If no data is retrieved, returns an empty map
-// If there was an issue with the retrieval, returns an error
-func (d *Database) GetAccountDataInRange(
-	ctx context.Context, userID string, r types.Range,
-	accountDataFilterPart *gomatrixserverlib.EventFilter,
-) (map[string][]string, types.StreamPosition, error) {
-	return d.AccountData.SelectAccountDataInRange(ctx, nil, userID, r, accountDataFilterPart)
-}
-
-// UpsertAccountData keeps track of new or updated account data, by saving the type
-// of the new/updated data, and the user ID and room ID the data is related to (empty)
-// room ID means the data isn't specific to any room)
-// If no data with the given type, user ID and room ID exists in the database,
-// creates a new row, else update the existing one
-// Returns an error if there was an issue with the upsert
-func (d *Database) UpsertAccountData(
-	ctx context.Context, userID, roomID, dataType string,
-) (sp types.StreamPosition, err error) {
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType)
-		return err
-	})
-	return
-}
-
-func (d *Database) StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent {
-	out := make([]*gomatrixserverlib.HeaderedEvent, len(in))
-	for i := 0; i < len(in); i++ {
-		out[i] = in[i].HeaderedEvent
-		if device != nil && in[i].TransactionID != nil {
-			if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID {
-				err := out[i].SetUnsignedField(
-					"transaction_id", in[i].TransactionID.TransactionID,
-				)
-				if err != nil {
-					logrus.WithFields(logrus.Fields{
-						"event_id": out[i].EventID(),
-					}).WithError(err).Warnf("Failed to add transaction ID to event")
-				}
-			}
-		}
-	}
-	return out
-}
-
-// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of
-// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table
-// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such.
-// This function should always be called within a sqlutil.Writer for safety in SQLite.
-func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error {
-	if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil {
-		return err
-	}
-
-	// Check if we have all of the event's previous events. If an event is
-	// missing, add it to the room's backward extremities.
-	prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs(), nil, false)
-	if err != nil {
-		return err
-	}
-	var found bool
-	for _, eID := range ev.PrevEventIDs() {
-		found = false
-		for _, prevEv := range prevEvents {
-			if eID == prevEv.EventID() {
-				found = true
-			}
-		}
-
-		// If the event is missing, consider it a backward extremity.
-		if !found {
-			if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (d *Database) PurgeRoomState(
-	ctx context.Context, roomID string,
-) error {
-	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		// If the event is a create event then we'll delete all of the existing
-		// data for the room. The only reason that a create event would be replayed
-		// to us in this way is if we're about to receive the entire room state.
-		if err := d.CurrentRoomState.DeleteRoomStateForRoom(ctx, txn, roomID); err != nil {
-			return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateForRoom: %w", err)
-		}
-		return nil
-	})
-}
-
-func (d *Database) WriteEvent(
-	ctx context.Context,
-	ev *gomatrixserverlib.HeaderedEvent,
-	addStateEvents []*gomatrixserverlib.HeaderedEvent,
-	addStateEventIDs, removeStateEventIDs []string,
-	transactionID *api.TransactionID, excludeFromSync bool,
-	historyVisibility gomatrixserverlib.HistoryVisibility,
-) (pduPosition types.StreamPosition, returnErr error) {
-	returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		var err error
-		ev.Visibility = historyVisibility
-		pos, err := d.OutputEvents.InsertEvent(
-			ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, historyVisibility,
-		)
-		if err != nil {
-			return fmt.Errorf("d.OutputEvents.InsertEvent: %w", err)
-		}
-		pduPosition = pos
-		var topoPosition types.StreamPosition
-		if topoPosition, err = d.Topology.InsertEventInTopology(ctx, txn, ev, pos); err != nil {
-			return fmt.Errorf("d.Topology.InsertEventInTopology: %w", err)
-		}
-
-		if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil {
-			return fmt.Errorf("d.handleBackwardExtremities: %w", err)
-		}
-
-		if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 {
-			// Nothing to do, the event may have just been a message event.
-			return nil
-		}
-		for i := range addStateEvents {
-			addStateEvents[i].Visibility = historyVisibility
-		}
-		return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition)
-	})
-
-	return pduPosition, returnErr
-}
-
-// This function should always be called within a sqlutil.Writer for safety in SQLite.
-func (d *Database) updateRoomState(
-	ctx context.Context, txn *sql.Tx,
-	removedEventIDs []string,
-	addedEvents []*gomatrixserverlib.HeaderedEvent,
-	pduPosition types.StreamPosition,
-	topoPosition types.StreamPosition,
-) error {
-	// remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add.
-	for _, eventID := range removedEventIDs {
-		if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil {
-			return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateByEventID: %w", err)
-		}
-	}
-
-	for _, event := range addedEvents {
-		if event.StateKey() == nil {
-			// ignore non state events
-			continue
-		}
-		var membership *string
-		if event.Type() == "m.room.member" {
-			value, err := event.Membership()
-			if err != nil {
-				return fmt.Errorf("event.Membership: %w", err)
-			}
-			membership = &value
-			if err = d.Memberships.UpsertMembership(ctx, txn, event, pduPosition, topoPosition); err != nil {
-				return fmt.Errorf("d.Memberships.UpsertMembership: %w", err)
-			}
-		}
-
-		if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil {
-			return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (d *Database) GetEventsInTopologicalRange(
-	ctx context.Context,
-	from, to *types.TopologyToken,
-	roomID string,
-	filter *gomatrixserverlib.RoomEventFilter,
-	backwardOrdering bool,
-) (events []types.StreamEvent, err error) {
-	var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition
-	if backwardOrdering {
-		// Backward ordering means the 'from' token has a higher depth than the 'to' token
-		minDepth = to.Depth
-		maxDepth = from.Depth
-		// for cases where we have say 5 events with the same depth, the TopologyToken needs to
-		// know which of the 5 the client has seen. This is done by using the PDU position.
-		// Events with the same maxDepth but less than this PDU position will be returned.
-		maxStreamPosForMaxDepth = from.PDUPosition
-	} else {
-		// Forward ordering means the 'from' token has a lower depth than the 'to' token.
-		minDepth = from.Depth
-		maxDepth = to.Depth
-	}
-
-	// Select the event IDs from the defined range.
-	var eIDs []string
-	eIDs, err = d.Topology.SelectEventIDsInRange(
-		ctx, nil, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, filter.Limit, !backwardOrdering,
-	)
-	if err != nil {
-		return
-	}
-
-	// Retrieve the events' contents using their IDs.
-	events, err = d.OutputEvents.SelectEvents(ctx, nil, eIDs, filter, true)
-	return
-}
-
-func (d *Database) BackwardExtremitiesForRoom(
-	ctx context.Context, roomID string,
-) (backwardExtremities map[string][]string, err error) {
-	return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, nil, roomID)
-}
-
-func (d *Database) MaxTopologicalPosition(
-	ctx context.Context, roomID string,
-) (types.TopologyToken, error) {
-	depth, streamPos, err := d.Topology.SelectMaxPositionInTopology(ctx, nil, roomID)
-	if err != nil {
-		return types.TopologyToken{}, err
-	}
-	return types.TopologyToken{Depth: depth, PDUPosition: streamPos}, nil
-}
-
-func (d *Database) EventPositionInTopology(
-	ctx context.Context, eventID string,
-) (types.TopologyToken, error) {
-	depth, stream, err := d.Topology.SelectPositionInTopology(ctx, nil, eventID)
-	if err != nil {
-		return types.TopologyToken{}, err
-	}
-	return types.TopologyToken{Depth: depth, PDUPosition: stream}, nil
-}
-
-func (d *Database) StreamToTopologicalPosition(
-	ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool,
-) (types.TopologyToken, error) {
-	topoPos, err := d.Topology.SelectStreamToTopologicalPosition(ctx, nil, roomID, streamPos, backwardOrdering)
-	switch {
-	case err == sql.ErrNoRows && backwardOrdering: // no events in range, going backward
-		return types.TopologyToken{PDUPosition: streamPos}, nil
-	case err == sql.ErrNoRows && !backwardOrdering: // no events in range, going forward
-		topoPos, streamPos, err = d.Topology.SelectMaxPositionInTopology(ctx, nil, roomID)
-		if err != nil {
-			return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectMaxPositionInTopology: %w", err)
-		}
-		return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil
-	case err != nil: // some other error happened
-		return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectStreamToTopologicalPosition: %w", err)
-	default:
-		return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil
-	}
-}
-
-func (d *Database) GetFilter(
-	ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string,
-) error {
-	return d.Filter.SelectFilter(ctx, nil, target, localpart, filterID)
-}
-
-func (d *Database) PutFilter(
-	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
-) (string, error) {
-	var filterID string
-	var err error
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		filterID, err = d.Filter.InsertFilter(ctx, txn, filter, localpart)
-		return err
-	})
-	return filterID, err
-}
-
-func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *gomatrixserverlib.HeaderedEvent) error {
-	redactedEvents, err := d.Events(ctx, []string{redactedEventID})
-	if err != nil {
-		return err
-	}
-	if len(redactedEvents) == 0 {
-		logrus.WithField("event_id", redactedEventID).WithField("redaction_event", redactedBecause.EventID()).Warnf("missing redacted event for redaction")
-		return nil
-	}
-	eventToRedact := redactedEvents[0].Unwrap()
-	redactionEvent := redactedBecause.Unwrap()
-	if err = eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil {
-		return err
-	}
-
-	newEvent := eventToRedact.Headered(redactedBecause.RoomVersion)
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.OutputEvents.UpdateEventJSON(ctx, txn, newEvent)
-	})
-	return err
-}
-
-// GetBackwardTopologyPos retrieves the backward topology position, i.e. the position of the
-// oldest event in the room's topology.
-func (d *Database) GetBackwardTopologyPos(
-	ctx context.Context,
-	events []types.StreamEvent,
-) (types.TopologyToken, error) {
-	zeroToken := types.TopologyToken{}
-	if len(events) == 0 {
-		return zeroToken, nil
-	}
-	pos, spos, err := d.Topology.SelectPositionInTopology(ctx, nil, events[0].EventID())
-	if err != nil {
-		return zeroToken, err
-	}
-	tok := types.TopologyToken{Depth: pos, PDUPosition: spos}
-	tok.Decrement()
-	return tok, nil
-}
-
-// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database.
-// Returns a map of room ID to list of events.
-func (d *Database) fetchStateEvents(
-	ctx context.Context, txn *sql.Tx,
-	roomIDToEventIDSet map[string]map[string]bool,
-	eventIDToEvent map[string]types.StreamEvent,
-) (map[string][]types.StreamEvent, error) {
-	stateBetween := make(map[string][]types.StreamEvent)
-	missingEvents := make(map[string][]string)
-	for roomID, ids := range roomIDToEventIDSet {
-		events := stateBetween[roomID]
-		for id, need := range ids {
-			if !need {
-				continue // deleted state
-			}
-			e, ok := eventIDToEvent[id]
-			if ok {
-				events = append(events, e)
-			} else {
-				m := missingEvents[roomID]
-				m = append(m, id)
-				missingEvents[roomID] = m
-			}
-		}
-		stateBetween[roomID] = events
-	}
-
-	if len(missingEvents) > 0 {
-		// This happens when add_state_ids has an event ID which is not in the provided range.
-		// We need to explicitly fetch them.
-		allMissingEventIDs := []string{}
-		for _, missingEvIDs := range missingEvents {
-			allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...)
-		}
-		evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs)
-		if err != nil {
-			return nil, err
-		}
-		// we know we got them all otherwise an error would've been returned, so just loop the events
-		for _, ev := range evs {
-			roomID := ev.RoomID()
-			stateBetween[roomID] = append(stateBetween[roomID], ev)
-		}
-	}
-	return stateBetween, nil
-}
-
-func (d *Database) fetchMissingStateEvents(
-	ctx context.Context, txn *sql.Tx, eventIDs []string,
-) ([]types.StreamEvent, error) {
-	// Fetch from the events table first so we pick up the stream ID for the
-	// event.
-	events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs, nil, false)
-	if err != nil {
-		return nil, err
-	}
-
-	have := map[string]bool{}
-	for _, event := range events {
-		have[event.EventID()] = true
-	}
-	var missing []string
-	for _, eventID := range eventIDs {
-		if !have[eventID] {
-			missing = append(missing, eventID)
-		}
-	}
-	if len(missing) == 0 {
-		return events, nil
-	}
-
-	// If they are missing from the events table then they should be state
-	// events that we received from outside the main event stream.
-	// These should be in the room state table.
-	stateEvents, err := d.CurrentRoomState.SelectEventsWithEventIDs(ctx, txn, missing)
-
-	if err != nil {
-		return nil, err
-	}
-	if len(stateEvents) != len(missing) {
-		logrus.WithContext(ctx).Warnf("Failed to map all event IDs to events (got %d, wanted %d)", len(stateEvents), len(missing))
-
-		// TODO: Why is this happening? It's probably the roomserver. Uncomment
-		// this error again when we work out what it is and fix it, otherwise we
-		// just end up returning lots of 500s to the client and that breaks
-		// pretty much everything, rather than just sending what we have.
-		//return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing))
-	}
-	events = append(events, stateEvents...)
-	return events, nil
-}
-
-// GetStateDeltas returns the state deltas between fromPos and toPos,
-// exclusive of oldPos, inclusive of newPos, for the rooms in which
-// the user has new membership events.
-// A list of joined room IDs is also returned in case the caller needs it.
-func (d *Database) GetStateDeltas(
-	ctx context.Context, device *userapi.Device,
-	r types.Range, userID string,
-	stateFilter *gomatrixserverlib.StateFilter,
-) (deltas []types.StateDelta, joinedRoomsIDs []string, err error) {
-	// Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821
-	// - Get membership list changes for this user in this sync response
-	// - For each room which has membership list changes:
-	//     * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO).
-	//       If it is, then we need to send the full room state down (and 'limited' is always true).
-	//     * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block.
-	//     * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block.
-	// - Get all CURRENTLY joined rooms, and add them to 'joined' block.
-	txn, err := d.readOnlySnapshot(ctx)
-	if err != nil {
-		return nil, nil, fmt.Errorf("d.readOnlySnapshot: %w", err)
-	}
-	var succeeded bool
-	defer sqlutil.EndTransactionWithCheck(txn, &succeeded, &err)
-
-	// Look up all memberships for the user. We only care about rooms that a
-	// user has ever interacted with — joined to, kicked/banned from, left.
-	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
-		}
-		return nil, nil, err
-	}
-
-	allRoomIDs := make([]string, 0, len(memberships))
-	joinedRoomIDs := make([]string, 0, len(memberships))
-	for roomID, membership := range memberships {
-		allRoomIDs = append(allRoomIDs, roomID)
-		if membership == gomatrixserverlib.Join {
-			joinedRoomIDs = append(joinedRoomIDs, roomID)
-		}
-	}
-
-	// get all the state events ever (i.e. for all available rooms) between these two positions
-	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
-		}
-		return nil, nil, err
-	}
-	state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
-		}
-		return nil, nil, err
-	}
-
-	// find out which rooms this user is peeking, if any.
-	// We do this before joins so any peeks get overwritten
-	peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r)
-	if err != nil && err != sql.ErrNoRows {
-		return nil, nil, err
-	}
-
-	// add peek blocks
-	for _, peek := range peeks {
-		if peek.New {
-			// send full room state down instead of a delta
-			var s []types.StreamEvent
-			s, err = d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter)
-			if err != nil {
-				if err == sql.ErrNoRows {
-					continue
-				}
-				return nil, nil, err
-			}
-			state[peek.RoomID] = s
-		}
-		if !peek.Deleted {
-			deltas = append(deltas, types.StateDelta{
-				Membership:  gomatrixserverlib.Peek,
-				StateEvents: d.StreamEventsToEvents(device, state[peek.RoomID]),
-				RoomID:      peek.RoomID,
-			})
-		}
-	}
-
-	// handle newly joined rooms and non-joined rooms
-	newlyJoinedRooms := make(map[string]bool, len(state))
-	for roomID, stateStreamEvents := range state {
-		for _, ev := range stateStreamEvents {
-			if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" {
-				if membership == gomatrixserverlib.Join && prevMembership != membership {
-					// send full room state down instead of a delta
-					var s []types.StreamEvent
-					s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter)
-					if err != nil {
-						if err == sql.ErrNoRows {
-							continue
-						}
-						return nil, nil, err
-					}
-					state[roomID] = s
-					newlyJoinedRooms[roomID] = true
-					continue // we'll add this room in when we do joined rooms
-				}
-
-				deltas = append(deltas, types.StateDelta{
-					Membership:    membership,
-					MembershipPos: ev.StreamPosition,
-					StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
-					RoomID:        roomID,
-				})
-				break
-			}
-		}
-	}
-
-	// Add in currently joined rooms
-	for _, joinedRoomID := range joinedRoomIDs {
-		deltas = append(deltas, types.StateDelta{
-			Membership:  gomatrixserverlib.Join,
-			StateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]),
-			RoomID:      joinedRoomID,
-			NewlyJoined: newlyJoinedRooms[joinedRoomID],
-		})
-	}
-
-	succeeded = true
-	return deltas, joinedRoomIDs, nil
-}
-
-// GetStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync
-// requests with full_state=true.
-// Fetches full state for all joined rooms and uses selectStateInRange to get
-// updates for other rooms.
-func (d *Database) GetStateDeltasForFullStateSync(
-	ctx context.Context, device *userapi.Device,
-	r types.Range, userID string,
-	stateFilter *gomatrixserverlib.StateFilter,
-) ([]types.StateDelta, []string, error) {
-	txn, err := d.readOnlySnapshot(ctx)
-	if err != nil {
-		return nil, nil, fmt.Errorf("d.readOnlySnapshot: %w", err)
-	}
-	var succeeded bool
-	defer sqlutil.EndTransactionWithCheck(txn, &succeeded, &err)
-
-	// Look up all memberships for the user. We only care about rooms that a
-	// user has ever interacted with — joined to, kicked/banned from, left.
-	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
-		}
-		return nil, nil, err
-	}
-
-	allRoomIDs := make([]string, 0, len(memberships))
-	joinedRoomIDs := make([]string, 0, len(memberships))
-	for roomID, membership := range memberships {
-		allRoomIDs = append(allRoomIDs, roomID)
-		if membership == gomatrixserverlib.Join {
-			joinedRoomIDs = append(joinedRoomIDs, roomID)
-		}
-	}
-
-	// Use a reasonable initial capacity
-	deltas := make(map[string]types.StateDelta)
-
-	peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r)
-	if err != nil && err != sql.ErrNoRows {
-		return nil, nil, err
-	}
-
-	// Add full states for all peeking rooms
-	for _, peek := range peeks {
-		if !peek.Deleted {
-			s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter)
-			if stateErr != nil {
-				if stateErr == sql.ErrNoRows {
-					continue
-				}
-				return nil, nil, stateErr
-			}
-			deltas[peek.RoomID] = types.StateDelta{
-				Membership:  gomatrixserverlib.Peek,
-				StateEvents: d.StreamEventsToEvents(device, s),
-				RoomID:      peek.RoomID,
-			}
-		}
-	}
-
-	// Get all the state events ever between these two positions
-	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
-		}
-		return nil, nil, err
-	}
-	state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
-		}
-		return nil, nil, err
-	}
-
-	for roomID, stateStreamEvents := range state {
-		for _, ev := range stateStreamEvents {
-			if membership, _ := getMembershipFromEvent(ev.Event, userID); membership != "" {
-				if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above.
-					deltas[roomID] = types.StateDelta{
-						Membership:    membership,
-						MembershipPos: ev.StreamPosition,
-						StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
-						RoomID:        roomID,
-					}
-				}
-
-				break
-			}
-		}
-	}
-
-	// Add full states for all joined rooms
-	for _, joinedRoomID := range joinedRoomIDs {
-		s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter)
-		if stateErr != nil {
-			if stateErr == sql.ErrNoRows {
-				continue
-			}
-			return nil, nil, stateErr
-		}
-		deltas[joinedRoomID] = types.StateDelta{
-			Membership:  gomatrixserverlib.Join,
-			StateEvents: d.StreamEventsToEvents(device, s),
-			RoomID:      joinedRoomID,
-		}
-	}
-
-	// Create a response array.
-	result := make([]types.StateDelta, len(deltas))
-	i := 0
-	for _, delta := range deltas {
-		result[i] = delta
-		i++
-	}
-
-	succeeded = true
-	return result, joinedRoomIDs, nil
-}
-
-func (d *Database) currentStateStreamEventsForRoom(
-	ctx context.Context, txn *sql.Tx, roomID string,
-	stateFilter *gomatrixserverlib.StateFilter,
-) ([]types.StreamEvent, error) {
-	allState, err := d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter, nil)
-	if err != nil {
-		return nil, err
-	}
-	s := make([]types.StreamEvent, len(allState))
-	for i := 0; i < len(s); i++ {
-		s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0}
-	}
-	return s, nil
-}
-
-func (d *Database) StoreNewSendForDeviceMessage(
-	ctx context.Context, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent,
-) (newPos types.StreamPosition, err error) {
-	j, err := json.Marshal(event)
-	if err != nil {
-		return 0, err
-	}
-	// Delegate the database write task to the SendToDeviceWriter. It'll guarantee
-	// that we don't lock the table for writes in more than one place.
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		newPos, err = d.SendToDevice.InsertSendToDeviceMessage(
-			ctx, txn, userID, deviceID, string(j),
-		)
-		return err
-	})
-	if err != nil {
-		return 0, err
-	}
-	return newPos, nil
-}
-
-func (d *Database) SendToDeviceUpdatesForSync(
-	ctx context.Context,
-	userID, deviceID string,
-	from, to types.StreamPosition,
-) (types.StreamPosition, []types.SendToDeviceEvent, error) {
-	// First of all, get our send-to-device updates for this user.
-	lastPos, events, err := d.SendToDevice.SelectSendToDeviceMessages(ctx, nil, userID, deviceID, from, to)
-	if err != nil {
-		return from, nil, fmt.Errorf("d.SendToDevice.SelectSendToDeviceMessages: %w", err)
-	}
-	// If there's nothing to do then stop here.
-	if len(events) == 0 {
-		return to, nil, nil
-	}
-	return lastPos, events, nil
-}
-
-func (d *Database) CleanSendToDeviceUpdates(
-	ctx context.Context,
-	userID, deviceID string, before types.StreamPosition,
-) (err error) {
-	if err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.SendToDevice.DeleteSendToDeviceMessages(ctx, txn, userID, deviceID, before)
-	}); err != nil {
-		logrus.WithError(err).Errorf("Failed to clean up old send-to-device messages for user %q device %q", userID, deviceID)
-		return err
-	}
-	return nil
-}
-
-// getMembershipFromEvent returns the value of content.membership iff the event is a state event
-// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned.
-func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) (string, string) {
-	if ev.Type() != "m.room.member" || !ev.StateKeyEquals(userID) {
-		return "", ""
-	}
-	membership, err := ev.Membership()
-	if err != nil {
-		return "", ""
-	}
-	prevMembership := gjson.GetBytes(ev.Unsigned(), "prev_content.membership").Str
-	return membership, prevMembership
-}
-
-// StoreReceipt stores user receipts
-func (d *Database) StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) {
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		pos, err = d.Receipts.UpsertReceipt(ctx, txn, roomId, receiptType, userId, eventId, timestamp)
-		return err
-	})
-	return
-}
-
-func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) {
-	_, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, nil, roomIDs, streamPos)
-	return receipts, err
-}
-
-func (d *Database) UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) {
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		pos, err = d.NotificationData.UpsertRoomUnreadCounts(ctx, txn, userID, roomID, notificationCount, highlightCount)
-		return err
-	})
-	return
-}
-
-func (d *Database) GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, rooms map[string]string) (map[string]*eventutil.NotificationData, error) {
-	roomIDs := make([]string, 0, len(rooms))
-	for roomID, membership := range rooms {
-		if membership != gomatrixserverlib.Join {
-			continue
-		}
-		roomIDs = append(roomIDs, roomID)
-	}
-	return d.NotificationData.SelectUserUnreadCountsForRooms(ctx, nil, userID, roomIDs)
-}
-
-func (d *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) {
-	return d.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID)
-}
-
-func (d *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) {
-	return d.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter)
-}
-func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) {
-	return d.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter)
-}
-
-func (d *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) {
-	return d.Ignores.SelectIgnores(ctx, nil, userID)
-}
-
-func (d *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error {
-	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.Ignores.UpsertIgnores(ctx, txn, userID, ignores)
-	})
-}
-
-func (d *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
-	var pos types.StreamPosition
-	var err error
-	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		pos, err = d.Presence.UpsertPresence(ctx, txn, userID, statusMsg, presence, lastActiveTS, fromSync)
-		return nil
-	})
-	return pos, err
-}
-
-func (d *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) {
-	return d.Presence.GetPresenceForUser(ctx, nil, userID)
-}
-
-func (d *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
-	return d.Presence.GetPresenceAfter(ctx, nil, after, filter)
-}
-
-func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
-	return d.Presence.GetMaxPresenceID(ctx, nil)
-}
-
-func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) {
-	return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos)
-}
-
-func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) {
-	return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{
-		gomatrixserverlib.MRoomName,
-		gomatrixserverlib.MRoomTopic,
-		"m.room.message",
-	})
-}
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index ba6d8126c..c4019fed2 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -367,7 +367,13 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
 	for start < len(eventIDs) {
 		n := minOfInts(len(eventIDs)-start, 999)
 		query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", sqlutil.QueryVariadic(n), 1)
-		rows, err := txn.QueryContext(ctx, query, iEventIDs[start:start+n]...)
+		var rows *sql.Rows
+		var err error
+		if txn == nil {
+			rows, err = s.db.QueryContext(ctx, query, iEventIDs[start:start+n]...)
+		} else {
+			rows, err = txn.QueryContext(ctx, query, iEventIDs[start:start+n]...)
+		}
 		if err != nil {
 			return nil, err
 		}
diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go
index 58ab8461e..e2dbcd5c8 100644
--- a/syncapi/storage/sqlite3/invites_table.go
+++ b/syncapi/storage/sqlite3/invites_table.go
@@ -50,7 +50,7 @@ const deleteInviteEventSQL = "" +
 	"UPDATE syncapi_invite_events SET deleted=true, id=$1 WHERE event_id = $2 AND deleted=false"
 
 const selectInviteEventsInRangeSQL = "" +
-	"SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" +
+	"SELECT id, room_id, headered_event_json, deleted FROM syncapi_invite_events" +
 	" WHERE target_user_id = $1 AND id > $2 AND id <= $3" +
 	" ORDER BY id DESC"
 
@@ -132,23 +132,28 @@ func (s *inviteEventsStatements) DeleteInviteEvent(
 // active invites for the target user ID in the supplied range.
 func (s *inviteEventsStatements) SelectInviteEventsInRange(
 	ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range,
-) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) {
+) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) {
+	var lastPos types.StreamPosition
 	stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt)
 	rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High())
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, lastPos, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed")
 	result := map[string]*gomatrixserverlib.HeaderedEvent{}
 	retired := map[string]*gomatrixserverlib.HeaderedEvent{}
 	for rows.Next() {
 		var (
+			id        types.StreamPosition
 			roomID    string
 			eventJSON []byte
 			deleted   bool
 		)
-		if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil {
-			return nil, nil, err
+		if err = rows.Scan(&id, &roomID, &eventJSON, &deleted); err != nil {
+			return nil, nil, lastPos, err
+		}
+		if id > lastPos {
+			lastPos = id
 		}
 
 		// if we have seen this room before, it has a higher stream position and hence takes priority
@@ -161,15 +166,19 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange(
 
 		var event *gomatrixserverlib.HeaderedEvent
 		if err := json.Unmarshal(eventJSON, &event); err != nil {
-			return nil, nil, err
+			return nil, nil, lastPos, err
 		}
+
 		if deleted {
 			retired[roomID] = event
 		} else {
 			result[roomID] = event
 		}
 	}
-	return result, retired, nil
+	if lastPos == 0 {
+		lastPos = r.To
+	}
+	return result, retired, lastPos, nil
 }
 
 func (s *inviteEventsStatements) SelectMaxInviteID(
diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go
index a84e2bd16..0879030a6 100644
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
@@ -49,6 +49,20 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
 	return &d, nil
 }
 
+func (d *SyncServerDatasource) NewDatabaseSnapshot(ctx context.Context) (*shared.DatabaseTransaction, error) {
+	return &shared.DatabaseTransaction{
+		Database: &d.Database,
+		// not setting a transaction because SQLite doesn't support it
+	}, nil
+}
+
+func (d *SyncServerDatasource) NewDatabaseTransaction(ctx context.Context) (*shared.DatabaseTransaction, error) {
+	return &shared.DatabaseTransaction{
+		Database: &d.Database,
+		// not setting a transaction because SQLite doesn't support it
+	}, nil
+}
+
 func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) {
 	if err = d.streamID.Prepare(d.db); err != nil {
 		return err
diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go
index a62818e9b..5ff185a32 100644
--- a/syncapi/storage/storage_test.go
+++ b/syncapi/storage/storage_test.go
@@ -60,6 +60,17 @@ func TestWriteEvents(t *testing.T) {
 	})
 }
 
+func WithSnapshot(t *testing.T, db storage.Database, f func(snapshot storage.DatabaseTransaction)) {
+	snapshot, err := db.NewDatabaseSnapshot(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f(snapshot)
+	if err := snapshot.Rollback(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // These tests assert basic functionality of RecentEvents for PDUs
 func TestRecentEventsPDU(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
@@ -79,10 +90,13 @@ func TestRecentEventsPDU(t *testing.T) {
 		// dummy room to make sure SQL queries are filtering on room ID
 		MustWriteEvents(t, db, test.NewRoom(t, alice).Events())
 
-		latest, err := db.MaxStreamPositionForPDUs(ctx)
-		if err != nil {
-			t.Fatalf("failed to get MaxStreamPositionForPDUs: %s", err)
-		}
+		var latest types.StreamPosition
+		WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) {
+			var err error
+			if latest, err = snapshot.MaxStreamPositionForPDUs(ctx); err != nil {
+				t.Fatal("failed to get MaxStreamPositionForPDUs: %w", err)
+			}
+		})
 
 		testCases := []struct {
 			Name         string
@@ -140,14 +154,19 @@ func TestRecentEventsPDU(t *testing.T) {
 			tc := testCases[i]
 			t.Run(tc.Name, func(st *testing.T) {
 				var filter gomatrixserverlib.RoomEventFilter
+				var gotEvents []types.StreamEvent
+				var limited bool
 				filter.Limit = tc.Limit
-				gotEvents, limited, err := db.RecentEvents(ctx, r.ID, types.Range{
-					From: tc.From,
-					To:   tc.To,
-				}, &filter, !tc.ReverseOrder, true)
-				if err != nil {
-					st.Fatalf("failed to do sync: %s", err)
-				}
+				WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) {
+					var err error
+					gotEvents, limited, err = snapshot.RecentEvents(ctx, r.ID, types.Range{
+						From: tc.From,
+						To:   tc.To,
+					}, &filter, !tc.ReverseOrder, true)
+					if err != nil {
+						st.Fatalf("failed to do sync: %s", err)
+					}
+				})
 				if limited != tc.WantLimited {
 					st.Errorf("got limited=%v want %v", limited, tc.WantLimited)
 				}
@@ -178,22 +197,24 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
 		events := r.Events()
 		_ = MustWriteEvents(t, db, events)
 
-		from, err := db.MaxTopologicalPosition(ctx, r.ID)
-		if err != nil {
-			t.Fatalf("failed to get MaxTopologicalPosition: %s", err)
-		}
-		t.Logf("max topo pos = %+v", from)
-		// head towards the beginning of time
-		to := types.TopologyToken{}
+		WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) {
+			from, err := snapshot.MaxTopologicalPosition(ctx, r.ID)
+			if err != nil {
+				t.Fatalf("failed to get MaxTopologicalPosition: %s", err)
+			}
+			t.Logf("max topo pos = %+v", from)
+			// head towards the beginning of time
+			to := types.TopologyToken{}
 
-		// backpaginate 5 messages starting at the latest position.
-		filter := &gomatrixserverlib.RoomEventFilter{Limit: 5}
-		paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true)
-		if err != nil {
-			t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err)
-		}
-		gots := db.StreamEventsToEvents(nil, paginatedEvents)
-		test.AssertEventsEqual(t, gots, test.Reversed(events[len(events)-5:]))
+			// backpaginate 5 messages starting at the latest position.
+			filter := &gomatrixserverlib.RoomEventFilter{Limit: 5}
+			paginatedEvents, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true)
+			if err != nil {
+				t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err)
+			}
+			gots := snapshot.StreamEventsToEvents(nil, paginatedEvents)
+			test.AssertEventsEqual(t, gots, test.Reversed(events[len(events)-5:]))
+		})
 	})
 }
 
@@ -414,13 +435,16 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 		defer closeBase()
 		// At this point there should be no messages. We haven't sent anything
 		// yet.
-		_, events, err := db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if len(events) != 0 {
-			t.Fatal("first call should have no updates")
-		}
+
+		WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) {
+			_, events, err := snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(events) != 0 {
+				t.Fatal("first call should have no updates")
+			}
+		})
 
 		// Try sending a message.
 		streamPos, err := db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{
@@ -432,51 +456,58 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// At this point we should get exactly one message. We're sending the sync position
-		// that we were given from the update and the send-to-device update will be updated
-		// in the database to reflect that this was the sync position we sent the message at.
-		streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if count := len(events); count != 1 {
-			t.Fatalf("second call should have one update, got %d", count)
-		}
+		WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) {
+			// At this point we should get exactly one message. We're sending the sync position
+			// that we were given from the update and the send-to-device update will be updated
+			// in the database to reflect that this was the sync position we sent the message at.
+			var events []types.SendToDeviceEvent
+			streamPos, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if count := len(events); count != 1 {
+				t.Fatalf("second call should have one update, got %d", count)
+			}
+
+			// At this point we should still have one message because we haven't progressed the
+			// sync position yet. This is equivalent to the client failing to /sync and retrying
+			// with the same position.
+			streamPos, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(events) != 1 {
+				t.Fatal("third call should have one update still")
+			}
+		})
 
-		// At this point we should still have one message because we haven't progressed the
-		// sync position yet. This is equivalent to the client failing to /sync and retrying
-		// with the same position.
-		streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if len(events) != 1 {
-			t.Fatal("third call should have one update still")
-		}
 		err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos)
 		if err != nil {
 			return
 		}
 
-		// At this point we should now have no updates, because we've progressed the sync
-		// position. Therefore the update from before will not be sent again.
-		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if len(events) != 0 {
-			t.Fatal("fourth call should have no updates")
-		}
+		WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) {
+			// At this point we should now have no updates, because we've progressed the sync
+			// position. Therefore the update from before will not be sent again.
+			var events []types.SendToDeviceEvent
+			_, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(events) != 0 {
+				t.Fatal("fourth call should have no updates")
+			}
 
-		// At this point we should still have no updates, because no new updates have been
-		// sent.
-		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if len(events) != 0 {
-			t.Fatal("fifth call should have no updates")
-		}
+			// At this point we should still have no updates, because no new updates have been
+			// sent.
+			_, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(events) != 0 {
+				t.Fatal("fifth call should have no updates")
+			}
+		})
 
 		// Send some more messages and verify the ordering is correct ("in order of arrival")
 		var lastPos types.StreamPosition = 0
@@ -492,18 +523,20 @@ func TestSendToDeviceBehaviour(t *testing.T) {
 			lastPos = streamPos
 		}
 
-		_, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, lastPos)
-		if err != nil {
-			t.Fatalf("unable to get events: %v", err)
-		}
-
-		for i := 0; i < 10; i++ {
-			want := json.RawMessage(fmt.Sprintf(`{"count":%d}`, i))
-			got := events[i].Content
-			if !bytes.Equal(got, want) {
-				t.Fatalf("messages are out of order\nwant: %s\ngot: %s", string(want), string(got))
+		WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) {
+			_, events, err := snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, lastPos)
+			if err != nil {
+				t.Fatalf("unable to get events: %v", err)
 			}
-		}
+
+			for i := 0; i < 10; i++ {
+				want := json.RawMessage(fmt.Sprintf(`{"count":%d}`, i))
+				got := events[i].Content
+				if !bytes.Equal(got, want) {
+					t.Fatalf("messages are out of order\nwant: %s\ngot: %s", string(want), string(got))
+				}
+			}
+		})
 	})
 }
 
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 89cb537af..2fdc3cfbb 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -37,7 +37,7 @@ type Invites interface {
 	DeleteInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string) (types.StreamPosition, error)
 	// SelectInviteEventsInRange returns a map of room ID to invite events. If multiple invite/retired invites exist in the given range, return the latest value
 	// for the room.
-	SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (invites map[string]*gomatrixserverlib.HeaderedEvent, retired map[string]*gomatrixserverlib.HeaderedEvent, err error)
+	SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (invites map[string]*gomatrixserverlib.HeaderedEvent, retired map[string]*gomatrixserverlib.HeaderedEvent, maxID types.StreamPosition, err error)
 	SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 }
 
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 0297d5c2f..3f2f7d134 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -5,22 +5,25 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 type AccountDataStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 	userAPI userapi.SyncUserAPI
 }
 
-func (p *AccountDataStreamProvider) Setup() {
-	p.StreamProvider.Setup()
+func (p *AccountDataStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
 
 	p.latestMutex.Lock()
 	defer p.latestMutex.Unlock()
 
-	id, err := p.DB.MaxStreamPositionForAccountData(context.Background())
+	id, err := snapshot.MaxStreamPositionForAccountData(ctx)
 	if err != nil {
 		panic(err)
 	}
@@ -29,13 +32,15 @@ func (p *AccountDataStreamProvider) Setup() {
 
 func (p *AccountDataStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *AccountDataStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
@@ -44,7 +49,7 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 		To:   to,
 	}
 
-	dataTypes, pos, err := p.DB.GetAccountDataInRange(
+	dataTypes, pos, err := snapshot.GetAccountDataInRange(
 		ctx, req.Device.UserID, r, &req.Filter.AccountData,
 	)
 	if err != nil {
diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go
index 5448ee5bd..7996c2038 100644
--- a/syncapi/streams/stream_devicelist.go
+++ b/syncapi/streams/stream_devicelist.go
@@ -6,17 +6,19 @@ import (
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/internal"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 type DeviceListStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 	rsAPI  api.SyncRoomserverAPI
 	keyAPI keyapi.SyncKeyAPI
 }
 
 func (p *DeviceListStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
 	return p.LatestPosition(ctx)
@@ -24,11 +26,12 @@ func (p *DeviceListStreamProvider) CompleteSync(
 
 func (p *DeviceListStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
 	var err error
-	to, _, err = internal.DeviceListCatchup(context.Background(), p.DB, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to)
+	to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
 		return from
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 925da32f2..17b3b8434 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -9,20 +9,23 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 type InviteStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 }
 
-func (p *InviteStreamProvider) Setup() {
-	p.StreamProvider.Setup()
+func (p *InviteStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
 
 	p.latestMutex.Lock()
 	defer p.latestMutex.Unlock()
 
-	id, err := p.DB.MaxStreamPositionForInvites(context.Background())
+	id, err := snapshot.MaxStreamPositionForInvites(ctx)
 	if err != nil {
 		panic(err)
 	}
@@ -31,13 +34,15 @@ func (p *InviteStreamProvider) Setup() {
 
 func (p *InviteStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *InviteStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
@@ -46,7 +51,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 		To:   to,
 	}
 
-	invites, retiredInvites, err := p.DB.InviteEventsInRange(
+	invites, retiredInvites, maxID, err := snapshot.InviteEventsInRange(
 		ctx, req.Device.UserID, r,
 	)
 	if err != nil {
@@ -86,5 +91,5 @@ func (p *InviteStreamProvider) IncrementalSync(
 		}
 	}
 
-	return to
+	return maxID
 }
diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go
index 33872734d..5a81fd09a 100644
--- a/syncapi/streams/stream_notificationdata.go
+++ b/syncapi/streams/stream_notificationdata.go
@@ -3,17 +3,23 @@ package streams
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 type NotificationDataStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 }
 
-func (p *NotificationDataStreamProvider) Setup() {
-	p.StreamProvider.Setup()
+func (p *NotificationDataStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
 
-	id, err := p.DB.MaxStreamPositionForNotificationData(context.Background())
+	p.latestMutex.Lock()
+	defer p.latestMutex.Unlock()
+
+	id, err := snapshot.MaxStreamPositionForNotificationData(ctx)
 	if err != nil {
 		panic(err)
 	}
@@ -22,20 +28,22 @@ func (p *NotificationDataStreamProvider) Setup() {
 
 func (p *NotificationDataStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *NotificationDataStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, _ types.StreamPosition,
 ) types.StreamPosition {
 	// Get the unread notifications for rooms in our join response.
 	// This is to ensure clients always have an unread notification section
 	// and can display the correct numbers.
-	countsByRoom, err := p.DB.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms)
+	countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms)
 	if err != nil {
 		req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed")
 		return from
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 0ab6de886..89c5ba35e 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -5,7 +5,6 @@ import (
 	"database/sql"
 	"fmt"
 	"sort"
-	"sync"
 	"time"
 
 	"github.com/matrix-org/dendrite/internal/caching"
@@ -18,7 +17,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
-	"go.uber.org/atomic"
 
 	"github.com/matrix-org/dendrite/syncapi/notifier"
 )
@@ -33,44 +31,23 @@ const PDU_STREAM_WORKERS = 256
 const PDU_STREAM_QUEUESIZE = PDU_STREAM_WORKERS * 8
 
 type PDUStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 
-	tasks   chan func()
-	workers atomic.Int32
 	// userID+deviceID -> lazy loading cache
 	lazyLoadCache caching.LazyLoadCache
 	rsAPI         roomserverAPI.SyncRoomserverAPI
 	notifier      *notifier.Notifier
 }
 
-func (p *PDUStreamProvider) worker() {
-	defer p.workers.Dec()
-	for {
-		select {
-		case f := <-p.tasks:
-			f()
-		case <-time.After(time.Second * 10):
-			return
-		}
-	}
-}
-
-func (p *PDUStreamProvider) queue(f func()) {
-	if p.workers.Load() < PDU_STREAM_WORKERS {
-		p.workers.Inc()
-		go p.worker()
-	}
-	p.tasks <- f
-}
-
-func (p *PDUStreamProvider) Setup() {
-	p.StreamProvider.Setup()
-	p.tasks = make(chan func(), PDU_STREAM_QUEUESIZE)
+func (p *PDUStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
 
 	p.latestMutex.Lock()
 	defer p.latestMutex.Unlock()
 
-	id, err := p.DB.MaxStreamPositionForPDUs(context.Background())
+	id, err := snapshot.MaxStreamPositionForPDUs(ctx)
 	if err != nil {
 		panic(err)
 	}
@@ -79,6 +56,7 @@ func (p *PDUStreamProvider) Setup() {
 
 func (p *PDUStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
 	from := types.StreamPosition(0)
@@ -94,7 +72,7 @@ func (p *PDUStreamProvider) CompleteSync(
 	}
 
 	// Extract room state and recent events for all rooms the user is joined to.
-	joinedRoomIDs, err := p.DB.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join)
+	joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed")
 		return from
@@ -103,7 +81,7 @@ func (p *PDUStreamProvider) CompleteSync(
 	stateFilter := req.Filter.Room.State
 	eventFilter := req.Filter.Room.Timeline
 
-	if err = p.addIgnoredUsersToFilter(ctx, req, &eventFilter); err != nil {
+	if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil {
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
 	}
 
@@ -117,33 +95,20 @@ func (p *PDUStreamProvider) CompleteSync(
 	}
 
 	// Build up a /sync response. Add joined rooms.
-	var reqMutex sync.Mutex
-	var reqWaitGroup sync.WaitGroup
-	reqWaitGroup.Add(len(joinedRoomIDs))
-	for _, room := range joinedRoomIDs {
-		roomID := room
-		p.queue(func() {
-			defer reqWaitGroup.Done()
-
-			jr, jerr := p.getJoinResponseForCompleteSync(
-				ctx, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, false,
-			)
-			if jerr != nil {
-				req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
-				return
-			}
-
-			reqMutex.Lock()
-			defer reqMutex.Unlock()
-			req.Response.Rooms.Join[roomID] = *jr
-			req.Rooms[roomID] = gomatrixserverlib.Join
-		})
+	for _, roomID := range joinedRoomIDs {
+		jr, jerr := p.getJoinResponseForCompleteSync(
+			ctx, snapshot, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, false,
+		)
+		if jerr != nil {
+			req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
+			continue // return from
+		}
+		req.Response.Rooms.Join[roomID] = *jr
+		req.Rooms[roomID] = gomatrixserverlib.Join
 	}
 
-	reqWaitGroup.Wait()
-
 	// Add peeked rooms.
-	peeks, err := p.DB.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r)
+	peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PeeksInRange failed")
 		return from
@@ -152,11 +117,11 @@ func (p *PDUStreamProvider) CompleteSync(
 		if !peek.Deleted {
 			var jr *types.JoinResponse
 			jr, err = p.getJoinResponseForCompleteSync(
-				ctx, peek.RoomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, true,
+				ctx, snapshot, peek.RoomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, true,
 			)
 			if err != nil {
 				req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed")
-				return from
+				continue // return from
 			}
 			req.Response.Rooms.Peek[peek.RoomID] = *jr
 		}
@@ -167,6 +132,7 @@ func (p *PDUStreamProvider) CompleteSync(
 
 func (p *PDUStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) (newPos types.StreamPosition) {
@@ -184,12 +150,12 @@ func (p *PDUStreamProvider) IncrementalSync(
 	eventFilter := req.Filter.Room.Timeline
 
 	if req.WantFullState {
-		if stateDeltas, syncJoinedRooms, err = p.DB.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
+		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed")
 			return
 		}
 	} else {
-		if stateDeltas, syncJoinedRooms, err = p.DB.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
+		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltas failed")
 			return
 		}
@@ -203,7 +169,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 		return to
 	}
 
-	if err = p.addIgnoredUsersToFilter(ctx, req, &eventFilter); err != nil {
+	if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil {
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
 	}
 
@@ -222,7 +188,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 			}
 		}
 		var pos types.StreamPosition
-		if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil {
+		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
 			return to
 		}
@@ -244,6 +210,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 // nolint:gocyclo
 func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	device *userapi.Device,
 	r types.Range,
 	delta types.StateDelta,
@@ -260,7 +227,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		// This is all "okay" assuming history_visibility == "shared" which it is by default.
 		r.To = delta.MembershipPos
 	}
-	recentStreamEvents, limited, err := p.DB.RecentEvents(
+	recentStreamEvents, limited, err := snapshot.RecentEvents(
 		ctx, delta.RoomID, r,
 		eventFilter, true, true,
 	)
@@ -270,9 +237,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		}
 		return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err)
 	}
-	recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents)
+	recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents)
 	delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back
-	prevBatch, err := p.DB.GetBackwardTopologyPos(ctx, recentStreamEvents)
+	prevBatch, err := snapshot.GetBackwardTopologyPos(ctx, recentStreamEvents)
 	if err != nil {
 		return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err)
 	}
@@ -291,7 +258,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	latestPosition := r.To
 	updateLatestPosition := func(mostRecentEventID string) {
 		var pos types.StreamPosition
-		if _, pos, err = p.DB.PositionInTopology(ctx, mostRecentEventID); err == nil {
+		if _, pos, err = snapshot.PositionInTopology(ctx, mostRecentEventID); err == nil {
 			switch {
 			case r.Backwards && pos < latestPosition:
 				fallthrough
@@ -303,7 +270,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 
 	if stateFilter.LazyLoadMembers {
 		delta.StateEvents, err = p.lazyLoadMembers(
-			ctx, delta.RoomID, true, limited, stateFilter,
+			ctx, snapshot, delta.RoomID, true, limited, stateFilter,
 			device, recentEvents, delta.StateEvents,
 		)
 		if err != nil && err != sql.ErrNoRows {
@@ -320,7 +287,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	}
 
 	// Applies the history visibility rules
-	events, err := applyHistoryVisibilityFilter(ctx, p.DB, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents)
+	events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents)
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
 	}
@@ -336,7 +303,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	case gomatrixserverlib.Join:
 		jr := types.NewJoinResponse()
 		if hasMembershipChange {
-			p.addRoomSummary(ctx, jr, delta.RoomID, device.UserID, latestPosition)
+			p.addRoomSummary(ctx, snapshot, jr, delta.RoomID, device.UserID, latestPosition)
 		}
 		jr.Timeline.PrevBatch = &prevBatch
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync)
@@ -376,7 +343,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 // sure we always return the required events in the timeline.
 func applyHistoryVisibilityFilter(
 	ctx context.Context,
-	db storage.Database,
+	snapshot storage.DatabaseTransaction,
 	rsAPI roomserverAPI.SyncRoomserverAPI,
 	roomID, userID string,
 	limit int,
@@ -384,7 +351,7 @@ func applyHistoryVisibilityFilter(
 ) ([]*gomatrixserverlib.HeaderedEvent, error) {
 	// We need to make sure we always include the latest states events, if they are in the timeline.
 	// We grep at least limit * 2 events, to ensure we really get the needed events.
-	stateEvents, err := db.CurrentState(ctx, roomID, &gomatrixserverlib.StateFilter{Limit: limit * 2}, nil)
+	stateEvents, err := snapshot.CurrentState(ctx, roomID, &gomatrixserverlib.StateFilter{Limit: limit * 2}, nil)
 	if err != nil {
 		// Not a fatal error, we can continue without the stateEvents,
 		// they are only needed if there are state events in the timeline.
@@ -395,7 +362,7 @@ func applyHistoryVisibilityFilter(
 		alwaysIncludeIDs[ev.EventID()] = struct{}{}
 	}
 	startTime := time.Now()
-	events, err := internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync")
+	events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync")
 	if err != nil {
 		return nil, err
 	}
@@ -408,10 +375,10 @@ func applyHistoryVisibilityFilter(
 	return events, nil
 }
 
-func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) {
+func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, snapshot storage.DatabaseTransaction, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) {
 	// Work out how many members are in the room.
-	joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition)
-	invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, latestPosition)
+	joinedCount, _ := snapshot.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition)
+	invitedCount, _ := snapshot.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, latestPosition)
 
 	jr.Summary.JoinedMemberCount = &joinedCount
 	jr.Summary.InvitedMemberCount = &invitedCount
@@ -439,7 +406,7 @@ func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinRe
 			}
 		}
 	}
-	heroes, err := p.DB.GetRoomHeroes(ctx, roomID, userID, []string{"join", "invite"})
+	heroes, err := snapshot.GetRoomHeroes(ctx, roomID, userID, []string{"join", "invite"})
 	if err != nil {
 		return
 	}
@@ -449,6 +416,7 @@ func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinRe
 
 func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	roomID string,
 	r types.Range,
 	stateFilter *gomatrixserverlib.StateFilter,
@@ -460,7 +428,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	jr = types.NewJoinResponse()
 	// TODO: When filters are added, we may need to call this multiple times to get enough events.
 	//       See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316
-	recentStreamEvents, limited, err := p.DB.RecentEvents(
+	recentStreamEvents, limited, err := snapshot.RecentEvents(
 		ctx, roomID, r, eventFilter, true, true,
 	)
 	if err != nil {
@@ -484,7 +452,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		}
 	}
 
-	stateEvents, err := p.DB.CurrentState(ctx, roomID, stateFilter, excludingEventIDs)
+	stateEvents, err := snapshot.CurrentState(ctx, roomID, stateFilter, excludingEventIDs)
 	if err != nil {
 		return
 	}
@@ -494,7 +462,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 	var prevBatch *types.TopologyToken
 	if len(recentStreamEvents) > 0 {
 		var backwardTopologyPos, backwardStreamPos types.StreamPosition
-		backwardTopologyPos, backwardStreamPos, err = p.DB.PositionInTopology(ctx, recentStreamEvents[0].EventID())
+		backwardTopologyPos, backwardStreamPos, err = snapshot.PositionInTopology(ctx, recentStreamEvents[0].EventID())
 		if err != nil {
 			return
 		}
@@ -505,18 +473,18 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		prevBatch.Decrement()
 	}
 
-	p.addRoomSummary(ctx, jr, roomID, device.UserID, r.From)
+	p.addRoomSummary(ctx, snapshot, jr, roomID, device.UserID, r.From)
 
 	// We don't include a device here as we don't need to send down
 	// transaction IDs for complete syncs, but we do it anyway because Sytest demands it for:
 	// "Can sync a room with a message with a transaction id" - which does a complete sync to check.
-	recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents)
+	recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents)
 	stateEvents = removeDuplicates(stateEvents, recentEvents)
 
 	events := recentEvents
 	// Only apply history visibility checks if the response is for joined rooms
 	if !isPeek {
-		events, err = applyHistoryVisibilityFilter(ctx, p.DB, p.rsAPI, roomID, device.UserID, eventFilter.Limit, recentEvents)
+		events, err = applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, roomID, device.UserID, eventFilter.Limit, recentEvents)
 		if err != nil {
 			logrus.WithError(err).Error("unable to apply history visibility filter")
 		}
@@ -530,7 +498,8 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 		if err != nil {
 			return nil, err
 		}
-		stateEvents, err = p.lazyLoadMembers(ctx, roomID,
+		stateEvents, err = p.lazyLoadMembers(
+			ctx, snapshot, roomID,
 			false, limited, stateFilter,
 			device, recentEvents, stateEvents,
 		)
@@ -549,7 +518,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
 }
 
 func (p *PDUStreamProvider) lazyLoadMembers(
-	ctx context.Context, roomID string,
+	ctx context.Context, snapshot storage.DatabaseTransaction, roomID string,
 	incremental, limited bool, stateFilter *gomatrixserverlib.StateFilter,
 	device *userapi.Device,
 	timelineEvents, stateEvents []*gomatrixserverlib.HeaderedEvent,
@@ -598,7 +567,7 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 	filter.Limit = stateFilter.Limit
 	filter.Senders = &wantUsers
 	filter.Types = &[]string{gomatrixserverlib.MRoomMember}
-	memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &filter)
+	memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter)
 	if err != nil {
 		return stateEvents, err
 	}
@@ -612,8 +581,8 @@ func (p *PDUStreamProvider) lazyLoadMembers(
 
 // addIgnoredUsersToFilter adds ignored users to the eventfilter and
 // the syncreq itself for further use in streams.
-func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, req *types.SyncRequest, eventFilter *gomatrixserverlib.RoomEventFilter) error {
-	ignores, err := p.DB.IgnoresForUser(ctx, req.Device.UserID)
+func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest, eventFilter *gomatrixserverlib.RoomEventFilter) error {
+	ignores, err := snapshot.IgnoresForUser(ctx, req.Device.UserID)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 15db4d30e..81cea7d5e 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -23,20 +23,26 @@ import (
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/syncapi/notifier"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 type PresenceStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 	// cache contains previously sent presence updates to avoid unneeded updates
 	cache    sync.Map
 	notifier *notifier.Notifier
 }
 
-func (p *PresenceStreamProvider) Setup() {
-	p.StreamProvider.Setup()
+func (p *PresenceStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
 
-	id, err := p.DB.MaxStreamPositionForPresence(context.Background())
+	p.latestMutex.Lock()
+	defer p.latestMutex.Unlock()
+
+	id, err := snapshot.MaxStreamPositionForPresence(ctx)
 	if err != nil {
 		panic(err)
 	}
@@ -45,18 +51,20 @@ func (p *PresenceStreamProvider) Setup() {
 
 func (p *PresenceStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *PresenceStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
 	// We pull out a larger number than the filter asks for, since we're filtering out events later
-	presences, err := p.DB.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000})
+	presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000})
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PresenceAfter failed")
 		return from
@@ -84,7 +92,7 @@ func (p *PresenceStreamProvider) IncrementalSync(
 				}
 				// Bear in mind that this might return nil, but at least populating
 				// a nil means that there's a map entry so we won't repeat this call.
-				presences[roomUsers[i]], err = p.DB.GetPresence(ctx, roomUsers[i])
+				presences[roomUsers[i]], err = snapshot.GetPresence(ctx, roomUsers[i])
 				if err != nil {
 					req.Log.WithError(err).Error("unable to query presence for user")
 					return from
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index f4e84c7d0..8818a5533 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -4,18 +4,24 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type ReceiptStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 }
 
-func (p *ReceiptStreamProvider) Setup() {
-	p.StreamProvider.Setup()
+func (p *ReceiptStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
 
-	id, err := p.DB.MaxStreamPositionForReceipts(context.Background())
+	p.latestMutex.Lock()
+	defer p.latestMutex.Unlock()
+
+	id, err := snapshot.MaxStreamPositionForReceipts(ctx)
 	if err != nil {
 		panic(err)
 	}
@@ -24,13 +30,15 @@ func (p *ReceiptStreamProvider) Setup() {
 
 func (p *ReceiptStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *ReceiptStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
@@ -41,7 +49,7 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 		}
 	}
 
-	lastPos, receipts, err := p.DB.RoomReceiptsAfter(ctx, joinedRooms, from)
+	lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed")
 		return from
diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go
index 31c6187cb..00b67cc42 100644
--- a/syncapi/streams/stream_sendtodevice.go
+++ b/syncapi/streams/stream_sendtodevice.go
@@ -3,17 +3,23 @@ package streams
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 type SendToDeviceStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 }
 
-func (p *SendToDeviceStreamProvider) Setup() {
-	p.StreamProvider.Setup()
+func (p *SendToDeviceStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
+	p.DefaultStreamProvider.Setup(ctx, snapshot)
 
-	id, err := p.DB.MaxStreamPositionForSendToDeviceMessages(context.Background())
+	p.latestMutex.Lock()
+	defer p.latestMutex.Unlock()
+
+	id, err := snapshot.MaxStreamPositionForSendToDeviceMessages(ctx)
 	if err != nil {
 		panic(err)
 	}
@@ -22,18 +28,20 @@ func (p *SendToDeviceStreamProvider) Setup() {
 
 func (p *SendToDeviceStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *SendToDeviceStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
 	// See if we have any new tasks to do for the send-to-device messaging.
-	lastPos, events, err := p.DB.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to)
+	lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed")
 		return from
diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go
index f781065be..a6f7c7a06 100644
--- a/syncapi/streams/stream_typing.go
+++ b/syncapi/streams/stream_typing.go
@@ -5,24 +5,27 @@ import (
 	"encoding/json"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type TypingStreamProvider struct {
-	StreamProvider
+	DefaultStreamProvider
 	EDUCache *caching.EDUCache
 }
 
 func (p *TypingStreamProvider) CompleteSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 ) types.StreamPosition {
-	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+	return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
 }
 
 func (p *TypingStreamProvider) IncrementalSync(
 	ctx context.Context,
+	snapshot storage.DatabaseTransaction,
 	req *types.SyncRequest,
 	from, to types.StreamPosition,
 ) types.StreamPosition {
diff --git a/syncapi/streams/streamprovider.go b/syncapi/streams/streamprovider.go
new file mode 100644
index 000000000..8b12e2eba
--- /dev/null
+++ b/syncapi/streams/streamprovider.go
@@ -0,0 +1,28 @@
+package streams
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+type StreamProvider interface {
+	Setup(ctx context.Context, snapshot storage.DatabaseTransaction)
+
+	// Advance will update the latest position of the stream based on
+	// an update and will wake callers waiting on StreamNotifyAfter.
+	Advance(latest types.StreamPosition)
+
+	// CompleteSync will update the response to include all updates as needed
+	// for a complete sync. It will always return immediately.
+	CompleteSync(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest) types.StreamPosition
+
+	// IncrementalSync will update the response to include all updates between
+	// the from and to sync positions. It will always return immediately,
+	// making no changes if the range contains no updates.
+	IncrementalSync(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition) types.StreamPosition
+
+	// LatestPosition returns the latest stream position for this stream.
+	LatestPosition(ctx context.Context) types.StreamPosition
+}
diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go
index dbc053bd8..eccbb3a4f 100644
--- a/syncapi/streams/streams.go
+++ b/syncapi/streams/streams.go
@@ -13,15 +13,15 @@ import (
 )
 
 type Streams struct {
-	PDUStreamProvider              types.StreamProvider
-	TypingStreamProvider           types.StreamProvider
-	ReceiptStreamProvider          types.StreamProvider
-	InviteStreamProvider           types.StreamProvider
-	SendToDeviceStreamProvider     types.StreamProvider
-	AccountDataStreamProvider      types.StreamProvider
-	DeviceListStreamProvider       types.StreamProvider
-	NotificationDataStreamProvider types.StreamProvider
-	PresenceStreamProvider         types.StreamProvider
+	PDUStreamProvider              StreamProvider
+	TypingStreamProvider           StreamProvider
+	ReceiptStreamProvider          StreamProvider
+	InviteStreamProvider           StreamProvider
+	SendToDeviceStreamProvider     StreamProvider
+	AccountDataStreamProvider      StreamProvider
+	DeviceListStreamProvider       StreamProvider
+	NotificationDataStreamProvider StreamProvider
+	PresenceStreamProvider         StreamProvider
 }
 
 func NewSyncStreamProviders(
@@ -31,51 +31,58 @@ func NewSyncStreamProviders(
 ) *Streams {
 	streams := &Streams{
 		PDUStreamProvider: &PDUStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
-			lazyLoadCache:  lazyLoadCache,
-			rsAPI:          rsAPI,
-			notifier:       notifier,
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
+			lazyLoadCache:         lazyLoadCache,
+			rsAPI:                 rsAPI,
+			notifier:              notifier,
 		},
 		TypingStreamProvider: &TypingStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
-			EDUCache:       eduCache,
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
+			EDUCache:              eduCache,
 		},
 		ReceiptStreamProvider: &ReceiptStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
 		},
 		InviteStreamProvider: &InviteStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
 		},
 		SendToDeviceStreamProvider: &SendToDeviceStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
 		},
 		AccountDataStreamProvider: &AccountDataStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
-			userAPI:        userAPI,
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
+			userAPI:               userAPI,
 		},
 		NotificationDataStreamProvider: &NotificationDataStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
 		},
 		DeviceListStreamProvider: &DeviceListStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
-			rsAPI:          rsAPI,
-			keyAPI:         keyAPI,
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
+			rsAPI:                 rsAPI,
+			keyAPI:                keyAPI,
 		},
 		PresenceStreamProvider: &PresenceStreamProvider{
-			StreamProvider: StreamProvider{DB: d},
-			notifier:       notifier,
+			DefaultStreamProvider: DefaultStreamProvider{DB: d},
+			notifier:              notifier,
 		},
 	}
 
-	streams.PDUStreamProvider.Setup()
-	streams.TypingStreamProvider.Setup()
-	streams.ReceiptStreamProvider.Setup()
-	streams.InviteStreamProvider.Setup()
-	streams.SendToDeviceStreamProvider.Setup()
-	streams.AccountDataStreamProvider.Setup()
-	streams.NotificationDataStreamProvider.Setup()
-	streams.DeviceListStreamProvider.Setup()
-	streams.PresenceStreamProvider.Setup()
+	ctx := context.TODO()
+	snapshot, err := d.NewDatabaseSnapshot(ctx)
+	if err != nil {
+		panic(err)
+	}
+	defer snapshot.Rollback() // nolint:errcheck
+
+	streams.PDUStreamProvider.Setup(ctx, snapshot)
+	streams.TypingStreamProvider.Setup(ctx, snapshot)
+	streams.ReceiptStreamProvider.Setup(ctx, snapshot)
+	streams.InviteStreamProvider.Setup(ctx, snapshot)
+	streams.SendToDeviceStreamProvider.Setup(ctx, snapshot)
+	streams.AccountDataStreamProvider.Setup(ctx, snapshot)
+	streams.NotificationDataStreamProvider.Setup(ctx, snapshot)
+	streams.DeviceListStreamProvider.Setup(ctx, snapshot)
+	streams.PresenceStreamProvider.Setup(ctx, snapshot)
 
 	return streams
 }
diff --git a/syncapi/streams/template_stream.go b/syncapi/streams/template_stream.go
index 15074cc10..f208d84e4 100644
--- a/syncapi/streams/template_stream.go
+++ b/syncapi/streams/template_stream.go
@@ -8,16 +8,18 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
-type StreamProvider struct {
+type DefaultStreamProvider struct {
 	DB          storage.Database
 	latest      types.StreamPosition
 	latestMutex sync.RWMutex
 }
 
-func (p *StreamProvider) Setup() {
+func (p *DefaultStreamProvider) Setup(
+	ctx context.Context, snapshot storage.DatabaseTransaction,
+) {
 }
 
-func (p *StreamProvider) Advance(
+func (p *DefaultStreamProvider) Advance(
 	latest types.StreamPosition,
 ) {
 	p.latestMutex.Lock()
@@ -28,7 +30,7 @@ func (p *StreamProvider) Advance(
 	}
 }
 
-func (p *StreamProvider) LatestPosition(
+func (p *DefaultStreamProvider) LatestPosition(
 	ctx context.Context,
 ) types.StreamPosition {
 	p.latestMutex.RLock()
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index b2ea105ff..1d0ac1a40 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -305,6 +305,13 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 			syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
 		}
 
+		snapshot, err := rp.db.NewDatabaseSnapshot(req.Context())
+		if err != nil {
+			logrus.WithError(err).Error("Failed to acquire database snapshot for sync request")
+			return jsonerror.InternalServerError()
+		}
+		defer snapshot.Rollback() // nolint:errcheck
+
 		if syncReq.Since.IsEmpty() {
 			// Complete sync
 			syncReq.Response.NextBatch = types.StreamingToken{
@@ -312,70 +319,70 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 				// might advance while processing other streams, resulting in flakey
 				// tests.
 				DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 				PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 				),
 			}
 		} else {
 			// Incremental sync
 			syncReq.Response.NextBatch = types.StreamingToken{
 				PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.PDUPosition, currentPos.PDUPosition,
 				),
 				TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.TypingPosition, currentPos.TypingPosition,
 				),
 				ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
 				),
 				InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.InvitePosition, currentPos.InvitePosition,
 				),
 				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
 				),
 				AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
 				),
 				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
 				),
 				DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
 				),
 				PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
-					syncReq.Context, syncReq,
+					syncReq.Context, snapshot, syncReq,
 					syncReq.Since.PresencePosition, currentPos.PresencePosition,
 				),
 			}
@@ -437,9 +444,15 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use
 		util.GetLogger(req.Context()).WithError(err).Error("newSyncRequest failed")
 		return jsonerror.InternalServerError()
 	}
-	rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), syncReq, fromToken.PDUPosition, toToken.PDUPosition)
+	snapshot, err := rp.db.NewDatabaseSnapshot(req.Context())
+	if err != nil {
+		logrus.WithError(err).Error("Failed to acquire database snapshot for key change")
+		return jsonerror.InternalServerError()
+	}
+	defer snapshot.Rollback() // nolint:errcheck
+	rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), snapshot, syncReq, fromToken.PDUPosition, toToken.PDUPosition)
 	_, _, err = internal.DeviceListCatchup(
-		req.Context(), rp.db, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID,
+		req.Context(), snapshot, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID,
 		syncReq.Response, fromToken.DeviceListPosition, toToken.DeviceListPosition,
 	)
 	if err != nil {
diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go
index a9ea234d0..378cafe99 100644
--- a/syncapi/types/provider.go
+++ b/syncapi/types/provider.go
@@ -41,23 +41,3 @@ func (r *SyncRequest) IsRoomPresent(roomID string) bool {
 		return false
 	}
 }
-
-type StreamProvider interface {
-	Setup()
-
-	// Advance will update the latest position of the stream based on
-	// an update and will wake callers waiting on StreamNotifyAfter.
-	Advance(latest StreamPosition)
-
-	// CompleteSync will update the response to include all updates as needed
-	// for a complete sync. It will always return immediately.
-	CompleteSync(ctx context.Context, req *SyncRequest) StreamPosition
-
-	// IncrementalSync will update the response to include all updates between
-	// the from and to sync positions. It will always return immediately,
-	// making no changes if the range contains no updates.
-	IncrementalSync(ctx context.Context, req *SyncRequest, from, to StreamPosition) StreamPosition
-
-	// LatestPosition returns the latest stream position for this stream.
-	LatestPosition(ctx context.Context) StreamPosition
-}

From 0e2fb63b4fe832befd266fe7e886832901eac00e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 13:31:05 +0100
Subject: [PATCH 17/73] Version 0.10.0 (#2753)

Changelog and version bump.
---
 CHANGES.md          | 22 ++++++++++++++++++++++
 internal/version.go |  4 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 657ca1920..eca575cf2 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,27 @@
 # Changelog
 
+## Dendrite 0.10.0 (2022-09-30)
+
+### Features
+
+* High performance full-text searching has been added to Dendrite
+  * Search must be enabled in the [`search` section of the `sync_api` config](https://github.com/matrix-org/dendrite/blob/6348486a1365c7469a498101f5035a9b6bd16d22/dendrite-sample.monolith.yaml#L279-L290) before it can be used
+  * The search index is stored on the filesystem rather than the sync API database, so a path to a suitable storage location on disk must be configured
+* Sync requests should now complete faster and use considerably less database connections as a result of better transactional isolation
+* The notifications code has been refactored to hopefully make notifications more reliable
+* A new `/_dendrite/admin/refreshDevices/{userID}` admin endpoint has been added for forcing a refresh of a remote user's device lists without having to modify the database by hand
+* A new `/_dendrite/admin/fulltext/reindex` admin endpoint has been added for rebuilding the search index (although this may take some time)
+
+### Fixes
+
+* A number of bugs in the device list updater have been fixed, which should help considerably with federated device list synchronisation and E2EE reliability
+* A state resolution bug has been fixed which should help to prevent unexpected state resets
+* The deprecated `"origin"` field in events will now be correctly ignored in all cases
+* Room versions 8 and 9 will now correctly evaluate `"knock"` join rules and membership states
+* A database index has been added to speed up finding room memberships in the sync API (contributed by [PiotrKozimor](https://github.com/PiotrKozimor))
+* The client API will now return an `M_UNRECOGNIZED` error for unknown endpoints/methods, which should help with client error handling
+* A bug has been fixed when updating push rules which could result in `database is locked` on SQLite
+
 ## Dendrite 0.9.9 (2022-09-22)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index f9b101702..0b19757c2 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 9
-	VersionPatch = 9
+	VersionMinor = 10
+	VersionPatch = 0
 	VersionTag   = "" // example: "rc1"
 )
 

From aa8ec1acbf3932b0543033bbf735225dac21676a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 15:39:05 +0100
Subject: [PATCH 18/73] Update 2_domainname.md

---
 docs/installation/2_domainname.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md
index 7d7fc86bd..e7b3495f7 100644
--- a/docs/installation/2_domainname.md
+++ b/docs/installation/2_domainname.md
@@ -87,6 +87,12 @@ and contain the following JSON document:
 For example, this can be done with the following Caddy config:
 
 ```
+handle /.well-known/matrix/server {
+	header Content-Type application/json
+	header Access-Control-Allow-Origin *
+	respond `"m.server": "matrix.example.com:8448"`
+}
+
 handle /.well-known/matrix/client {
 	header Content-Type application/json
 	header Access-Control-Allow-Origin *

From ee40a29e55eb7986e16bb24bd388fb710e43aea9 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 16:07:18 +0100
Subject: [PATCH 19/73] Fix broken `/sync` due to transaction error

---
 syncapi/consumers/roomserver.go            | 5 ++++-
 syncapi/notifier/notifier.go               | 9 +++++++--
 syncapi/routing/context.go                 | 5 ++++-
 syncapi/routing/search.go                  | 5 ++++-
 syncapi/storage/interface.go               | 2 ++
 syncapi/streams/stream_accountdata.go      | 1 +
 syncapi/streams/stream_devicelist.go       | 2 ++
 syncapi/streams/stream_invite.go           | 1 +
 syncapi/streams/stream_notificationdata.go | 1 +
 syncapi/streams/stream_pdu.go              | 4 ++++
 syncapi/streams/stream_presence.go         | 2 ++
 syncapi/streams/stream_receipt.go          | 1 +
 syncapi/streams/stream_sendtodevice.go     | 1 +
 syncapi/streams/streams.go                 | 5 ++++-
 syncapi/sync/requestpool.go                | 9 +++++++--
 15 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index e5e8fe293..c7a11dbb4 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -27,6 +27,7 @@ import (
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/internal/fulltext"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -454,7 +455,8 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head
 	if err != nil {
 		return nil, err
 	}
-	defer snapshot.Rollback() // nolint:errcheck
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 	prevEvent, err := snapshot.GetStateEvent(
 		s.ctx, event.RoomID(), event.Type(), stateKey,
@@ -474,6 +476,7 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head
 	}
 
 	event.Event, err = event.SetUnsigned(prev)
+	succeeded = true
 	return event, err
 }
 
diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index a8e5bf9ad..db18c6b77 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -19,6 +19,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -323,7 +324,8 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error {
 	if err != nil {
 		return err
 	}
-	defer snapshot.Rollback() // nolint:errcheck
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 	roomToUsers, err := snapshot.AllJoinedUsersInRooms(ctx)
 	if err != nil {
@@ -337,6 +339,7 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error {
 	}
 	n.setPeekingDevices(roomToPeekingDevices)
 
+	succeeded = true
 	return nil
 }
 
@@ -349,7 +352,8 @@ func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs [
 	if err != nil {
 		return err
 	}
-	defer snapshot.Rollback() // nolint:errcheck
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 	roomToUsers, err := snapshot.AllJoinedUsersInRoom(ctx, roomIDs)
 	if err != nil {
@@ -357,6 +361,7 @@ func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs [
 	}
 	n.setUsersJoinedToRooms(roomToUsers)
 
+	succeeded = true
 	return nil
 }
 
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 1ce34b85a..0ed164c7e 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	roomserver "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/internal"
 	"github.com/matrix-org/dendrite/syncapi/storage"
@@ -55,7 +56,8 @@ func Context(
 	if err != nil {
 		return jsonerror.InternalServerError()
 	}
-	defer snapshot.Rollback() // nolint:errcheck
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 	filter, err := parseRoomEventFilter(req)
 	if err != nil {
@@ -184,6 +186,7 @@ func Context(
 		response.End = end.String()
 		response.Start = start.String()
 	}
+	succeeded = true
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: response,
diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go
index bac534a2c..aef355def 100644
--- a/syncapi/routing/search.go
+++ b/syncapi/routing/search.go
@@ -31,6 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/fulltext"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/userapi/api"
 )
@@ -65,7 +66,8 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 	if err != nil {
 		return jsonerror.InternalServerError()
 	}
-	defer snapshot.Rollback() // nolint:errcheck
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 	// only search rooms the user is actually joined to
 	joinedRooms, err := snapshot.RoomIDsWithMembership(ctx, device.UserID, "join")
@@ -249,6 +251,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
 
 	logrus.Debugf("Full search request took %v", time.Since(start))
 
+	succeeded = true
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: res,
diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 3732e43fb..4a03aca74 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -20,6 +20,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"
 	"github.com/matrix-org/dendrite/syncapi/types"
@@ -27,6 +28,7 @@ import (
 )
 
 type DatabaseTransaction interface {
+	sqlutil.Transaction
 	SharedUsers
 
 	MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error)
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 3f2f7d134..34135d65a 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -54,6 +54,7 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 	)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go
index 7996c2038..10ede573a 100644
--- a/syncapi/streams/stream_devicelist.go
+++ b/syncapi/streams/stream_devicelist.go
@@ -34,11 +34,13 @@ func (p *DeviceListStreamProvider) IncrementalSync(
 	to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 	err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 17b3b8434..b52eaaab1 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -56,6 +56,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 	)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.InviteEventsInRange failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go
index 5a81fd09a..e1ee02b21 100644
--- a/syncapi/streams/stream_notificationdata.go
+++ b/syncapi/streams/stream_notificationdata.go
@@ -46,6 +46,7 @@ func (p *NotificationDataStreamProvider) IncrementalSync(
 	countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms)
 	if err != nil {
 		req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 89c5ba35e..43e2ce8ba 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -75,6 +75,7 @@ func (p *PDUStreamProvider) CompleteSync(
 	joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
@@ -101,6 +102,7 @@ func (p *PDUStreamProvider) CompleteSync(
 		)
 		if jerr != nil {
 			req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
+			_ = snapshot.Rollback()
 			continue // return from
 		}
 		req.Response.Rooms.Join[roomID] = *jr
@@ -111,6 +113,7 @@ func (p *PDUStreamProvider) CompleteSync(
 	peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PeeksInRange failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 	for _, peek := range peeks {
@@ -121,6 +124,7 @@ func (p *PDUStreamProvider) CompleteSync(
 			)
 			if err != nil {
 				req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed")
+				_ = snapshot.Rollback()
 				continue // return from
 			}
 			req.Response.Rooms.Peek[peek.RoomID] = *jr
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 81cea7d5e..d24c85620 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -67,6 +67,7 @@ func (p *PresenceStreamProvider) IncrementalSync(
 	presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000})
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PresenceAfter failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
@@ -95,6 +96,7 @@ func (p *PresenceStreamProvider) IncrementalSync(
 				presences[roomUsers[i]], err = snapshot.GetPresence(ctx, roomUsers[i])
 				if err != nil {
 					req.Log.WithError(err).Error("unable to query presence for user")
+					_ = snapshot.Rollback()
 					return from
 				}
 				if len(presences) > req.Filter.Presence.Limit {
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 8818a5533..40e5bd01e 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -52,6 +52,7 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 	lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go
index 00b67cc42..3262832a3 100644
--- a/syncapi/streams/stream_sendtodevice.go
+++ b/syncapi/streams/stream_sendtodevice.go
@@ -44,6 +44,7 @@ func (p *SendToDeviceStreamProvider) IncrementalSync(
 	lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed")
+		_ = snapshot.Rollback()
 		return from
 	}
 
diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go
index eccbb3a4f..dc8547621 100644
--- a/syncapi/streams/streams.go
+++ b/syncapi/streams/streams.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/notifier"
@@ -72,7 +73,8 @@ func NewSyncStreamProviders(
 	if err != nil {
 		panic(err)
 	}
-	defer snapshot.Rollback() // nolint:errcheck
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 	streams.PDUStreamProvider.Setup(ctx, snapshot)
 	streams.TypingStreamProvider.Setup(ctx, snapshot)
@@ -84,6 +86,7 @@ func NewSyncStreamProviders(
 	streams.DeviceListStreamProvider.Setup(ctx, snapshot)
 	streams.PresenceStreamProvider.Setup(ctx, snapshot)
 
+	succeeded = true
 	return streams
 }
 
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index 1d0ac1a40..bef0a5683 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -31,6 +31,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -310,7 +311,8 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 			logrus.WithError(err).Error("Failed to acquire database snapshot for sync request")
 			return jsonerror.InternalServerError()
 		}
-		defer snapshot.Rollback() // nolint:errcheck
+		var succeeded bool
+		defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 		if syncReq.Since.IsEmpty() {
 			// Complete sync
@@ -409,6 +411,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 			}
 		}
 
+		succeeded = true
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: syncReq.Response,
@@ -449,7 +452,8 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use
 		logrus.WithError(err).Error("Failed to acquire database snapshot for key change")
 		return jsonerror.InternalServerError()
 	}
-	defer snapshot.Rollback() // nolint:errcheck
+	var succeeded bool
+	defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 	rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), snapshot, syncReq, fromToken.PDUPosition, toToken.PDUPosition)
 	_, _, err = internal.DeviceListCatchup(
 		req.Context(), snapshot, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID,
@@ -459,6 +463,7 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use
 		util.GetLogger(req.Context()).WithError(err).Error("Failed to DeviceListCatchup info")
 		return jsonerror.InternalServerError()
 	}
+	succeeded = true
 	return util.JSONResponse{
 		Code: 200,
 		JSON: struct {

From 60ec9180e6f5b032f147be66aeb0c6713bdfdbe4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 16:10:44 +0100
Subject: [PATCH 20/73] Update NATS Server to v2.9.2

---
 go.mod | 10 +++++-----
 go.sum | 20 ++++++++++----------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/go.mod b/go.mod
index 0985b986c..c82f76d41 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15
-	github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5
+	github.com/nats-io/nats-server/v2 v2.9.2
 	github.com/nats-io/nats.go v1.17.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
@@ -43,7 +43,7 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c
 	go.uber.org/atomic v1.10.0
-	golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0
+	golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be
 	golang.org/x/image v0.0.0-20220902085622-e7cb96979f69
 	golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105
 	golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1
@@ -91,7 +91,7 @@ require (
 	github.com/h2non/filetype v1.1.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
-	github.com/klauspost/compress v1.15.10 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/lucas-clemente/quic-go v0.29.0 // indirect
 	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
@@ -120,9 +120,9 @@ require (
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
+	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
 	golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect
-	golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 // indirect
+	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
diff --git a/go.sum b/go.sum
index c3db3e09e..a99599cb1 100644
--- a/go.sum
+++ b/go.sum
@@ -347,8 +347,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo=
-github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
+github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
+github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -422,8 +422,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
 github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
-github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5 h1:G/YGSXcJ2bUofD8Ts49it4VNezaJLQldI6fZR+wIUts=
-github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5/go.mod h1:BWKY6217RvhI+FDoOLZ2BH+hOC37xeKRBlQ1Lz7teKI=
+github.com/nats-io/nats-server/v2 v2.9.2 h1:XNDgJgOYYaYlquLdbSHI3xssLipfKUOq3EmYIMNCOsE=
+github.com/nats-io/nats-server/v2 v2.9.2/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk=
 github.com/nats-io/nats.go v1.17.0 h1:1jp5BThsdGlN91hW0k3YEfJbfACjiOYtUiLXG0RL4IE=
 github.com/nats-io/nats.go v1.17.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
@@ -625,8 +625,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY=
-golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A=
+golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -810,8 +810,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
-golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
+golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w=
@@ -829,8 +829,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 h1:yuLAip3bfURHClMG9VBdzPrQvCWjWiWUTBGV+/fCbUs=
-golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
+golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From 3617d5a0ff580c7ca0058490308f7f946df3882f Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 30 Sep 2022 17:31:41 +0200
Subject: [PATCH 21/73] Remove SendAccountData, since InputAccountData is
 already doing that

---
 userapi/internal/api.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 3e761a886..591faffd6 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -796,11 +796,6 @@ func (a *UserInternalAPI) PerformPushRulesPut(
 	if err := a.InputAccountData(ctx, &userReq, &userRes); err != nil {
 		return err
 	}
-	if err := a.SyncProducer.SendAccountData(req.UserID, eventutil.AccountData{
-		Type: pushRulesAccountDataType,
-	}); err != nil {
-		util.GetLogger(ctx).WithError(err).Errorf("syncProducer.SendData failed")
-	}
 	return nil
 }
 

From 7d9545ceead42b4761ee9cce169bc8f5858521ed Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 16:34:06 +0100
Subject: [PATCH 22/73] Another `/sync` fix

---
 syncapi/streams/stream_pdu.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 43e2ce8ba..ba90706e8 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -156,11 +156,13 @@ func (p *PDUStreamProvider) IncrementalSync(
 	if req.WantFullState {
 		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed")
+			_ = snapshot.Rollback()
 			return
 		}
 	} else {
 		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltas failed")
+			_ = snapshot.Rollback()
 			return
 		}
 	}
@@ -175,6 +177,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 
 	if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil {
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
+		_ = snapshot.Rollback()
 	}
 
 	newPos = from
@@ -194,7 +197,11 @@ func (p *PDUStreamProvider) IncrementalSync(
 		var pos types.StreamPosition
 		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
-			return to
+			_ = snapshot.Rollback()
+			if err == context.DeadlineExceeded || err == context.Canceled {
+				return newPos
+			}
+			continue // return to
 		}
 		// Reset the position, as it is only for the special case of newly joined rooms
 		if delta.NewlyJoined {

From 16048be2366889692bba16089f0246865fbb9c48 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 16:55:10 +0100
Subject: [PATCH 23/73] Handle case when applying history visibility failed

---
 syncapi/streams/stream_pdu.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index ba90706e8..92e1bccf0 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -301,6 +301,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents)
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
+		_ = snapshot.Rollback()
 	}
 
 	if len(delta.StateEvents) > 0 {

From 0116db79c601d752cc097fc13fe7d22bdb01faae Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 17:07:37 +0100
Subject: [PATCH 24/73] Reset transaction after a failure

---
 syncapi/storage/interface.go               |  1 +
 syncapi/storage/shared/storage_consumer.go |  2 ++
 syncapi/storage/shared/storage_sync.go     | 14 +++++++++++++
 syncapi/streams/stream_accountdata.go      |  2 +-
 syncapi/streams/stream_devicelist.go       |  4 ++--
 syncapi/streams/stream_invite.go           |  2 +-
 syncapi/streams/stream_notificationdata.go |  2 +-
 syncapi/streams/stream_pdu.go              | 24 ++++++++++++++--------
 syncapi/streams/stream_presence.go         |  2 +-
 syncapi/streams/stream_receipt.go          |  2 +-
 syncapi/streams/stream_sendtodevice.go     |  2 +-
 11 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index 4a03aca74..be75f8ad0 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -29,6 +29,7 @@ import (
 
 type DatabaseTransaction interface {
 	sqlutil.Transaction
+	Reset() (err error)
 	SharedUsers
 
 	MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error)
diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go
index fb3b295e9..937ced3a2 100644
--- a/syncapi/storage/shared/storage_consumer.go
+++ b/syncapi/storage/shared/storage_consumer.go
@@ -77,6 +77,7 @@ func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransactio
 		}
 		return &DatabaseTransaction{
 			Database: d,
+			ctx:      ctx,
 			txn:      txn,
 		}, nil
 	*/
@@ -89,6 +90,7 @@ func (d *Database) NewDatabaseTransaction(ctx context.Context) (*DatabaseTransac
 	}
 	return &DatabaseTransaction{
 		Database: d,
+		ctx:      ctx,
 		txn:      txn,
 	}, nil
 }
diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go
index a19135a69..6cc83ebc8 100644
--- a/syncapi/storage/shared/storage_sync.go
+++ b/syncapi/storage/shared/storage_sync.go
@@ -13,6 +13,7 @@ import (
 
 type DatabaseTransaction struct {
 	*Database
+	ctx context.Context
 	txn *sql.Tx
 }
 
@@ -30,6 +31,19 @@ func (d *DatabaseTransaction) Rollback() error {
 	return d.txn.Rollback()
 }
 
+func (d *DatabaseTransaction) Reset() (err error) {
+	if d.txn == nil {
+		return nil
+	}
+	if err = d.txn.Rollback(); err != nil {
+		return err
+	}
+	if d.txn, err = d.DB.BeginTx(d.ctx, nil); err != nil {
+		return err
+	}
+	return
+}
+
 func (d *DatabaseTransaction) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) {
 	id, err := d.OutputEvents.SelectMaxEventID(ctx, d.txn)
 	if err != nil {
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 34135d65a..f3e7fbdaa 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -54,7 +54,7 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 	)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go
index 10ede573a..307099b8f 100644
--- a/syncapi/streams/stream_devicelist.go
+++ b/syncapi/streams/stream_devicelist.go
@@ -34,13 +34,13 @@ func (p *DeviceListStreamProvider) IncrementalSync(
 	to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 	err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index b52eaaab1..4c889b8f5 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -56,7 +56,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 	)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.InviteEventsInRange failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go
index e1ee02b21..5154dd332 100644
--- a/syncapi/streams/stream_notificationdata.go
+++ b/syncapi/streams/stream_notificationdata.go
@@ -46,7 +46,7 @@ func (p *NotificationDataStreamProvider) IncrementalSync(
 	countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms)
 	if err != nil {
 		req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 92e1bccf0..01ddf9ac9 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -75,7 +75,7 @@ func (p *PDUStreamProvider) CompleteSync(
 	joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 
@@ -102,7 +102,9 @@ func (p *PDUStreamProvider) CompleteSync(
 		)
 		if jerr != nil {
 			req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
-			_ = snapshot.Rollback()
+			if err = snapshot.Reset(); err != nil {
+				return from
+			}
 			continue // return from
 		}
 		req.Response.Rooms.Join[roomID] = *jr
@@ -113,7 +115,7 @@ func (p *PDUStreamProvider) CompleteSync(
 	peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PeeksInRange failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 	for _, peek := range peeks {
@@ -124,7 +126,9 @@ func (p *PDUStreamProvider) CompleteSync(
 			)
 			if err != nil {
 				req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed")
-				_ = snapshot.Rollback()
+				if err = snapshot.Reset(); err != nil {
+					return from
+				}
 				continue // return from
 			}
 			req.Response.Rooms.Peek[peek.RoomID] = *jr
@@ -156,13 +160,13 @@ func (p *PDUStreamProvider) IncrementalSync(
 	if req.WantFullState {
 		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed")
-			_ = snapshot.Rollback()
+			_ = snapshot.Reset()
 			return
 		}
 	} else {
 		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltas failed")
-			_ = snapshot.Rollback()
+			_ = snapshot.Reset()
 			return
 		}
 	}
@@ -177,7 +181,7 @@ func (p *PDUStreamProvider) IncrementalSync(
 
 	if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil {
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 	}
 
 	newPos = from
@@ -197,10 +201,12 @@ func (p *PDUStreamProvider) IncrementalSync(
 		var pos types.StreamPosition
 		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
-			_ = snapshot.Rollback()
 			if err == context.DeadlineExceeded || err == context.Canceled {
 				return newPos
 			}
+			if err = snapshot.Reset(); err != nil {
+				return from
+			}
 			continue // return to
 		}
 		// Reset the position, as it is only for the special case of newly joined rooms
@@ -301,7 +307,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents)
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 	}
 
 	if len(delta.StateEvents) > 0 {
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index d24c85620..8a3f01c24 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -67,7 +67,7 @@ func (p *PresenceStreamProvider) IncrementalSync(
 	presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000})
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PresenceAfter failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 40e5bd01e..79fd65bfe 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -52,7 +52,7 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 	lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go
index 3262832a3..c79efad06 100644
--- a/syncapi/streams/stream_sendtodevice.go
+++ b/syncapi/streams/stream_sendtodevice.go
@@ -44,7 +44,7 @@ func (p *SendToDeviceStreamProvider) IncrementalSync(
 	lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed")
-		_ = snapshot.Rollback()
+		_ = snapshot.Reset()
 		return from
 	}
 

From a050503d8d2f606fc909e6c9b04c81b91c1bd984 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 30 Sep 2022 17:15:43 +0100
Subject: [PATCH 25/73] Version 0.10.1

---
 CHANGES.md          | 11 +++++++++++
 internal/version.go |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index eca575cf2..dbe2ccf02 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,16 @@
 # Changelog
 
+## Dendrite 0.10.1 (2022-09-30)
+
+### Features
+
+* The built-in NATS Server has been updated to version 2.9.2
+
+### Fixes
+
+* A regression introduced in 0.10.0 in `/sync` as a result of transaction errors has been fixed
+* Account data updates will no longer send duplicate output events
+
 ## Dendrite 0.10.0 (2022-09-30)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index 0b19757c2..d508517be 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 10
-	VersionPatch = 0
+	VersionPatch = 1
 	VersionTag   = "" // example: "rc1"
 )
 

From d4710217f8d96ee6889e1e4e7c26defc5456b523 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Sun, 2 Oct 2022 11:31:40 +0200
Subject: [PATCH 26/73] Use non-HTTPS as default URL, as most people will be
 running behind a reverse proxy

---
 cmd/create-account/main.go           | 2 +-
 docs/administration/1_createusers.md | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index a9357f6db..52301415f 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -64,7 +64,7 @@ var (
 	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
 	isAdmin            = flag.Bool("admin", false, "Create an admin account")
 	resetPassword      = flag.Bool("reset-password", false, "Deprecated")
-	serverURL          = flag.String("url", "https://localhost:8448", "The URL to connect to.")
+	serverURL          = flag.String("url", "http://localhost:8008", "The URL to connect to.")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 	timeout            = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
 )
diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md
index 3468398ac..94399a04a 100644
--- a/docs/administration/1_createusers.md
+++ b/docs/administration/1_createusers.md
@@ -1,3 +1,4 @@
+
 ---
 title: Creating user accounts
 parent: Administration
@@ -31,11 +32,11 @@ To create a new **admin account**, add the `-admin` flag:
 ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
 ```
 
-By default `create-account` uses `https://localhost:8448` to connect to Dendrite, this can be overwritten using 
+By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using 
 the `-url` flag:
 
 ```bash
-./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url http://localhost:8008
+./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url https://localhost:8448
 ```
 
 An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`:

From d32f60249d8e248d8f8334b044e016ee384797aa Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 3 Oct 2022 11:38:20 +0100
Subject: [PATCH 27/73] Modify sync transaction behaviour (#2758)

This now uses a transaction per stream, so that errors in one stream
don't propagate to another, and we therefore no longer need to do hacks
to reopen a new transaction after aborting a failed one.
---
 syncapi/storage/interface.go               |   1 -
 syncapi/storage/shared/storage_sync.go     |  13 --
 syncapi/streams/stream_accountdata.go      |   1 -
 syncapi/streams/stream_devicelist.go       |   2 -
 syncapi/streams/stream_invite.go           |   1 -
 syncapi/streams/stream_notificationdata.go |   1 -
 syncapi/streams/stream_pdu.go              |  25 +--
 syncapi/streams/stream_presence.go         |   1 -
 syncapi/streams/stream_receipt.go          |   1 -
 syncapi/streams/stream_sendtodevice.go     |   1 -
 syncapi/sync/requestpool.go                | 199 +++++++++++++++------
 11 files changed, 155 insertions(+), 91 deletions(-)

diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go
index be75f8ad0..4a03aca74 100644
--- a/syncapi/storage/interface.go
+++ b/syncapi/storage/interface.go
@@ -29,7 +29,6 @@ import (
 
 type DatabaseTransaction interface {
 	sqlutil.Transaction
-	Reset() (err error)
 	SharedUsers
 
 	MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error)
diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go
index 6cc83ebc8..0e19d97d2 100644
--- a/syncapi/storage/shared/storage_sync.go
+++ b/syncapi/storage/shared/storage_sync.go
@@ -31,19 +31,6 @@ func (d *DatabaseTransaction) Rollback() error {
 	return d.txn.Rollback()
 }
 
-func (d *DatabaseTransaction) Reset() (err error) {
-	if d.txn == nil {
-		return nil
-	}
-	if err = d.txn.Rollback(); err != nil {
-		return err
-	}
-	if d.txn, err = d.DB.BeginTx(d.ctx, nil); err != nil {
-		return err
-	}
-	return
-}
-
 func (d *DatabaseTransaction) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) {
 	id, err := d.OutputEvents.SelectMaxEventID(ctx, d.txn)
 	if err != nil {
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index f3e7fbdaa..3f2f7d134 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -54,7 +54,6 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 	)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go
index 307099b8f..7996c2038 100644
--- a/syncapi/streams/stream_devicelist.go
+++ b/syncapi/streams/stream_devicelist.go
@@ -34,13 +34,11 @@ func (p *DeviceListStreamProvider) IncrementalSync(
 	to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
-		_ = snapshot.Reset()
 		return from
 	}
 	err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response)
 	if err != nil {
 		req.Log.WithError(err).Error("internal.DeviceListCatchup failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 4c889b8f5..17b3b8434 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -56,7 +56,6 @@ func (p *InviteStreamProvider) IncrementalSync(
 	)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.InviteEventsInRange failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go
index 5154dd332..5a81fd09a 100644
--- a/syncapi/streams/stream_notificationdata.go
+++ b/syncapi/streams/stream_notificationdata.go
@@ -46,7 +46,6 @@ func (p *NotificationDataStreamProvider) IncrementalSync(
 	countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms)
 	if err != nil {
 		req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 01ddf9ac9..d252265ff 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -75,7 +75,6 @@ func (p *PDUStreamProvider) CompleteSync(
 	joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
@@ -102,10 +101,10 @@ func (p *PDUStreamProvider) CompleteSync(
 		)
 		if jerr != nil {
 			req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed")
-			if err = snapshot.Reset(); err != nil {
+			if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone {
 				return from
 			}
-			continue // return from
+			continue
 		}
 		req.Response.Rooms.Join[roomID] = *jr
 		req.Rooms[roomID] = gomatrixserverlib.Join
@@ -115,7 +114,6 @@ func (p *PDUStreamProvider) CompleteSync(
 	peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PeeksInRange failed")
-		_ = snapshot.Reset()
 		return from
 	}
 	for _, peek := range peeks {
@@ -126,10 +124,10 @@ func (p *PDUStreamProvider) CompleteSync(
 			)
 			if err != nil {
 				req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed")
-				if err = snapshot.Reset(); err != nil {
+				if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone {
 					return from
 				}
-				continue // return from
+				continue
 			}
 			req.Response.Rooms.Peek[peek.RoomID] = *jr
 		}
@@ -160,14 +158,12 @@ func (p *PDUStreamProvider) IncrementalSync(
 	if req.WantFullState {
 		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed")
-			_ = snapshot.Reset()
-			return
+			return from
 		}
 	} else {
 		if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil {
 			req.Log.WithError(err).Error("p.DB.GetStateDeltas failed")
-			_ = snapshot.Reset()
-			return
+			return from
 		}
 	}
 
@@ -181,7 +177,6 @@ func (p *PDUStreamProvider) IncrementalSync(
 
 	if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil {
 		req.Log.WithError(err).Error("unable to update event filter with ignored users")
-		_ = snapshot.Reset()
 	}
 
 	newPos = from
@@ -201,13 +196,10 @@ func (p *PDUStreamProvider) IncrementalSync(
 		var pos types.StreamPosition
 		if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil {
 			req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
-			if err == context.DeadlineExceeded || err == context.Canceled {
+			if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone {
 				return newPos
 			}
-			if err = snapshot.Reset(); err != nil {
-				return from
-			}
-			continue // return to
+			continue
 		}
 		// Reset the position, as it is only for the special case of newly joined rooms
 		if delta.NewlyJoined {
@@ -307,7 +299,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 	events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents)
 	if err != nil {
 		logrus.WithError(err).Error("unable to apply history visibility filter")
-		_ = snapshot.Reset()
 	}
 
 	if len(delta.StateEvents) > 0 {
diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go
index 8a3f01c24..8b87af452 100644
--- a/syncapi/streams/stream_presence.go
+++ b/syncapi/streams/stream_presence.go
@@ -67,7 +67,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
 	presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000})
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.PresenceAfter failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 79fd65bfe..8818a5533 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -52,7 +52,6 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 	lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go
index c79efad06..00b67cc42 100644
--- a/syncapi/streams/stream_sendtodevice.go
+++ b/syncapi/streams/stream_sendtodevice.go
@@ -44,7 +44,6 @@ func (p *SendToDeviceStreamProvider) IncrementalSync(
 	lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to)
 	if err != nil {
 		req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed")
-		_ = snapshot.Reset()
 		return from
 	}
 
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index bef0a5683..a71d32ab8 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -306,13 +306,19 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 			syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately")
 		}
 
-		snapshot, err := rp.db.NewDatabaseSnapshot(req.Context())
-		if err != nil {
-			logrus.WithError(err).Error("Failed to acquire database snapshot for sync request")
-			return jsonerror.InternalServerError()
+		withTransaction := func(from types.StreamPosition, f func(snapshot storage.DatabaseTransaction) types.StreamPosition) types.StreamPosition {
+			var succeeded bool
+			snapshot, err := rp.db.NewDatabaseSnapshot(req.Context())
+			if err != nil {
+				logrus.WithError(err).Error("Failed to acquire database snapshot for sync request")
+				return from
+			}
+			defer func() {
+				succeeded = err == nil
+				sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
+			}()
+			return f(snapshot)
 		}
-		var succeeded bool
-		defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err)
 
 		if syncReq.Since.IsEmpty() {
 			// Complete sync
@@ -320,72 +326,162 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 				// Get the current DeviceListPosition first, as the currentPosition
 				// might advance while processing other streams, resulting in flakey
 				// tests.
-				DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				DeviceListPosition: withTransaction(
+					syncReq.Since.DeviceListPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.DeviceListStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				PDUPosition: rp.streams.PDUStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				PDUPosition: withTransaction(
+					syncReq.Since.PDUPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.PDUStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				TypingPosition: rp.streams.TypingStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				TypingPosition: withTransaction(
+					syncReq.Since.TypingPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.TypingStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				ReceiptPosition: withTransaction(
+					syncReq.Since.ReceiptPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.ReceiptStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				InvitePosition: rp.streams.InviteStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				InvitePosition: withTransaction(
+					syncReq.Since.InvitePosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.InviteStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				SendToDevicePosition: withTransaction(
+					syncReq.Since.SendToDevicePosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.SendToDeviceStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				AccountDataPosition: withTransaction(
+					syncReq.Since.AccountDataPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.AccountDataStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				NotificationDataPosition: withTransaction(
+					syncReq.Since.NotificationDataPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.NotificationDataStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
-				PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync(
-					syncReq.Context, snapshot, syncReq,
+				PresencePosition: withTransaction(
+					syncReq.Since.PresencePosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.PresenceStreamProvider.CompleteSync(
+							syncReq.Context, txn, syncReq,
+						)
+					},
 				),
 			}
 		} else {
 			// Incremental sync
 			syncReq.Response.NextBatch = types.StreamingToken{
-				PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.PDUPosition, currentPos.PDUPosition,
+				PDUPosition: withTransaction(
+					syncReq.Since.PDUPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.PDUStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.PDUPosition, currentPos.PDUPosition,
+						)
+					},
 				),
-				TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.TypingPosition, currentPos.TypingPosition,
+				TypingPosition: withTransaction(
+					syncReq.Since.TypingPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.TypingStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.TypingPosition, currentPos.TypingPosition,
+						)
+					},
 				),
-				ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
+				ReceiptPosition: withTransaction(
+					syncReq.Since.ReceiptPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.ReceiptStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
+						)
+					},
 				),
-				InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.InvitePosition, currentPos.InvitePosition,
+				InvitePosition: withTransaction(
+					syncReq.Since.InvitePosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.InviteStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.InvitePosition, currentPos.InvitePosition,
+						)
+					},
 				),
-				SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
+				SendToDevicePosition: withTransaction(
+					syncReq.Since.SendToDevicePosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.SendToDeviceStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
+						)
+					},
 				),
-				AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
+				AccountDataPosition: withTransaction(
+					syncReq.Since.AccountDataPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.AccountDataStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
+						)
+					},
 				),
-				NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
+				NotificationDataPosition: withTransaction(
+					syncReq.Since.NotificationDataPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.NotificationDataStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
+						)
+					},
 				),
-				DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
+				DeviceListPosition: withTransaction(
+					syncReq.Since.DeviceListPosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.DeviceListStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
+						)
+					},
 				),
-				PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync(
-					syncReq.Context, snapshot, syncReq,
-					syncReq.Since.PresencePosition, currentPos.PresencePosition,
+				PresencePosition: withTransaction(
+					syncReq.Since.PresencePosition,
+					func(txn storage.DatabaseTransaction) types.StreamPosition {
+						return rp.streams.PresenceStreamProvider.IncrementalSync(
+							syncReq.Context, txn, syncReq,
+							syncReq.Since.PresencePosition, currentPos.PresencePosition,
+						)
+					},
 				),
 			}
 			// it's possible for there to be no updates for this user even though since < current pos,
@@ -411,7 +507,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 			}
 		}
 
-		succeeded = true
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: syncReq.Response,

From b7f4bab358ea6de0932037242f71152bfc27c289 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 3 Oct 2022 11:38:31 +0100
Subject: [PATCH 28/73] Hopefully fix P2P `--config` error (re. #2756)

---
 cmd/dendrite-demo-pinecone/main.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index da63f9a2c..fb988ebe1 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -89,6 +89,7 @@ func main() {
 	if configFlagSet {
 		cfg = setup.ParseFlags(true)
 		sk = cfg.Global.PrivateKey
+		pk = sk.Public().(ed25519.PublicKey)
 	} else {
 		keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
 		if _, err := os.Stat(keyfile); os.IsNotExist(err) {

From c1e16fd41e24eb1c139b4e4acbef1a309d7d573b Mon Sep 17 00:00:00 2001
From: Ashley Nelson <fant@shley.email>
Date: Mon, 3 Oct 2022 05:57:21 -0500
Subject: [PATCH 29/73] Fix fragility of selectEventsWithEventIDsSQL queries
 (#2757)

This fixes a temporary workaround with the `selectEventsWithEventIDsSQL`
queries where fields need to be artificially added to the queries so the
row results match the format of the `syncapi_output_room_events` table.
I made similar functions that accept row results from the
`syncapi_current_room_state` table and convert them into StreamEvents
without the fields that are specific to output room events.

There is also a unit test in the first commit to ensure the resulting
behavior doesn't change from the modified queries and functions.

Fixes #601.

### Pull Request Checklist

<!-- Please read docs/CONTRIBUTING.md before submitting your pull
request -->

* [x] I have added tests for PR _or_ I have justified why this PR
doesn't need tests.
* [x] Pull request includes a [sign
off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)

Signed-off-by: `Ashley Nelson <fant@shley.email>`

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 .../postgres/current_room_state_table.go      | 38 ++++++--
 .../sqlite3/current_room_state_table.go       | 38 ++++++--
 .../storage/tables/current_room_state_test.go | 88 +++++++++++++++++++
 3 files changed, 150 insertions(+), 14 deletions(-)
 create mode 100644 syncapi/storage/tables/current_room_state_test.go

diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index 4ffd29610..2ccf0be1a 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -104,12 +104,7 @@ const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
 
 const selectEventsWithEventIDsSQL = "" +
-	// TODO: The session_id and transaction_id blanks are here because
-	// the rowsToStreamEvents expects there to be exactly seven columns. We need to
-	// figure out if these really need to be in the DB, and if so, we need a
-	// better permanent fix for this. - neilalexander, 2 Jan 2020
-	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" +
-	" FROM syncapi_current_room_state WHERE event_id = ANY($1)"
+	"SELECT event_id, added_at, headered_event_json, history_visibility FROM syncapi_current_room_state WHERE event_id = ANY($1)"
 
 const selectSharedUsersSQL = "" +
 	"SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" +
@@ -365,7 +360,36 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
 		return nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed")
-	return rowsToStreamEvents(rows)
+	return currentRoomStateRowsToStreamEvents(rows)
+}
+
+func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
+	var events []types.StreamEvent
+	for rows.Next() {
+		var (
+			eventID           string
+			streamPos         types.StreamPosition
+			eventBytes        []byte
+			historyVisibility gomatrixserverlib.HistoryVisibility
+		)
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &historyVisibility); err != nil {
+			return nil, err
+		}
+		// TODO: Handle redacted events
+		var ev gomatrixserverlib.HeaderedEvent
+		if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil {
+			return nil, err
+		}
+
+		ev.Visibility = historyVisibility
+
+		events = append(events, types.StreamEvent{
+			HeaderedEvent:  &ev,
+			StreamPosition: streamPos,
+		})
+	}
+
+	return events, nil
 }
 
 func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) {
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index c4019fed2..ff45e786e 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -88,12 +88,7 @@ const selectStateEventSQL = "" +
 	"SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
 
 const selectEventsWithEventIDsSQL = "" +
-	// TODO: The session_id and transaction_id blanks are here because
-	// the rowsToStreamEvents expects there to be exactly seven columns. We need to
-	// figure out if these really need to be in the DB, and if so, we need a
-	// better permanent fix for this. - neilalexander, 2 Jan 2020
-	"SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" +
-	" FROM syncapi_current_room_state WHERE event_id IN ($1)"
+	"SELECT event_id, added_at, headered_event_json, history_visibility FROM syncapi_current_room_state WHERE event_id IN ($1)"
 
 const selectSharedUsersSQL = "" +
 	"SELECT state_key FROM syncapi_current_room_state WHERE room_id IN(" +
@@ -378,7 +373,7 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
 			return nil, err
 		}
 		start = start + n
-		events, err := rowsToStreamEvents(rows)
+		events, err := currentRoomStateRowsToStreamEvents(rows)
 		internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed")
 		if err != nil {
 			return nil, err
@@ -388,6 +383,35 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
 	return res, nil
 }
 
+func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
+	var events []types.StreamEvent
+	for rows.Next() {
+		var (
+			eventID           string
+			streamPos         types.StreamPosition
+			eventBytes        []byte
+			historyVisibility gomatrixserverlib.HistoryVisibility
+		)
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &historyVisibility); err != nil {
+			return nil, err
+		}
+		// TODO: Handle redacted events
+		var ev gomatrixserverlib.HeaderedEvent
+		if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil {
+			return nil, err
+		}
+
+		ev.Visibility = historyVisibility
+
+		events = append(events, types.StreamEvent{
+			HeaderedEvent:  &ev,
+			StreamPosition: streamPos,
+		})
+	}
+
+	return events, nil
+}
+
 func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) {
 	result := []*gomatrixserverlib.HeaderedEvent{}
 	for rows.Next() {
diff --git a/syncapi/storage/tables/current_room_state_test.go b/syncapi/storage/tables/current_room_state_test.go
new file mode 100644
index 000000000..23287c500
--- /dev/null
+++ b/syncapi/storage/tables/current_room_state_test.go
@@ -0,0 +1,88 @@
+package tables_test
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/storage/postgres"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/syncapi/storage/tables"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/test"
+)
+
+func newCurrentRoomStateTable(t *testing.T, dbType test.DBType) (tables.CurrentRoomState, *sql.DB, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	if err != nil {
+		t.Fatalf("failed to open db: %s", err)
+	}
+
+	var tab tables.CurrentRoomState
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresCurrentRoomStateTable(db)
+	case test.DBTypeSQLite:
+		var stream sqlite3.StreamIDStatements
+		if err = stream.Prepare(db); err != nil {
+			t.Fatalf("failed to prepare stream stmts: %s", err)
+		}
+		tab, err = sqlite3.NewSqliteCurrentRoomStateTable(db, &stream)
+	}
+	if err != nil {
+		t.Fatalf("failed to make new table: %s", err)
+	}
+	return tab, db, close
+}
+
+func TestCurrentRoomStateTable(t *testing.T) {
+	ctx := context.Background()
+	alice := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		tab, db, close := newCurrentRoomStateTable(t, dbType)
+		defer close()
+		events := room.CurrentState()
+		err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error {
+			for i, ev := range events {
+				err := tab.UpsertRoomState(ctx, txn, ev, nil, types.StreamPosition(i))
+				if err != nil {
+					return fmt.Errorf("failed to UpsertRoomState: %w", err)
+				}
+			}
+			wantEventIDs := []string{
+				events[0].EventID(), events[1].EventID(), events[2].EventID(), events[3].EventID(),
+			}
+			gotEvents, err := tab.SelectEventsWithEventIDs(ctx, txn, wantEventIDs)
+			if err != nil {
+				return fmt.Errorf("failed to SelectEventsWithEventIDs: %w", err)
+			}
+			if len(gotEvents) != len(wantEventIDs) {
+				return fmt.Errorf("SelectEventsWithEventIDs\ngot %d, want %d results", len(gotEvents), len(wantEventIDs))
+			}
+			gotEventIDs := make(map[string]struct{}, len(gotEvents))
+			for _, event := range gotEvents {
+				if event.ExcludeFromSync {
+					return fmt.Errorf("SelectEventsWithEventIDs ExcludeFromSync should be false for current room state event %+v", event)
+				}
+				gotEventIDs[event.EventID()] = struct{}{}
+			}
+			for _, id := range wantEventIDs {
+				if _, ok := gotEventIDs[id]; !ok {
+					return fmt.Errorf("SelectEventsWithEventIDs\nexpected id %q not returned", id)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	})
+}

From fec3ee384bad7f05e13893a97f65590d2284438b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 3 Oct 2022 12:59:56 +0100
Subject: [PATCH 30/73] Stop CPU burn in `PerformMarkAsStaleIfNeeded`

---
 keyserver/internal/internal.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 6309066d7..017c29e84 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -233,7 +233,7 @@ func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *ap
 		return err
 	}
 	if len(knownDevices) == 0 {
-		return fmt.Errorf("unknown user %s", req.UserID)
+		return nil // fmt.Errorf("unknown user %s", req.UserID)
 	}
 
 	for i := range knownDevices {

From 50fa50a3430eed79ddf897d9841fe0b652a32341 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 3 Oct 2022 13:05:58 +0100
Subject: [PATCH 31/73] Update P2P base directories

---
 cmd/dendrite-demo-pinecone/main.go  | 3 +++
 cmd/dendrite-demo-yggdrasil/main.go | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index fb988ebe1..be34365b4 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -143,6 +143,9 @@ func main() {
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.ClientAPI.RegistrationDisabled = false
 		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+		cfg.MediaAPI.BasePath = config.Path(*instanceDir)
+		cfg.SyncAPI.Fulltext.Enabled = true
+		cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir)
 		if err := cfg.Derive(); err != nil {
 			panic(err)
 		}
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index cd0066679..38c25cdec 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -134,6 +134,9 @@ func main() {
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.ClientAPI.RegistrationDisabled = false
 		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+		cfg.MediaAPI.BasePath = config.Path(*instanceDir)
+		cfg.SyncAPI.Fulltext.Enabled = true
+		cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir)
 		if err := cfg.Derive(); err != nil {
 			panic(err)
 		}

From 34451d21b8b069e233700dfe4f000f45984be6ea Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 3 Oct 2022 14:35:10 +0100
Subject: [PATCH 32/73] P2P demo tweaks

---
 build/gobind-pinecone/monolith.go | 38 ++++++++++++++++---------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index b0d3fa285..5e9ff8ef3 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -103,15 +103,15 @@ func (m *DendriteMonolith) SessionCount() int {
 
 func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) {
 	m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{
-		Name: name,
-		Index: index,
-		Mtu: mtu,
-		Up: up,
-		Broadcast: broadcast,
-		Loopback: loopback,
+		Name:         name,
+		Index:        index,
+		Mtu:          mtu,
+		Up:           up,
+		Broadcast:    broadcast,
+		Loopback:     loopback,
 		PointToPoint: pointToPoint,
-		Multicast: multicast,
-		Addrs: addrs,
+		Multicast:    multicast,
+		Addrs:        addrs,
 	})
 }
 
@@ -279,19 +279,21 @@ func (m *DendriteMonolith) Start() {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Global.JetStream.InMemory = true
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix))
-	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
-	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
+	cfg.Global.JetStream.InMemory = false
+	cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix))
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix)))
+	cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
+	cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
 	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+	cfg.SyncAPI.Fulltext.Enabled = true
+	cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search"))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}

From ba66b5a3b9844f4cb3b3f8f2c814894ebb415695 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 3 Oct 2022 14:43:38 +0100
Subject: [PATCH 33/73] Allow multiple static peers in Pinecone iOS/Android
 demos

---
 build/gobind-pinecone/monolith.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go
index 5e9ff8ef3..4a96e4bef 100644
--- a/build/gobind-pinecone/monolith.go
+++ b/build/gobind-pinecone/monolith.go
@@ -126,7 +126,9 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 
 func (m *DendriteMonolith) SetStaticPeer(uri string) {
 	m.PineconeManager.RemovePeers()
-	m.PineconeManager.AddPeer(strings.TrimSpace(uri))
+	for _, uri := range strings.Split(uri, ",") {
+		m.PineconeManager.AddPeer(strings.TrimSpace(uri))
+	}
 }
 
 func (m *DendriteMonolith) DisconnectType(peertype int) {

From 34ed316584df916f6959808669c998580f76d88f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 3 Oct 2022 19:35:26 +0100
Subject: [PATCH 34/73] Fix docs

---
 docs/administration/1_createusers.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md
index 94399a04a..24eba666d 100644
--- a/docs/administration/1_createusers.md
+++ b/docs/administration/1_createusers.md
@@ -1,4 +1,3 @@
-
 ---
 title: Creating user accounts
 parent: Administration
@@ -15,7 +14,7 @@ User accounts can be created on a Dendrite instance in a number of ways.
 The `create-account` tool is built in the `bin` folder when building Dendrite with
 the `build.sh` script.
 
-It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires 
+It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires
 shared secret registration to be enabled as explained below.
 
 An example of using `create-account` to create a **normal account**:
@@ -32,7 +31,7 @@ To create a new **admin account**, add the `-admin` flag:
 ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
 ```
 
-By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using 
+By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using
 the `-url` flag:
 
 ```bash
@@ -44,6 +43,7 @@ An example of using `create-account` when running in **Docker**, having found th
 ```bash
 docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME
 ```
+
 ```bash
 docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
 ```

From e6c992ba8bcbc59706d6dd55db3b237ebbdc8e7e Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:41:06 +0200
Subject: [PATCH 35/73] Update Dockerfile (#2342)

Updates/adds a new multistage (build-kit) Dockerfile. (if accepted,
could make `Dockerfile.monolith` and `Dockerfile.polylith` in
`build/docker` obsolete)
There's no huge difference between the dockerfiles, except this uses a
non-root user when running the container, also doesn't copy the working
directory to the image when building.
Also adds vulnerabilities scans using
[Trivy](https://github.com/aquasecurity/trivy) for the created docker
images. (untested)

Building images is done using
```
docker build . --target image-monolith -t dendrite-monolith
docker build . --target image-polylith -t dendrite-polylith
```

As noted in the comments, only adds `dendrite-polylith-multi` to the
polylith image and all required binaries to the monolith image.
Probably needs some docs updating, if this is accepted.

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 .github/workflows/dendrite.yml                |   6 +-
 .github/workflows/docker.yml                  | 106 +++++++++++++-----
 Dockerfile                                    |  99 ++++++++++++++++
 .../dendritejs-pinecone/jsServer.go           |   0
 {cmd => build}/dendritejs-pinecone/main.go    |   0
 .../dendritejs-pinecone/main_noop.go          |   0
 .../dendritejs-pinecone/main_test.go          |   0
 build/docker/Dockerfile.demo-pinecone         |  25 -----
 build/docker/Dockerfile.monolith              |  25 -----
 build/docker/Dockerfile.polylith              |  25 -----
 build/docker/README.md                        |  14 ++-
 build/docker/crossbuild.sh                    |  67 +++++++++++
 build/docker/images-build.sh                  |   5 +-
 13 files changed, 257 insertions(+), 115 deletions(-)
 create mode 100644 Dockerfile
 rename {cmd => build}/dendritejs-pinecone/jsServer.go (100%)
 rename {cmd => build}/dendritejs-pinecone/main.go (100%)
 rename {cmd => build}/dendritejs-pinecone/main_noop.go (100%)
 rename {cmd => build}/dendritejs-pinecone/main_test.go (100%)
 delete mode 100644 build/docker/Dockerfile.demo-pinecone
 delete mode 100644 build/docker/Dockerfile.monolith
 delete mode 100644 build/docker/Dockerfile.polylith
 create mode 100644 build/docker/crossbuild.sh

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index be3c7c173..524d36039 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -284,7 +284,7 @@ jobs:
         API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Run Sytest
         run: /bootstrap.sh dendrite
         working-directory: /src
@@ -344,8 +344,8 @@ jobs:
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
 
-      - name: Run actions/checkout@v2 for dendrite
-        uses: actions/checkout@v2
+      - name: Run actions/checkout@v3 for dendrite
+        uses: actions/checkout@v3
         with:
           path: dendrite
 
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b4e24e52f..358037c02 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -21,26 +21,32 @@ jobs:
   monolith:
     name: Monolith image
     runs-on: ubuntu-latest
+    needs: build-flags
     permissions:
       contents: read
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
+        uses: actions/checkout@v3
+      - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -49,27 +55,41 @@ jobs:
       - name: Build main monolith image
         if: github.ref_name == 'main'
         id: docker_build_monolith
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.monolith
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: "trivy-results.sarif"
+
       - name: Build release monolith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_monolith_release
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.monolith
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -81,26 +101,32 @@ jobs:
   polylith:
     name: Polylith image
     runs-on: ubuntu-latest
+    needs: build-flags
     permissions:
       contents: read
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
+        uses: actions/checkout@v3
+      - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -109,27 +135,40 @@ jobs:
       - name: Build main polylith image
         if: github.ref_name == 'main'
         id: docker_build_polylith
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.polylith
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: "trivy-results.sarif"
+
       - name: Build release polylith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_polylith_release
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.polylith
+          target: polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -141,26 +180,32 @@ jobs:
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest
+    needs: build-flags
     permissions:
       contents: read
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get release tag
+        uses: actions/checkout@v3
+      - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
+          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
+          [ ${BRANCH} == "main" ] && BRANCH=""
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -169,12 +214,12 @@ jobs:
       - name: Build main pinecone demo image
         if: github.ref_name == 'main'
         id: docker_build_demo_pinecone
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.demo-pinecone
+          target: demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -184,12 +229,13 @@ jobs:
       - name: Build release pinecone demo image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_demo_pinecone_release
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          file: ./build/docker/Dockerfile.demo-pinecone
+          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
+          target: demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000..bf5831832
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,99 @@
+#syntax=docker/dockerfile:1.2
+
+#
+# base installs required dependencies and runs go mod download to cache dependencies
+#
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
+RUN apk --update --no-cache add bash build-base curl
+
+#
+# build creates all needed binaries
+#
+FROM base AS build
+WORKDIR /src
+ARG TARGETOS
+ARG TARGETARCH
+ARG FLAGS
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    sh ./build/docker/crossbuild.sh
+
+#
+# The dendrite base image; mainly creates a user and switches to it
+#
+FROM alpine:latest AS dendrite-base
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
+LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
+RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D
+USER dendrite
+WORKDIR /home/dendrite
+
+#
+# Builds the polylith image and only contains the polylith binary
+#
+FROM dendrite-base AS polylith
+LABEL org.opencontainers.image.title="Dendrite (Polylith)"
+
+COPY --from=build /out/dendrite-polylith-multi /usr/bin/
+
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
+
+#
+# Builds the monolith image and contains all required binaries
+#
+FROM dendrite-base AS monolith
+LABEL org.opencontainers.image.title="Dendrite (Monolith)"
+
+COPY --from=build /out/create-account /usr/bin/create-account
+COPY --from=build /out/generate-config /usr/bin/generate-config
+COPY --from=build /out/generate-keys /usr/bin/generate-keys
+COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
+EXPOSE 8008 8448
+
+#
+# Builds the P2P demo image and contains all required binaries
+#
+FROM dendrite-base AS demo-pinecone
+LABEL org.opencontainers.image.title="Dendrite (P2P Demo)"
+
+COPY --from=build /out/create-account /usr/bin/create-account
+COPY --from=build /out/generate-config /usr/bin/generate-config
+COPY --from=build /out/generate-keys /usr/bin/generate-keys
+COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
+EXPOSE 8008 8448
+
+#
+# Builds the Complement image, used for integration tests
+#
+FROM base AS complement
+LABEL org.opencontainers.image.title="Dendrite (Complement)"
+RUN apk add --no-cache sqlite openssl ca-certificates
+COPY --from=build /out/* /usr/bin/
+RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone
+
+WORKDIR /dendrite
+RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
+    mkdir /ca && \
+    openssl genrsa -out /ca/ca.key 2048 && \
+    openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
+
+ENV SERVER_NAME=localhost
+ENV API=0
+EXPOSE 8008 8448
+
+# At runtime, generate TLS cert based on the CA now mounted at /ca
+# At runtime, replace the SERVER_NAME with what we are told
+CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
+    /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+    cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+    /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/cmd/dendritejs-pinecone/jsServer.go b/build/dendritejs-pinecone/jsServer.go
similarity index 100%
rename from cmd/dendritejs-pinecone/jsServer.go
rename to build/dendritejs-pinecone/jsServer.go
diff --git a/cmd/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go
similarity index 100%
rename from cmd/dendritejs-pinecone/main.go
rename to build/dendritejs-pinecone/main.go
diff --git a/cmd/dendritejs-pinecone/main_noop.go b/build/dendritejs-pinecone/main_noop.go
similarity index 100%
rename from cmd/dendritejs-pinecone/main_noop.go
rename to build/dendritejs-pinecone/main_noop.go
diff --git a/cmd/dendritejs-pinecone/main_test.go b/build/dendritejs-pinecone/main_test.go
similarity index 100%
rename from cmd/dendritejs-pinecone/main_test.go
rename to build/dendritejs-pinecone/main_test.go
diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone
deleted file mode 100644
index 133c63c53..000000000
--- a/build/docker/Dockerfile.demo-pinecone
+++ /dev/null
@@ -1,25 +0,0 @@
-FROM docker.io/golang:1.19-alpine AS base
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
-RUN go build -trimpath -o bin/ ./cmd/create-account
-RUN go build -trimpath -o bin/ ./cmd/generate-keys
-
-FROM alpine:latest
-LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-
-COPY --from=base /build/bin/* /usr/bin/
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
deleted file mode 100644
index 3180e9626..000000000
--- a/build/docker/Dockerfile.monolith
+++ /dev/null
@@ -1,25 +0,0 @@
-FROM docker.io/golang:1.19-alpine AS base
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
-RUN go build -trimpath -o bin/ ./cmd/create-account
-RUN go build -trimpath -o bin/ ./cmd/generate-keys
-
-FROM alpine:latest
-LABEL org.opencontainers.image.title="Dendrite (Monolith)"
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-
-COPY --from=base /build/bin/* /usr/bin/
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
deleted file mode 100644
index 79f8a5f23..000000000
--- a/build/docker/Dockerfile.polylith
+++ /dev/null
@@ -1,25 +0,0 @@
-FROM docker.io/golang:1.19-alpine AS base
-
-RUN apk --update --no-cache add bash build-base
-
-WORKDIR /build
-
-COPY . /build
-
-RUN mkdir -p bin
-RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
-RUN go build -trimpath -o bin/ ./cmd/create-account
-RUN go build -trimpath -o bin/ ./cmd/generate-keys
-
-FROM alpine:latest
-LABEL org.opencontainers.image.title="Dendrite (Polylith)"
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-
-COPY --from=base /build/bin/* /usr/bin/
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
diff --git a/build/docker/README.md b/build/docker/README.md
index 261519fde..14a9c8594 100644
--- a/build/docker/README.md
+++ b/build/docker/README.md
@@ -9,11 +9,15 @@ They can be found on Docker Hub:
 
 ## Dockerfiles
 
-The `Dockerfile` builds the base image which contains all of the Dendrite
-components. The `Dockerfile.component` file takes the given component, as
-specified with `--buildarg component=` from the base image and produce
-smaller component-specific images, which are substantially smaller and do
-not contain the Go toolchain etc.
+The `Dockerfile` is a multistage file which can build all three Dendrite
+images depending on the supplied `--target`. From the root of the Dendrite
+repository, run:
+
+```
+docker build . --target monolith -t matrixdotorg/dendrite-monolith
+docker build . --target polylith -t matrixdotorg/dendrite-monolith
+docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith
+```
 
 ## Compose files
 
diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh
new file mode 100644
index 000000000..46e5d7e9b
--- /dev/null
+++ b/build/docker/crossbuild.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+set -e
+
+# In order to cross-compile with the multi-stage Docker builds, we need to
+# ensure that the suitable toolchain for cross-compiling is installed. Since
+# the images are Alpine-based, we will use musl. Download and install the
+# toolchain inside the build container.
+
+USERARCH=`go env GOARCH`
+GOARCH="$TARGETARCH"
+GOOS="linux"
+
+echo "Target arch: $TARGETARCH"
+echo "User arch:   $USERARCH"
+
+if [ "$TARGETARCH" != "$USERARCH" ]; then
+    if [ "$USERARCH" != "amd64" ]; then
+      echo "Cross-compiling only supported on amd64"
+      exit 1
+    fi
+
+    echo "Cross compile"
+    case $GOARCH in
+    arm64)
+        curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=aarch64-linux-musl-gcc
+        ;;
+
+    amd64)
+        curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=x86_64-linux-musl-gcc
+        ;;
+
+    386)
+        curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=i686-linux-musl-gcc
+        ;;
+
+    arm)
+        curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=armv7l-linux-musleabihf-gcc
+        ;;
+
+    s390x)
+        curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=s390x-linux-musl-gcc
+        ;;
+
+    ppc64le)
+        curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
+        export CC=powerpc64le-linux-musl-gcc
+        ;;
+
+    *)
+        echo "Unsupported GOARCH=${GOARCH}"
+        exit 1
+        ;;
+    esac
+else
+    echo "Native compile"
+fi
+
+# Output the go environment just in case it is useful for debugging.
+go env
+
+# Build Dendrite and tools, statically linking them.
+CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/...
diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index c2c140685..1a8326151 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -6,5 +6,6 @@ TAG=${1:-latest}
 
 echo "Building tag '${TAG}'"
 
-docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
-docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .
\ No newline at end of file
+docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
+docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
+docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
\ No newline at end of file

From ede4632835929bf93d0cb57df58c06efd94fa7e0 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 10:43:58 +0100
Subject: [PATCH 36/73] Fix Docker GHA

---
 .github/workflows/docker.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 358037c02..c15b6c1a4 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -21,7 +21,6 @@ jobs:
   monolith:
     name: Monolith image
     runs-on: ubuntu-latest
-    needs: build-flags
     permissions:
       contents: read
       packages: write
@@ -101,7 +100,6 @@ jobs:
   polylith:
     name: Polylith image
     runs-on: ubuntu-latest
-    needs: build-flags
     permissions:
       contents: read
       packages: write
@@ -180,7 +178,6 @@ jobs:
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest
-    needs: build-flags
     permissions:
       contents: read
       packages: write

From 98b73652e00edb8304aa4a3471aa29151f025297 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:07:54 +0100
Subject: [PATCH 37/73] Try to populate `-ldflags` in Docker builds

---
 .github/workflows/docker.yml | 6 +++---
 build/docker/crossbuild.sh   | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index c15b6c1a4..e0db280fa 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -35,6 +35,7 @@ jobs:
           BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
           [ ${BRANCH} == "main" ] && BRANCH=""
           echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -59,7 +60,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
           target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -114,6 +114,7 @@ jobs:
           BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
           [ ${BRANCH} == "main" ] && BRANCH=""
           echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -138,7 +139,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
           target: polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -192,6 +192,7 @@ jobs:
           BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
           [ ${BRANCH} == "main" ] && BRANCH=""
           echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -231,7 +232,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
           target: demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh
index 46e5d7e9b..957893a0a 100644
--- a/build/docker/crossbuild.sh
+++ b/build/docker/crossbuild.sh
@@ -12,6 +12,7 @@ GOOS="linux"
 
 echo "Target arch: $TARGETARCH"
 echo "User arch:   $USERARCH"
+echo "Link flags:  $FLAGS"
 
 if [ "$TARGETARCH" != "$USERARCH" ]; then
     if [ "$USERARCH" != "amd64" ]; then

From 085bf5e28b49948209b464c5b12c02775e6c1d70 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:33:05 +0100
Subject: [PATCH 38/73] Revert Docker changes

---
 .github/workflows/dendrite.yml                |   6 +-
 .github/workflows/docker.yml                  | 103 +++++-------------
 Dockerfile                                    |  99 -----------------
 build/docker/Dockerfile.demo-pinecone         |  25 +++++
 build/docker/Dockerfile.monolith              |  25 +++++
 build/docker/Dockerfile.polylith              |  25 +++++
 build/docker/README.md                        |  14 +--
 build/docker/crossbuild.sh                    |  68 ------------
 build/docker/images-build.sh                  |   5 +-
 .../dendritejs-pinecone/jsServer.go           |   0
 {build => cmd}/dendritejs-pinecone/main.go    |   0
 .../dendritejs-pinecone/main_noop.go          |   0
 .../dendritejs-pinecone/main_test.go          |   0
 13 files changed, 115 insertions(+), 255 deletions(-)
 delete mode 100644 Dockerfile
 create mode 100644 build/docker/Dockerfile.demo-pinecone
 create mode 100644 build/docker/Dockerfile.monolith
 create mode 100644 build/docker/Dockerfile.polylith
 delete mode 100644 build/docker/crossbuild.sh
 rename {build => cmd}/dendritejs-pinecone/jsServer.go (100%)
 rename {build => cmd}/dendritejs-pinecone/main.go (100%)
 rename {build => cmd}/dendritejs-pinecone/main_noop.go (100%)
 rename {build => cmd}/dendritejs-pinecone/main_test.go (100%)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index 524d36039..be3c7c173 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -284,7 +284,7 @@ jobs:
         API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Run Sytest
         run: /bootstrap.sh dendrite
         working-directory: /src
@@ -344,8 +344,8 @@ jobs:
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
 
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v2 for dendrite
+        uses: actions/checkout@v2
         with:
           path: dendrite
 
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index e0db280fa..b4e24e52f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -26,27 +26,21 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
+        uses: actions/checkout@v2
+      - name: Get release tag
         if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -55,40 +49,27 @@ jobs:
       - name: Build main monolith image
         if: github.ref_name == 'main'
         id: docker_build_monolith
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: monolith
+          file: ./build/docker/Dockerfile.monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release monolith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_monolith_release
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}"
-          target: monolith
+          file: ./build/docker/Dockerfile.monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -105,27 +86,21 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
+        uses: actions/checkout@v2
+      - name: Get release tag
         if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -134,39 +109,27 @@ jobs:
       - name: Build main polylith image
         if: github.ref_name == 'main'
         id: docker_build_polylith
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: polylith
+          file: ./build/docker/Dockerfile.polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
             ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release polylith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_polylith_release
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: polylith
+          file: ./build/docker/Dockerfile.polylith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -183,27 +146,21 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
+        uses: actions/checkout@v2
+      - name: Get release tag
         if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-          echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -212,12 +169,12 @@ jobs:
       - name: Build main pinecone demo image
         if: github.ref_name == 'main'
         id: docker_build_demo_pinecone
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: demo-pinecone
+          file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -227,12 +184,12 @@ jobs:
       - name: Build release pinecone demo image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_demo_pinecone_release
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          target: demo-pinecone
+          file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index bf5831832..000000000
--- a/Dockerfile
+++ /dev/null
@@ -1,99 +0,0 @@
-#syntax=docker/dockerfile:1.2
-
-#
-# base installs required dependencies and runs go mod download to cache dependencies
-#
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
-RUN apk --update --no-cache add bash build-base curl
-
-#
-# build creates all needed binaries
-#
-FROM base AS build
-WORKDIR /src
-ARG TARGETOS
-ARG TARGETARCH
-ARG FLAGS
-RUN --mount=target=. \
-    --mount=type=cache,target=/root/.cache/go-build \
-    sh ./build/docker/crossbuild.sh
-
-#
-# The dendrite base image; mainly creates a user and switches to it
-#
-FROM alpine:latest AS dendrite-base
-LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
-LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
-LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
-RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D
-USER dendrite
-WORKDIR /home/dendrite
-
-#
-# Builds the polylith image and only contains the polylith binary
-#
-FROM dendrite-base AS polylith
-LABEL org.opencontainers.image.title="Dendrite (Polylith)"
-
-COPY --from=build /out/dendrite-polylith-multi /usr/bin/
-
-ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
-
-#
-# Builds the monolith image and contains all required binaries
-#
-FROM dendrite-base AS monolith
-LABEL org.opencontainers.image.title="Dendrite (Monolith)"
-
-COPY --from=build /out/create-account /usr/bin/create-account
-COPY --from=build /out/generate-config /usr/bin/generate-config
-COPY --from=build /out/generate-keys /usr/bin/generate-keys
-COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
-
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
-EXPOSE 8008 8448
-
-#
-# Builds the P2P demo image and contains all required binaries
-#
-FROM dendrite-base AS demo-pinecone
-LABEL org.opencontainers.image.title="Dendrite (P2P Demo)"
-
-COPY --from=build /out/create-account /usr/bin/create-account
-COPY --from=build /out/generate-config /usr/bin/generate-config
-COPY --from=build /out/generate-keys /usr/bin/generate-keys
-COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
-EXPOSE 8008 8448
-
-#
-# Builds the Complement image, used for integration tests
-#
-FROM base AS complement
-LABEL org.opencontainers.image.title="Dendrite (Complement)"
-RUN apk add --no-cache sqlite openssl ca-certificates
-COPY --from=build /out/* /usr/bin/
-RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone
-
-WORKDIR /dendrite
-RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
-    mkdir /ca && \
-    openssl genrsa -out /ca/ca.key 2048 && \
-    openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
-
-ENV SERVER_NAME=localhost
-ENV API=0
-EXPOSE 8008 8448
-
-# At runtime, generate TLS cert based on the CA now mounted at /ca
-# At runtime, replace the SERVER_NAME with what we are told
-CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
-    /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
-    cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
\ No newline at end of file
diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone
new file mode 100644
index 000000000..133c63c53
--- /dev/null
+++ b/build/docker/Dockerfile.demo-pinecone
@@ -0,0 +1,25 @@
+FROM docker.io/golang:1.19-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
+
+FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+COPY --from=base /build/bin/* /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith
new file mode 100644
index 000000000..3180e9626
--- /dev/null
+++ b/build/docker/Dockerfile.monolith
@@ -0,0 +1,25 @@
+FROM docker.io/golang:1.19-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
+
+FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Monolith)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+COPY --from=base /build/bin/* /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith
new file mode 100644
index 000000000..79f8a5f23
--- /dev/null
+++ b/build/docker/Dockerfile.polylith
@@ -0,0 +1,25 @@
+FROM docker.io/golang:1.19-alpine AS base
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
+RUN go build -trimpath -o bin/ ./cmd/create-account
+RUN go build -trimpath -o bin/ ./cmd/generate-keys
+
+FROM alpine:latest
+LABEL org.opencontainers.image.title="Dendrite (Polylith)"
+LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
+LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+COPY --from=base /build/bin/* /usr/bin/
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
diff --git a/build/docker/README.md b/build/docker/README.md
index 14a9c8594..261519fde 100644
--- a/build/docker/README.md
+++ b/build/docker/README.md
@@ -9,15 +9,11 @@ They can be found on Docker Hub:
 
 ## Dockerfiles
 
-The `Dockerfile` is a multistage file which can build all three Dendrite
-images depending on the supplied `--target`. From the root of the Dendrite
-repository, run:
-
-```
-docker build . --target monolith -t matrixdotorg/dendrite-monolith
-docker build . --target polylith -t matrixdotorg/dendrite-monolith
-docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith
-```
+The `Dockerfile` builds the base image which contains all of the Dendrite
+components. The `Dockerfile.component` file takes the given component, as
+specified with `--buildarg component=` from the base image and produce
+smaller component-specific images, which are substantially smaller and do
+not contain the Go toolchain etc.
 
 ## Compose files
 
diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh
deleted file mode 100644
index 957893a0a..000000000
--- a/build/docker/crossbuild.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/sh
-set -e
-
-# In order to cross-compile with the multi-stage Docker builds, we need to
-# ensure that the suitable toolchain for cross-compiling is installed. Since
-# the images are Alpine-based, we will use musl. Download and install the
-# toolchain inside the build container.
-
-USERARCH=`go env GOARCH`
-GOARCH="$TARGETARCH"
-GOOS="linux"
-
-echo "Target arch: $TARGETARCH"
-echo "User arch:   $USERARCH"
-echo "Link flags:  $FLAGS"
-
-if [ "$TARGETARCH" != "$USERARCH" ]; then
-    if [ "$USERARCH" != "amd64" ]; then
-      echo "Cross-compiling only supported on amd64"
-      exit 1
-    fi
-
-    echo "Cross compile"
-    case $GOARCH in
-    arm64)
-        curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=aarch64-linux-musl-gcc
-        ;;
-
-    amd64)
-        curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=x86_64-linux-musl-gcc
-        ;;
-
-    386)
-        curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=i686-linux-musl-gcc
-        ;;
-
-    arm)
-        curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=armv7l-linux-musleabihf-gcc
-        ;;
-
-    s390x)
-        curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=s390x-linux-musl-gcc
-        ;;
-
-    ppc64le)
-        curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr
-        export CC=powerpc64le-linux-musl-gcc
-        ;;
-
-    *)
-        echo "Unsupported GOARCH=${GOARCH}"
-        exit 1
-        ;;
-    esac
-else
-    echo "Native compile"
-fi
-
-# Output the go environment just in case it is useful for debugging.
-go env
-
-# Build Dendrite and tools, statically linking them.
-CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/...
diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh
index 1a8326151..c2c140685 100755
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@@ -6,6 +6,5 @@ TAG=${1:-latest}
 
 echo "Building tag '${TAG}'"
 
-docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
-docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
-docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
\ No newline at end of file
+docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
+docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .
\ No newline at end of file
diff --git a/build/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go
similarity index 100%
rename from build/dendritejs-pinecone/jsServer.go
rename to cmd/dendritejs-pinecone/jsServer.go
diff --git a/build/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
similarity index 100%
rename from build/dendritejs-pinecone/main.go
rename to cmd/dendritejs-pinecone/main.go
diff --git a/build/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go
similarity index 100%
rename from build/dendritejs-pinecone/main_noop.go
rename to cmd/dendritejs-pinecone/main_noop.go
diff --git a/build/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go
similarity index 100%
rename from build/dendritejs-pinecone/main_test.go
rename to cmd/dendritejs-pinecone/main_test.go

From a767102f8a74031b5d0612d985c7589dfe0e0821 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 11:34:52 +0100
Subject: [PATCH 39/73] Reduce `max_open_conns` in monolith sample config

---
 dendrite-sample.monolith.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index e41e83d7c..f0fa386d1 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -37,7 +37,7 @@ global:
   # you must configure the "database" block for each component instead.
   database:
     connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable
-    max_open_conns: 100
+    max_open_conns: 90
     max_idle_conns: 5
     conn_max_lifetime: -1
 

From 3da182212e86daca1d7019efbd424b2bd38b8b3c Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 13:02:41 +0100
Subject: [PATCH 40/73] Track reasons why the process is in a degraded state

---
 setup/base/base.go       |  8 +++++++-
 setup/jetstream/nats.go  |  6 +++---
 setup/process/process.go | 35 +++++++++++++++++++++++------------
 3 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/setup/base/base.go b/setup/base/base.go
index 0636c7b8d..2e3a3a195 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -467,8 +468,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		w.WriteHeader(200)
 	})
 	b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) {
-		if b.ProcessContext.IsDegraded() {
+		if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded {
 			w.WriteHeader(503)
+			_ = json.NewEncoder(w).Encode(struct {
+				Warnings []string `json:"warnings"`
+			}{
+				Warnings: reasons,
+			})
 			return
 		}
 		w.WriteHeader(200)
diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go
index 7409fd6c8..af4eb2949 100644
--- a/setup/jetstream/nats.go
+++ b/setup/jetstream/nats.go
@@ -169,9 +169,9 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
 					// We've managed to add the stream in memory.  What's on the
 					// disk will be left alone, but our ability to recover from a
 					// future crash will be limited. Yell about it.
-					sentry.CaptureException(fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible", namespaced.Name))
-					logrus.Warn("Stream is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible")
-					process.Degraded()
+					err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name)
+					sentry.CaptureException(err)
+					process.Degraded(err)
 				}
 			}
 		}
diff --git a/setup/process/process.go b/setup/process/process.go
index 06ef60217..b2d2844a8 100644
--- a/setup/process/process.go
+++ b/setup/process/process.go
@@ -2,19 +2,18 @@ package process
 
 import (
 	"context"
-	"fmt"
 	"sync"
 
 	"github.com/getsentry/sentry-go"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 )
 
 type ProcessContext struct {
-	wg       *sync.WaitGroup    // used to wait for components to shutdown
-	ctx      context.Context    // cancelled when Stop is called
-	shutdown context.CancelFunc // shut down Dendrite
-	degraded atomic.Bool
+	mu       sync.RWMutex
+	wg       *sync.WaitGroup     // used to wait for components to shutdown
+	ctx      context.Context     // cancelled when Stop is called
+	shutdown context.CancelFunc  // shut down Dendrite
+	degraded map[string]struct{} // reasons why the process is degraded
 }
 
 func NewProcessContext() *ProcessContext {
@@ -50,13 +49,25 @@ func (b *ProcessContext) WaitForComponentsToFinish() {
 	b.wg.Wait()
 }
 
-func (b *ProcessContext) Degraded() {
-	if b.degraded.CompareAndSwap(false, true) {
-		logrus.Warn("Dendrite is running in a degraded state")
-		sentry.CaptureException(fmt.Errorf("Process is running in a degraded state"))
+func (b *ProcessContext) Degraded(err error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if _, ok := b.degraded[err.Error()]; !ok {
+		logrus.WithError(err).Warn("Dendrite has entered a degraded state")
+		sentry.CaptureException(err)
+		b.degraded[err.Error()] = struct{}{}
 	}
 }
 
-func (b *ProcessContext) IsDegraded() bool {
-	return b.degraded.Load()
+func (b *ProcessContext) IsDegraded() (bool, []string) {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	if len(b.degraded) == 0 {
+		return false, nil
+	}
+	reasons := make([]string, 0, len(b.degraded))
+	for reason := range b.degraded {
+		reasons = append(reasons, reason)
+	}
+	return true, reasons
 }

From ae10aac456e90dc6a3ea56ca4aceb4a4e5aa8e04 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 15:40:04 +0100
Subject: [PATCH 41/73] Don't perform a federated join after invite if we are
 already joined to the room (#2762)

If we are already joined to the room then it shouldn't matter if you
were invited or not, so this looks like a bug.
---
 roomserver/internal/perform/perform_join.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 43be54beb..167b375b7 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -237,7 +237,7 @@ func (r *Joiner) performJoinRoomByID(
 	// Force a federated join if we're dealing with a pending invite
 	// and we aren't in the room.
 	isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID)
-	if err == nil && isInvitePending {
+	if err == nil && !serverInRoom && isInvitePending {
 		_, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender)
 		if ierr != nil {
 			return "", "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err)

From 21f88819858dec022dec5919d6db5554605a9b8b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 4 Oct 2022 16:43:10 +0100
Subject: [PATCH 42/73] Add indexes that optimise `selectStateInRangeSQL`
 (#2764)

This gets rid of some expensive scans on `add_state_ids` and
`remove_state_ids`, turning them into much cheaper and faster index
scans instead.
---
 syncapi/storage/postgres/output_room_events_table.go | 2 ++
 syncapi/storage/sqlite3/output_room_events_table.go  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index cb092150d..b562e6804 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -76,6 +76,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL));
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL));
 `
 
 const insertEventSQL = "" +
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 165943027..d6a674b9c 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -55,6 +55,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
 CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL));
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL));
 `
 
 const insertEventSQL = "" +

From e0703522932ab4bf41b23d5229c7a4540b59d830 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 5 Oct 2022 11:14:33 +0200
Subject: [PATCH 43/73] Side effect import bleve analyzer languages (#2763)

... to actually allow different languages. Fixes #2761
Binary size increases by ~1MB.
---
 internal/fulltext/bleve.go | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go
index da8932f5c..7187861dd 100644
--- a/internal/fulltext/bleve.go
+++ b/internal/fulltext/bleve.go
@@ -21,6 +21,29 @@ import (
 	"strings"
 
 	"github.com/blevesearch/bleve/v2"
+
+	// side effect imports to allow all possible languages
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ar"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/cjk"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ckb"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/da"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/de"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/en"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/es"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fa"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fi"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fr"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hi"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hr"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hu"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/it"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/nl"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/no"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/pt"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ro"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ru"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/sv"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/tr"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/matrix-org/gomatrixserverlib"
 

From ebd137cf6b2fbd767625dc5289b0bef6d1e51971 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 11:07:17 +0100
Subject: [PATCH 44/73] Check PostgreSQL connection count (#2760)

This PR queries PostgreSQL for the `max_connections` and
`superuser_reserved_connections` settings and then ensures that
Dendrite's `max_open_conns` doesn't exceed the allowed value.

This is a really common source of configuration problems and can either
result in blocking queries or deadlocks, so it seems reasonable that we
complain as loudly as possible when it happens.
---
 internal/sqlutil/sqlutil.go | 39 +++++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go
index 0cdae6d30..789bceeac 100644
--- a/internal/sqlutil/sqlutil.go
+++ b/internal/sqlutil/sqlutil.go
@@ -2,6 +2,7 @@ package sqlutil
 
 import (
 	"database/sql"
+	"flag"
 	"fmt"
 	"regexp"
 
@@ -9,6 +10,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+var skipSanityChecks = flag.Bool("skip-db-sanity", false, "Ignore sanity checks on the database connections (NOT RECOMMENDED!)")
+
 // Open opens a database specified by its database driver name and a driver-specific data source name,
 // usually consisting of at least a database name and connection information. Includes tracing driver
 // if DENDRITE_TRACE_SQL=1
@@ -37,15 +40,39 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error)
 		return nil, err
 	}
 	if driverName != "sqlite3" {
-		logrus.WithFields(logrus.Fields{
-			"MaxOpenConns":    dbProperties.MaxOpenConns(),
-			"MaxIdleConns":    dbProperties.MaxIdleConns(),
-			"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
-			"dataSourceName":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
-		}).Debug("Setting DB connection limits")
+		logger := logrus.WithFields(logrus.Fields{
+			"max_open_conns":    dbProperties.MaxOpenConns(),
+			"max_idle_conns":    dbProperties.MaxIdleConns(),
+			"conn_max_lifetime": dbProperties.ConnMaxLifetime(),
+			"data_source_name":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
+		})
+		logger.Debug("Setting DB connection limits")
 		db.SetMaxOpenConns(dbProperties.MaxOpenConns())
 		db.SetMaxIdleConns(dbProperties.MaxIdleConns())
 		db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime())
+
+		if !*skipSanityChecks {
+			if dbProperties.MaxOpenConns() == 0 {
+				logrus.Warnf("WARNING: Configuring 'max_open_conns' to be unlimited is not recommended. This can result in bad performance or deadlocks.")
+			}
+
+			switch driverName {
+			case "postgres":
+				// Perform a quick sanity check if possible that we aren't trying to use more database
+				// connections than PostgreSQL is willing to give us.
+				var max, reserved int
+				if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='max_connections';").Scan(&max); err != nil {
+					return nil, fmt.Errorf("failed to find maximum connections: %w", err)
+				}
+				if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='superuser_reserved_connections';").Scan(&reserved); err != nil {
+					return nil, fmt.Errorf("failed to find reserved connections: %w", err)
+				}
+				if configured, allowed := dbProperties.MaxOpenConns(), max-reserved; configured > allowed {
+					logrus.Errorf("ERROR: The configured 'max_open_conns' is greater than the %d non-superuser connections that PostgreSQL is configured to allow. This can result in bad performance or deadlocks. Please pay close attention to your configured database connection counts. If you REALLY know what you are doing and want to override this error, pass the --skip-db-sanity option to Dendrite.", allowed)
+					return nil, fmt.Errorf("database sanity checks failed")
+				}
+			}
+		}
 	}
 	return db, nil
 }

From 8c0c3441d88a612ca8e9ba4f83c6ff29ca73f5d0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 5 Oct 2022 12:12:42 +0200
Subject: [PATCH 45/73] Add `RoomEventType` nats.Header to avoid unneeded
 unmarshalling (#2765)

---
 appservice/consumers/roomserver.go    |  5 +++++
 federationapi/consumers/roomserver.go |  7 +++++++
 roomserver/producers/roomevent.go     | 13 ++++++-------
 setup/jetstream/streams.go            |  7 ++++---
 test/testrig/jetstream.go             |  9 ++++-----
 userapi/consumers/roomserver.go       |  7 ++++---
 6 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go
index d44f32b38..ac68f4bd4 100644
--- a/appservice/consumers/roomserver.go
+++ b/appservice/consumers/roomserver.go
@@ -101,6 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage(
 	log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs))
 	events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs))
 	for _, msg := range msgs {
+		// Only handle events we care about
+		receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
+		if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInviteEvent {
+			continue
+		}
 		// Parse out the event JSON
 		var output api.OutputEvent
 		if err := json.Unmarshal(msg.Data, &output); err != nil {
diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go
index 349b50b05..a42733628 100644
--- a/federationapi/consumers/roomserver.go
+++ b/federationapi/consumers/roomserver.go
@@ -79,6 +79,13 @@ func (s *OutputRoomEventConsumer) Start() error {
 // realises that it cannot update the room state using the deltas.
 func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
+	receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
+
+	// Only handle events we care about
+	if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek {
+		return true
+	}
+
 	// Parse out the event JSON
 	var output api.OutputEvent
 	if err := json.Unmarshal(msg.Data, &output); err != nil {
diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go
index 987e6c942..9c4521986 100644
--- a/roomserver/producers/roomevent.go
+++ b/roomserver/producers/roomevent.go
@@ -17,12 +17,13 @@ package producers
 import (
 	"encoding/json"
 
-	"github.com/matrix-org/dendrite/roomserver/acls"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/nats-io/nats.go"
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/roomserver/acls"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 )
 
 var keyContentFields = map[string]string{
@@ -40,10 +41,8 @@ type RoomEventProducer struct {
 func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.OutputEvent) error {
 	var err error
 	for _, update := range updates {
-		msg := &nats.Msg{
-			Subject: r.Topic,
-			Header:  nats.Header{},
-		}
+		msg := nats.NewMsg(r.Topic)
+		msg.Header.Set(jetstream.RoomEventType, string(update.Type))
 		msg.Header.Set(jetstream.RoomID, roomID)
 		msg.Data, err = json.Marshal(update)
 		if err != nil {
diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go
index ee9810dae..590f0cbd9 100644
--- a/setup/jetstream/streams.go
+++ b/setup/jetstream/streams.go
@@ -9,9 +9,10 @@ import (
 )
 
 const (
-	UserID  = "user_id"
-	RoomID  = "room_id"
-	EventID = "event_id"
+	UserID        = "user_id"
+	RoomID        = "room_id"
+	EventID       = "event_id"
+	RoomEventType = "output_room_event_type"
 )
 
 var (
diff --git a/test/testrig/jetstream.go b/test/testrig/jetstream.go
index 74cf95062..b880eea43 100644
--- a/test/testrig/jetstream.go
+++ b/test/testrig/jetstream.go
@@ -4,10 +4,11 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/nats-io/nats.go"
+
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/nats-io/nats.go"
 )
 
 func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Msg) {
@@ -21,10 +22,8 @@ func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Ms
 
 func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg {
 	t.Helper()
-	msg := &nats.Msg{
-		Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
-		Header:  nats.Header{},
-	}
+	msg := nats.NewMsg(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent))
+	msg.Header.Set(jetstream.RoomEventType, string(update.Type))
 	msg.Header.Set(jetstream.RoomID, roomID)
 	var err error
 	msg.Data, err = json.Marshal(update)
diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go
index 952de98f7..a12876946 100644
--- a/userapi/consumers/roomserver.go
+++ b/userapi/consumers/roomserver.go
@@ -72,15 +72,16 @@ func (s *OutputRoomEventConsumer) Start() error {
 
 func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
+	// Only handle events we care about
+	if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent {
+		return true
+	}
 	var output rsapi.OutputEvent
 	if err := json.Unmarshal(msg.Data, &output); err != nil {
 		// If the message was invalid, log it and move on to the next message in the stream
 		log.WithError(err).Errorf("roomserver output log: message parse failure")
 		return true
 	}
-	if output.Type != rsapi.OutputTypeNewRoomEvent {
-		return true
-	}
 	event := output.NewRoomEvent.Event
 	if event == nil {
 		log.Errorf("userapi consumer: expected event")

From 6f602bb0969fac6d455de1088bf3980f77a4017f Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 11:16:05 +0100
Subject: [PATCH 46/73] Demote `Failed to query device keys for some users`
 warning to `level=debug`

Many of these warnings are due to dead servers and are quite annoying when they fill up the logs.
---
 keyserver/internal/device_list_update.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index fcfcd092d..8b02f3d6c 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -424,7 +424,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 			"succeeded": successCount,
 			"failed":    len(userIDs) - successCount,
 			"wait_time": waitTime,
-		}).Warn("Failed to query device keys for some users")
+		}).Debug("Failed to query device keys for some users")
 	}
 	return waitTime, !allUsersSucceeded
 }

From c85bc3434fc930423e9e27c8bca9b31b5ad7a441 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 12:47:53 +0100
Subject: [PATCH 47/73] Optimise `QuerySharedUsers` so that we can only work on
 local users (#2766)

Otherwise the sync API key change consumer wastes a lot of time trying
to wake up the notifiers for non-local users.
---
 roomserver/api/query.go                       |  1 +
 roomserver/internal/query/query.go            |  2 +-
 roomserver/storage/interface.go               |  2 +-
 .../storage/postgres/membership_table.go      | 17 +++++++++-----
 roomserver/storage/shared/storage.go          |  4 ++--
 .../storage/sqlite3/membership_table.go       | 23 +++++++++++--------
 roomserver/storage/tables/interface.go        |  2 +-
 .../storage/tables/membership_table_test.go   |  2 +-
 syncapi/consumers/keychange.go                |  6 +++--
 9 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index aa7dc4735..d63c24785 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -278,6 +278,7 @@ type QuerySharedUsersRequest struct {
 	OtherUserIDs   []string
 	ExcludeRoomIDs []string
 	IncludeRoomIDs []string
+	LocalOnly      bool
 }
 
 type QuerySharedUsersResponse struct {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index b41a92e94..ee8e1cfe7 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -799,7 +799,7 @@ func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUser
 	}
 	roomIDs = roomIDs[:j]
 
-	users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs)
+	users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs, req.LocalOnly)
 	if err != nil {
 		return err
 	}
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 43e8da7bb..11e175f55 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -157,7 +157,7 @@ type Database interface {
 	// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
 	GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
 	// JoinedUsersSetInRooms returns how many times each of the given users appears across the given rooms.
-	JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error)
+	JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error)
 	// GetLocalServerInRoom returns true if we think we're in a given room or false otherwise.
 	GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error)
 	// GetServerInRoom returns true if we think a server is in a given room or false otherwise.
diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go
index bd3fd5592..0150534e1 100644
--- a/roomserver/storage/postgres/membership_table.go
+++ b/roomserver/storage/postgres/membership_table.go
@@ -68,14 +68,18 @@ CREATE TABLE IF NOT EXISTS roomserver_membership (
 
 var selectJoinedUsersSetForRoomsAndUserSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid = ANY($1) AND target_nid = ANY($2) AND" +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false)" +
+	" AND room_nid = ANY($2) AND target_nid = ANY($3)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 var selectJoinedUsersSetForRoomsSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid = ANY($1) AND" +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false) " +
+	" AND room_nid = ANY($2)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 // Insert a row in to membership table so that it can be locked by the
@@ -334,6 +338,7 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(
 	ctx context.Context, txn *sql.Tx,
 	roomNIDs []types.RoomNID,
 	userNIDs []types.EventStateKeyNID,
+	localOnly bool,
 ) (map[types.EventStateKeyNID]int, error) {
 	var (
 		rows *sql.Rows
@@ -342,9 +347,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(
 	stmt := sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsStmt)
 	if len(userNIDs) > 0 {
 		stmt = sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsAndUserStmt)
-		rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(userNIDs))
+		rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs), pq.Array(userNIDs))
 	} else {
-		rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs))
+		rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs))
 	}
 
 	if err != nil {
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 593abbea1..d83a1ff74 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -1280,7 +1280,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu
 }
 
 // JoinedUsersSetInRooms returns a map of how many times the given users appear in the specified rooms.
-func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) {
+func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) {
 	roomNIDs, err := d.RoomsTable.BulkSelectRoomNIDs(ctx, nil, roomIDs)
 	if err != nil {
 		return nil, err
@@ -1295,7 +1295,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs [
 		userNIDs = append(userNIDs, nid)
 		nidToUserID[nid] = id
 	}
-	userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs)
+	userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs, localOnly)
 	if err != nil {
 		return nil, err
 	}
diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go
index f3303eb0e..cd149f0ed 100644
--- a/roomserver/storage/sqlite3/membership_table.go
+++ b/roomserver/storage/sqlite3/membership_table.go
@@ -44,14 +44,18 @@ const membershipSchema = `
 
 var selectJoinedUsersSetForRoomsAndUserSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid IN ($1) AND target_nid IN ($2) AND" +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false)" +
+	" AND room_nid IN ($2) AND target_nid IN ($3)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 var selectJoinedUsersSetForRoomsSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" +
-	" WHERE room_nid IN ($1) AND " +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" WHERE (target_local OR $1 = false)" +
+	" AND room_nid IN ($2)" +
+	" AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) +
+	" AND forgotten = false" +
 	" GROUP BY target_nid"
 
 // Insert a row in to membership table so that it can be locked by the
@@ -305,8 +309,9 @@ func (s *membershipStatements) SelectRoomsWithMembership(
 	return roomNIDs, nil
 }
 
-func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) {
-	params := make([]interface{}, 0, len(roomNIDs)+len(userNIDs))
+func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) {
+	params := make([]interface{}, 0, 1+len(roomNIDs)+len(userNIDs))
+	params = append(params, localOnly)
 	for _, v := range roomNIDs {
 		params = append(params, v)
 	}
@@ -314,10 +319,10 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context,
 		params = append(params, v)
 	}
 
-	query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
+	query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1)
 	if len(userNIDs) > 0 {
-		query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1)
-		query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)), 1)
+		query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1)
+		query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)+1), 1)
 	}
 	var rows *sql.Rows
 	var err error
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 68d30f994..d7bcc95ab 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -137,7 +137,7 @@ type Membership interface {
 	UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID, forgotten bool) (bool, error)
 	SelectRoomsWithMembership(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, membershipState MembershipState) ([]types.RoomNID, error)
 	// SelectJoinedUsersSetForRooms returns how many times each of the given users appears across the given rooms.
-	SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error)
+	SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error)
 	SelectKnownUsers(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error)
 	UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error
 	SelectLocalServerInRoom(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) (bool, error)
diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go
index f789ef4ac..c9541d9d2 100644
--- a/roomserver/storage/tables/membership_table_test.go
+++ b/roomserver/storage/tables/membership_table_test.go
@@ -79,7 +79,7 @@ func TestMembershipTable(t *testing.T) {
 		assert.NoError(t, err)
 		assert.True(t, inRoom)
 
-		userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs)
+		userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs, false)
 		assert.NoError(t, err)
 		assert.Equal(t, 1, len(userJoinedToRooms))
 
diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go
index dc7d9e207..96ebba7ef 100644
--- a/syncapi/consumers/keychange.go
+++ b/syncapi/consumers/keychange.go
@@ -111,7 +111,8 @@ func (s *OutputKeyChangeEventConsumer) onDeviceKeyMessage(m api.DeviceMessage, d
 	// work out who we need to notify about the new key
 	var queryRes roomserverAPI.QuerySharedUsersResponse
 	err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{
-		UserID: output.UserID,
+		UserID:    output.UserID,
+		LocalOnly: true,
 	}, &queryRes)
 	if err != nil {
 		logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server")
@@ -135,7 +136,8 @@ func (s *OutputKeyChangeEventConsumer) onCrossSigningMessage(m api.DeviceMessage
 	// work out who we need to notify about the new key
 	var queryRes roomserverAPI.QuerySharedUsersResponse
 	err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{
-		UserID: output.UserID,
+		UserID:    output.UserID,
+		LocalOnly: true,
 	}, &queryRes)
 	if err != nil {
 		logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server")

From 0f777d421c56e386b4c483233277f2b96c4da3a0 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Wed, 5 Oct 2022 14:47:13 +0200
Subject: [PATCH 48/73] Remove empty fields from `/sync` response (#2755)

First attempt at removing empty fields from `/sync` responses. Needs
https://github.com/matrix-org/sytest/pull/1298 to keep Sytest happy.

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
---
 syncapi/internal/keychange_test.go    |  33 +++--
 syncapi/streams/stream_accountdata.go |   6 +-
 syncapi/streams/stream_invite.go      |   4 +-
 syncapi/streams/stream_pdu.go         |  10 +-
 syncapi/streams/stream_receipt.go     |   5 +-
 syncapi/streams/stream_typing.go      |   9 +-
 syncapi/types/types.go                | 201 ++++++++++++++++++--------
 7 files changed, 178 insertions(+), 90 deletions(-)

diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go
index 3b9c8221c..53f3e5a40 100644
--- a/syncapi/internal/keychange_test.go
+++ b/syncapi/internal/keychange_test.go
@@ -170,9 +170,12 @@ func joinResponseWithRooms(syncResponse *types.Response, userID string, roomIDs
 				Content:  []byte(`{"membership":"join"}`),
 			},
 		}
-
-		jr := syncResponse.Rooms.Join[roomID]
-		jr.State.Events = roomEvents
+		jr, ok := syncResponse.Rooms.Join[roomID]
+		if !ok {
+			jr = types.NewJoinResponse()
+		}
+		jr.Timeline = &types.Timeline{}
+		jr.State = &types.ClientEvents{Events: roomEvents}
 		syncResponse.Rooms.Join[roomID] = jr
 	}
 	return syncResponse
@@ -191,8 +194,11 @@ func leaveResponseWithRooms(syncResponse *types.Response, userID string, roomIDs
 			},
 		}
 
-		lr := syncResponse.Rooms.Leave[roomID]
-		lr.Timeline.Events = roomEvents
+		lr, ok := syncResponse.Rooms.Leave[roomID]
+		if !ok {
+			lr = types.NewLeaveResponse()
+		}
+		lr.Timeline = &types.Timeline{Events: roomEvents}
 		syncResponse.Rooms.Leave[roomID] = lr
 	}
 	return syncResponse
@@ -328,9 +334,13 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) {
 		},
 	}
 
-	jr := syncResponse.Rooms.Join[roomID]
-	jr.State.Events = roomStateEvents
-	jr.Timeline.Events = roomTimelineEvents
+	jr, ok := syncResponse.Rooms.Join[roomID]
+	if !ok {
+		jr = types.NewJoinResponse()
+	}
+
+	jr.State = &types.ClientEvents{Events: roomStateEvents}
+	jr.Timeline = &types.Timeline{Events: roomTimelineEvents}
 	syncResponse.Rooms.Join[roomID] = jr
 
 	rsAPI := &mockRoomserverAPI{
@@ -442,8 +452,11 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) {
 		},
 	}
 
-	lr := syncResponse.Rooms.Leave[roomID]
-	lr.Timeline.Events = roomEvents
+	lr, ok := syncResponse.Rooms.Leave[roomID]
+	if !ok {
+		lr = types.NewLeaveResponse()
+	}
+	lr.Timeline = &types.Timeline{Events: roomEvents}
 	syncResponse.Rooms.Leave[roomID] = lr
 
 	rsAPI := &mockRoomserverAPI{
diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go
index 3f2f7d134..3593a6563 100644
--- a/syncapi/streams/stream_accountdata.go
+++ b/syncapi/streams/stream_accountdata.go
@@ -90,9 +90,9 @@ func (p *AccountDataStreamProvider) IncrementalSync(
 				}
 			} else {
 				if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok {
-					joinData := *types.NewJoinResponse()
-					if existing, ok := req.Response.Rooms.Join[roomID]; ok {
-						joinData = existing
+					joinData, ok := req.Response.Rooms.Join[roomID]
+					if !ok {
+						joinData = types.NewJoinResponse()
 					}
 					joinData.AccountData.Events = append(
 						joinData.AccountData.Events,
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 17b3b8434..7875ffa35 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -65,7 +65,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 			continue
 		}
 		ir := types.NewInviteResponse(inviteEvent)
-		req.Response.Rooms.Invite[roomID] = *ir
+		req.Response.Rooms.Invite[roomID] = ir
 	}
 
 	// When doing an initial sync, we don't want to add retired invites, as this
@@ -87,7 +87,7 @@ func (p *InviteStreamProvider) IncrementalSync(
 				Type:           "m.room.member",
 				Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
 			})
-			req.Response.Rooms.Leave[roomID] = *lr
+			req.Response.Rooms.Leave[roomID] = lr
 		}
 	}
 
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index d252265ff..613ac434f 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -106,7 +106,7 @@ func (p *PDUStreamProvider) CompleteSync(
 			}
 			continue
 		}
-		req.Response.Rooms.Join[roomID] = *jr
+		req.Response.Rooms.Join[roomID] = jr
 		req.Rooms[roomID] = gomatrixserverlib.Join
 	}
 
@@ -129,7 +129,7 @@ func (p *PDUStreamProvider) CompleteSync(
 				}
 				continue
 			}
-			req.Response.Rooms.Peek[peek.RoomID] = *jr
+			req.Response.Rooms.Peek[peek.RoomID] = jr
 		}
 	}
 
@@ -320,7 +320,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		// didn't "remove" events, return that the response is limited.
 		jr.Timeline.Limited = limited && len(events) == len(recentEvents)
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
-		res.Rooms.Join[delta.RoomID] = *jr
+		res.Rooms.Join[delta.RoomID] = jr
 
 	case gomatrixserverlib.Peek:
 		jr := types.NewJoinResponse()
@@ -329,7 +329,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		jr.Timeline.Limited = limited
 		jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
-		res.Rooms.Peek[delta.RoomID] = *jr
+		res.Rooms.Peek[delta.RoomID] = jr
 
 	case gomatrixserverlib.Leave:
 		fallthrough // transitions to leave are the same as ban
@@ -342,7 +342,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
 		// didn't "remove" events, return that the response is limited.
 		lr.Timeline.Limited = limited && len(events) == len(recentEvents)
 		lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync)
-		res.Rooms.Leave[delta.RoomID] = *lr
+		res.Rooms.Leave[delta.RoomID] = lr
 	}
 
 	return latestPosition, nil
diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 8818a5533..76927cc36 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -4,9 +4,10 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type ReceiptStreamProvider struct {
@@ -76,7 +77,7 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 			continue
 		}
 
-		jr := *types.NewJoinResponse()
+		jr := types.NewJoinResponse()
 		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
 			jr = existing
 		}
diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go
index a6f7c7a06..84c199b39 100644
--- a/syncapi/streams/stream_typing.go
+++ b/syncapi/streams/stream_typing.go
@@ -4,10 +4,11 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type TypingStreamProvider struct {
@@ -35,9 +36,9 @@ func (p *TypingStreamProvider) IncrementalSync(
 			continue
 		}
 
-		jr := *types.NewJoinResponse()
-		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
-			jr = existing
+		jr, ok := req.Response.Rooms.Join[roomID]
+		if !ok {
+			jr = types.NewJoinResponse()
 		}
 
 		if users, updated := p.EDUCache.GetTypingUsersIfUpdatedAfter(
diff --git a/syncapi/types/types.go b/syncapi/types/types.go
index 3b85db4a4..b6d340f93 100644
--- a/syncapi/types/types.go
+++ b/syncapi/types/types.go
@@ -327,29 +327,57 @@ type PrevEventRef struct {
 	PrevSender    string          `json:"prev_sender"`
 }
 
+type DeviceLists struct {
+	Changed []string `json:"changed,omitempty"`
+	Left    []string `json:"left,omitempty"`
+}
+
+type RoomsResponse struct {
+	Join   map[string]*JoinResponse   `json:"join,omitempty"`
+	Peek   map[string]*JoinResponse   `json:"peek,omitempty"`
+	Invite map[string]*InviteResponse `json:"invite,omitempty"`
+	Leave  map[string]*LeaveResponse  `json:"leave,omitempty"`
+}
+
+type ToDeviceResponse struct {
+	Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"`
+}
+
 // Response represents a /sync API response. See https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-client-r0-sync
 type Response struct {
-	NextBatch   StreamingToken `json:"next_batch"`
-	AccountData struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"account_data,omitempty"`
-	Presence struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
-	} `json:"presence,omitempty"`
-	Rooms struct {
-		Join   map[string]JoinResponse   `json:"join,omitempty"`
-		Peek   map[string]JoinResponse   `json:"peek,omitempty"`
-		Invite map[string]InviteResponse `json:"invite,omitempty"`
-		Leave  map[string]LeaveResponse  `json:"leave,omitempty"`
-	} `json:"rooms,omitempty"`
-	ToDevice struct {
-		Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"`
-	} `json:"to_device,omitempty"`
-	DeviceLists struct {
-		Changed []string `json:"changed,omitempty"`
-		Left    []string `json:"left,omitempty"`
-	} `json:"device_lists,omitempty"`
-	DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
+	NextBatch           StreamingToken    `json:"next_batch"`
+	AccountData         *ClientEvents     `json:"account_data,omitempty"`
+	Presence            *ClientEvents     `json:"presence,omitempty"`
+	Rooms               *RoomsResponse    `json:"rooms,omitempty"`
+	ToDevice            *ToDeviceResponse `json:"to_device,omitempty"`
+	DeviceLists         *DeviceLists      `json:"device_lists,omitempty"`
+	DeviceListsOTKCount map[string]int    `json:"device_one_time_keys_count,omitempty"`
+}
+
+func (r Response) MarshalJSON() ([]byte, error) {
+	type alias Response
+	a := alias(r)
+	if r.AccountData != nil && len(r.AccountData.Events) == 0 {
+		a.AccountData = nil
+	}
+	if r.Presence != nil && len(r.Presence.Events) == 0 {
+		a.Presence = nil
+	}
+	if r.DeviceLists != nil {
+		if len(r.DeviceLists.Left) == 0 && len(r.DeviceLists.Changed) == 0 {
+			a.DeviceLists = nil
+		}
+	}
+	if r.Rooms != nil {
+		if len(r.Rooms.Join) == 0 && len(r.Rooms.Peek) == 0 &&
+			len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 {
+			a.Rooms = nil
+		}
+	}
+	if r.ToDevice != nil && len(r.ToDevice.Events) == 0 {
+		a.ToDevice = nil
+	}
+	return json.Marshal(a)
 }
 
 func (r *Response) HasUpdates() bool {
@@ -370,18 +398,21 @@ func NewResponse() *Response {
 	res := Response{}
 	// Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section,
 	// so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors.
-	res.Rooms.Join = map[string]JoinResponse{}
-	res.Rooms.Peek = map[string]JoinResponse{}
-	res.Rooms.Invite = map[string]InviteResponse{}
-	res.Rooms.Leave = map[string]LeaveResponse{}
+	res.Rooms = &RoomsResponse{
+		Join:   map[string]*JoinResponse{},
+		Peek:   map[string]*JoinResponse{},
+		Invite: map[string]*InviteResponse{},
+		Leave:  map[string]*LeaveResponse{},
+	}
 
 	// Also pre-intialise empty slices or else we'll insert 'null' instead of '[]' for the value.
 	// TODO: We really shouldn't have to do all this to coerce encoding/json to Do The Right Thing. We should
 	//       really be using our own Marshal/Unmarshal implementations otherwise this may prove to be a CPU bottleneck.
 	//       This also applies to NewJoinResponse, NewInviteResponse and NewLeaveResponse.
-	res.AccountData.Events = []gomatrixserverlib.ClientEvent{}
-	res.Presence.Events = []gomatrixserverlib.ClientEvent{}
-	res.ToDevice.Events = []gomatrixserverlib.SendToDeviceEvent{}
+	res.AccountData = &ClientEvents{}
+	res.Presence = &ClientEvents{}
+	res.DeviceLists = &DeviceLists{}
+	res.ToDevice = &ToDeviceResponse{}
 	res.DeviceListsOTKCount = map[string]int{}
 
 	return &res
@@ -403,38 +434,73 @@ type UnreadNotifications struct {
 	NotificationCount int `json:"notification_count"`
 }
 
+type ClientEvents struct {
+	Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
+}
+
+type Timeline struct {
+	Events    []gomatrixserverlib.ClientEvent `json:"events"`
+	Limited   bool                            `json:"limited"`
+	PrevBatch *TopologyToken                  `json:"prev_batch,omitempty"`
+}
+
+type Summary struct {
+	Heroes             []string `json:"m.heroes,omitempty"`
+	JoinedMemberCount  *int     `json:"m.joined_member_count,omitempty"`
+	InvitedMemberCount *int     `json:"m.invited_member_count,omitempty"`
+}
+
 // JoinResponse represents a /sync response for a room which is under the 'join' or 'peek' key.
 type JoinResponse struct {
-	Summary struct {
-		Heroes             []string `json:"m.heroes,omitempty"`
-		JoinedMemberCount  *int     `json:"m.joined_member_count,omitempty"`
-		InvitedMemberCount *int     `json:"m.invited_member_count,omitempty"`
-	} `json:"summary"`
-	State struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"state"`
-	Timeline struct {
-		Events    []gomatrixserverlib.ClientEvent `json:"events"`
-		Limited   bool                            `json:"limited"`
-		PrevBatch *TopologyToken                  `json:"prev_batch,omitempty"`
-	} `json:"timeline"`
-	Ephemeral struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"ephemeral"`
-	AccountData struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"account_data"`
+	Summary              *Summary      `json:"summary,omitempty"`
+	State                *ClientEvents `json:"state,omitempty"`
+	Timeline             *Timeline     `json:"timeline,omitempty"`
+	Ephemeral            *ClientEvents `json:"ephemeral,omitempty"`
+	AccountData          *ClientEvents `json:"account_data,omitempty"`
 	*UnreadNotifications `json:"unread_notifications,omitempty"`
 }
 
+func (jr JoinResponse) MarshalJSON() ([]byte, error) {
+	type alias JoinResponse
+	a := alias(jr)
+	if jr.State != nil && len(jr.State.Events) == 0 {
+		a.State = nil
+	}
+	if jr.Ephemeral != nil && len(jr.Ephemeral.Events) == 0 {
+		a.Ephemeral = nil
+	}
+	if jr.AccountData != nil && len(jr.AccountData.Events) == 0 {
+		a.AccountData = nil
+	}
+	if jr.Timeline != nil && len(jr.Timeline.Events) == 0 {
+		a.Timeline = nil
+	}
+	if jr.Summary != nil {
+		var nilPtr int
+		joinedEmpty := jr.Summary.JoinedMemberCount == nil || jr.Summary.JoinedMemberCount == &nilPtr
+		invitedEmpty := jr.Summary.InvitedMemberCount == nil || jr.Summary.InvitedMemberCount == &nilPtr
+		if joinedEmpty && invitedEmpty && len(jr.Summary.Heroes) == 0 {
+			a.Summary = nil
+		}
+
+	}
+	if jr.UnreadNotifications != nil &&
+		jr.UnreadNotifications.NotificationCount == 0 && jr.UnreadNotifications.HighlightCount == 0 {
+		a.UnreadNotifications = nil
+	}
+	return json.Marshal(a)
+}
+
 // NewJoinResponse creates an empty response with initialised arrays.
 func NewJoinResponse() *JoinResponse {
-	res := JoinResponse{}
-	res.State.Events = []gomatrixserverlib.ClientEvent{}
-	res.Timeline.Events = []gomatrixserverlib.ClientEvent{}
-	res.Ephemeral.Events = []gomatrixserverlib.ClientEvent{}
-	res.AccountData.Events = []gomatrixserverlib.ClientEvent{}
-	return &res
+	return &JoinResponse{
+		Summary:             &Summary{},
+		State:               &ClientEvents{},
+		Timeline:            &Timeline{},
+		Ephemeral:           &ClientEvents{},
+		AccountData:         &ClientEvents{},
+		UnreadNotifications: &UnreadNotifications{},
+	}
 }
 
 // InviteResponse represents a /sync response for a room which is under the 'invite' key.
@@ -469,21 +535,28 @@ func NewInviteResponse(event *gomatrixserverlib.HeaderedEvent) *InviteResponse {
 
 // LeaveResponse represents a /sync response for a room which is under the 'leave' key.
 type LeaveResponse struct {
-	State struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
-	} `json:"state"`
-	Timeline struct {
-		Events    []gomatrixserverlib.ClientEvent `json:"events"`
-		Limited   bool                            `json:"limited"`
-		PrevBatch *TopologyToken                  `json:"prev_batch,omitempty"`
-	} `json:"timeline"`
+	State    *ClientEvents `json:"state,omitempty"`
+	Timeline *Timeline     `json:"timeline,omitempty"`
+}
+
+func (lr LeaveResponse) MarshalJSON() ([]byte, error) {
+	type alias LeaveResponse
+	a := alias(lr)
+	if lr.State != nil && len(lr.State.Events) == 0 {
+		a.State = nil
+	}
+	if lr.Timeline != nil && len(lr.Timeline.Events) == 0 {
+		a.Timeline = nil
+	}
+	return json.Marshal(a)
 }
 
 // NewLeaveResponse creates an empty response with initialised arrays.
 func NewLeaveResponse() *LeaveResponse {
-	res := LeaveResponse{}
-	res.State.Events = []gomatrixserverlib.ClientEvent{}
-	res.Timeline.Events = []gomatrixserverlib.ClientEvent{}
+	res := LeaveResponse{
+		State:    &ClientEvents{},
+		Timeline: &Timeline{},
+	}
 	return &res
 }
 

From 9ba3103f8839243ea8529bf1f563afded3e7b591 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 5 Oct 2022 14:54:19 +0100
Subject: [PATCH 49/73] Document database connection limits

---
 docs/installation/11_optimisation.md | 35 ++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/docs/installation/11_optimisation.md b/docs/installation/11_optimisation.md
index f2f67c947..686ec2eb9 100644
--- a/docs/installation/11_optimisation.md
+++ b/docs/installation/11_optimisation.md
@@ -11,6 +11,41 @@ permalink: /installation/start/optimisation
 Now that you have Dendrite running, the following tweaks will improve the reliability
 and performance of your installation.
 
+## PostgreSQL connection limit
+
+A PostgreSQL database engine is configured to allow only a certain number of connections.
+This is typically controlled by the `max_connections` and `superuser_reserved_connections`
+configuration items in `postgresql.conf`. Once these limits are violated, **PostgreSQL will
+immediately stop accepting new connections** until some of the existing connections are closed.
+This is a common source of misconfiguration and requires particular care.
+
+If your PostgreSQL `max_connections` is set to `100` and `superuser_reserved_connections` is
+set to `3` then you have an effective connection limit of 97 database connections. It is
+therefore important to ensure that Dendrite doesn't violate that limit, otherwise database
+queries will unexpectedly fail and this will cause problems both within Dendrite and for users.
+
+If you are also running other software that uses the same PostgreSQL database engine, then you
+must also take into account that some connections will be already used by your other software
+and therefore will not be available to Dendrite. Check the configuration of any other software
+using the same database engine for their configured connection limits and adjust your calculations
+accordingly.
+
+Dendrite has a `max_open_conns` configuration item in each `database` block to control how many
+connections it will open to the database.
+
+**If you are using the `global` database pool** then you only need to configure the
+`max_open_conns` setting once in the `global` section.
+
+**If you are defining a `database` config per component** then you will need to ensure that
+the **sum total** of all configured `max_open_conns` to a given database server do not exceed
+the connection limit. If you configure a total that adds up to more connections than are available
+then this will cause database queries to fail.
+
+You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate
+additional connections, in which case you should also update the `max_open_conns` in your
+Dendrite configuration accordingly. However be aware that this is only advisable on particularly
+powerful servers that can handle the concurrent load of additional queries running at one time.
+
 ## File descriptor limit
 
 Most platforms have a limit on how many file descriptors a single process can open. All

From e53dcb25a9a0258b48fd4f7afddd2c3c54d55b7a Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 6 Oct 2022 10:07:13 +0100
Subject: [PATCH 50/73] Tweak logging for federated room joins

---
 federationapi/internal/perform.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 84702f4ce..4cdd3a5eb 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -259,7 +259,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	if err != nil {
 		return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
 	}
-	logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts")
+	logrus.WithField("room", roomID).Infof("Joined federated room with %d hosts", len(joinedHosts))
 	if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil {
 		return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err)
 	}

From 3f82bceb70050c1233b7de6d87ffa5510596d145 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 6 Oct 2022 10:51:06 +0100
Subject: [PATCH 51/73] Don't try to talk to ourselves when finding missing
 events

---
 roomserver/internal/input/input_events.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index 01fd62010..d1b6bc73e 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -173,12 +173,15 @@ func (r *Inputer) processRoomEvent(
 		for _, server := range serverRes.ServerNames {
 			servers[server] = struct{}{}
 		}
+		// Don't try to talk to ourselves.
+		delete(servers, r.Cfg.Matrix.ServerName)
+		// Now build up the list of servers.
 		serverRes.ServerNames = serverRes.ServerNames[:0]
-		if input.Origin != "" {
+		if input.Origin != "" && input.Origin != r.Cfg.Matrix.ServerName {
 			serverRes.ServerNames = append(serverRes.ServerNames, input.Origin)
 			delete(servers, input.Origin)
 		}
-		if senderDomain != input.Origin {
+		if senderDomain != input.Origin && senderDomain != r.Cfg.Matrix.ServerName {
 			serverRes.ServerNames = append(serverRes.ServerNames, senderDomain)
 			delete(servers, senderDomain)
 		}

From ec5d1d681d1362f5746c5cb45e93829d6a68aa4d Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 6 Oct 2022 12:30:24 +0200
Subject: [PATCH 52/73] Always return `one_time_key_counts` on `/keys/upload`
 (#2769)

The OTK count is
[required](https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload)
in responses to `/keys/upload`, so return those.
---
 clientapi/routing/keys.go      | 4 ++--
 keyserver/internal/internal.go | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go
index b7a76b47e..5c3681382 100644
--- a/clientapi/routing/keys.go
+++ b/clientapi/routing/keys.go
@@ -19,11 +19,12 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/util"
 )
 
 type uploadKeysRequest struct {
@@ -77,7 +78,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi
 		}
 	}
 	keyCount := make(map[string]int)
-	// we only return key counts when the client uploads OTKs
 	if len(uploadRes.OneTimeKeyCounts) > 0 {
 		keyCount = uploadRes.OneTimeKeyCounts[0].KeyCount
 	}
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 017c29e84..a0280dff4 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -70,6 +70,11 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform
 	if len(req.OneTimeKeys) > 0 {
 		a.uploadOneTimeKeys(ctx, req, res)
 	}
+	otks, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID)
+	if err != nil {
+		return err
+	}
+	res.OneTimeKeyCounts = []api.OneTimeKeysCount{*otks}
 	return nil
 }
 

From d605d928bce87b381e2f64b8835619d803e67a54 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 6 Oct 2022 11:56:00 +0100
Subject: [PATCH 53/73] Allow specifying old signing keys with the public key
 and key ID only (#2770)

If the private key file is lost, it's often possible to retrieve the
public key from another server elsewhere, so we should make it possible
to configure it in that way.
---
 dendrite-sample.monolith.yaml |  7 ++++-
 dendrite-sample.polylith.yaml |  7 ++++-
 federationapi/routing/keys.go |  2 +-
 setup/config/config.go        | 48 +++++++++++++++++++++++------------
 setup/config/config_global.go |  7 +++--
 5 files changed, 50 insertions(+), 21 deletions(-)

diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml
index f0fa386d1..eadb74a2a 100644
--- a/dendrite-sample.monolith.yaml
+++ b/dendrite-sample.monolith.yaml
@@ -18,12 +18,17 @@ global:
   private_key: matrix_key.pem
 
   # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
-  # to old signing private keys that were formerly in use on this domain. These
+  # to old signing keys that were formerly in use on this domain name. These
   # keys will not be used for federation request or event signing, but will be
   # provided to any other homeserver that asks when trying to verify old events.
   old_private_keys:
+  #  If the old private key file is available:
   #  - private_key: old_matrix_key.pem
   #    expired_at: 1601024554498
+  #  If only the public key (in base64 format) and key ID are known:
+  #  - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
+  #    key_id: ed25519:mykeyid
+  #    expired_at: 1601024554498
 
   # How long a remote server can cache our server signing key before requesting it
   # again. Increasing this number will reduce the number of requests made by other
diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml
index 0ae4cc8fb..aa7e0cc38 100644
--- a/dendrite-sample.polylith.yaml
+++ b/dendrite-sample.polylith.yaml
@@ -18,12 +18,17 @@ global:
   private_key: matrix_key.pem
 
   # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
-  # to old signing private keys that were formerly in use on this domain. These
+  # to old signing keys that were formerly in use on this domain name. These
   # keys will not be used for federation request or event signing, but will be
   # provided to any other homeserver that asks when trying to verify old events.
   old_private_keys:
+  #  If the old private key file is available:
   #  - private_key: old_matrix_key.pem
   #    expired_at: 1601024554498
+  #  If only the public key (in base64 format) and key ID are known:
+  #  - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
+  #    key_id: ed25519:mykeyid
+  #    expired_at: 1601024554498
 
   # How long a remote server can cache our server signing key before requesting it
   # again. Increasing this number will reduce the number of requests made by other
diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go
index b03d4c1d6..8931830f3 100644
--- a/federationapi/routing/keys.go
+++ b/federationapi/routing/keys.go
@@ -160,7 +160,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver
 	for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys {
 		keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{
 			VerifyKey: gomatrixserverlib.VerifyKey{
-				Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)),
+				Key: oldVerifyKey.PublicKey,
 			},
 			ExpiredTS: oldVerifyKey.ExpiredAt,
 		}
diff --git a/setup/config/config.go b/setup/config/config.go
index 5a618d671..e99852ec9 100644
--- a/setup/config/config.go
+++ b/setup/config/config.go
@@ -231,24 +231,40 @@ func loadConfig(
 		return nil, err
 	}
 
-	for i, oldPrivateKey := range c.Global.OldVerifyKeys {
-		var oldPrivateKeyData []byte
+	for _, key := range c.Global.OldVerifyKeys {
+		switch {
+		case key.PrivateKeyPath != "":
+			var oldPrivateKeyData []byte
+			oldPrivateKeyPath := absPath(basePath, key.PrivateKeyPath)
+			oldPrivateKeyData, err = readFile(oldPrivateKeyPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read %q: %w", oldPrivateKeyPath, err)
+			}
 
-		oldPrivateKeyPath := absPath(basePath, oldPrivateKey.PrivateKeyPath)
-		oldPrivateKeyData, err = readFile(oldPrivateKeyPath)
-		if err != nil {
-			return nil, err
+			// NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are
+			// a number of private keys out there with non-compatible symbols in them due
+			// to lack of validation in Synapse, we won't enforce that for old verify keys.
+			keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false)
+			if perr != nil {
+				return nil, fmt.Errorf("failed to parse %q: %w", oldPrivateKeyPath, perr)
+			}
+
+			key.KeyID = keyID
+			key.PrivateKey = privateKey
+			key.PublicKey = gomatrixserverlib.Base64Bytes(privateKey.Public().(ed25519.PublicKey))
+
+		case key.KeyID == "":
+			return nil, fmt.Errorf("'key_id' must be specified if 'public_key' is specified")
+
+		case len(key.PublicKey) == ed25519.PublicKeySize:
+			continue
+
+		case len(key.PublicKey) > 0:
+			return nil, fmt.Errorf("the supplied 'public_key' is the wrong length")
+
+		default:
+			return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'")
 		}
-
-		// NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are
-		// a number of private keys out there with non-compatible symbols in them due
-		// to lack of validation in Synapse, we won't enforce that for old verify keys.
-		keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false)
-		if perr != nil {
-			return nil, perr
-		}
-
-		c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey
 	}
 
 	c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath))
diff --git a/setup/config/config_global.go b/setup/config/config_global.go
index acc608dd7..2efae0d5a 100644
--- a/setup/config/config_global.go
+++ b/setup/config/config_global.go
@@ -27,7 +27,7 @@ type Global struct {
 	// Information about old private keys that used to be used to sign requests and
 	// events on this domain. They will not be used but will be advertised to other
 	// servers that ask for them to help verify old events.
-	OldVerifyKeys []OldVerifyKeys `yaml:"old_private_keys"`
+	OldVerifyKeys []*OldVerifyKeys `yaml:"old_private_keys"`
 
 	// How long a remote server can cache our server key for before requesting it again.
 	// Increasing this number will reduce the number of requests made by remote servers
@@ -127,8 +127,11 @@ type OldVerifyKeys struct {
 	// The private key itself.
 	PrivateKey ed25519.PrivateKey `yaml:"-"`
 
+	// The public key, in case only that part is known.
+	PublicKey gomatrixserverlib.Base64Bytes `yaml:"public_key"`
+
 	// The key ID of the private key.
-	KeyID gomatrixserverlib.KeyID `yaml:"-"`
+	KeyID gomatrixserverlib.KeyID `yaml:"key_id"`
 
 	// When the private key was designed as "expired", as a UNIX timestamp
 	// in millisecond precision.

From 8c5b16678475112f5a1c13d616931283421aee05 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Thu, 6 Oct 2022 12:57:13 +0200
Subject: [PATCH 54/73] Use the stream positions of the notifier (#2768)

Use the stream positions of the notifier, which might have advanced
since setting it at the beginning of the loop. This possibly helps in
reducing roundtrips to the SyncAPI, just because we didn't fetch the
latest data.
Also fixes a minor oversight in the receipts stream.
---
 syncapi/streams/stream_receipt.go |  6 +++---
 syncapi/sync/requestpool.go       | 18 +++++++++---------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go
index 76927cc36..bba911022 100644
--- a/syncapi/streams/stream_receipt.go
+++ b/syncapi/streams/stream_receipt.go
@@ -77,9 +77,9 @@ func (p *ReceiptStreamProvider) IncrementalSync(
 			continue
 		}
 
-		jr := types.NewJoinResponse()
-		if existing, ok := req.Response.Rooms.Join[roomID]; ok {
-			jr = existing
+		jr, ok := req.Response.Rooms.Join[roomID]
+		if !ok {
+			jr = types.NewJoinResponse()
 		}
 
 		ev := gomatrixserverlib.ClientEvent{
diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go
index a71d32ab8..29d92b293 100644
--- a/syncapi/sync/requestpool.go
+++ b/syncapi/sync/requestpool.go
@@ -407,7 +407,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.PDUStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.PDUPosition, currentPos.PDUPosition,
+							syncReq.Since.PDUPosition, rp.Notifier.CurrentPosition().PDUPosition,
 						)
 					},
 				),
@@ -416,7 +416,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.TypingStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.TypingPosition, currentPos.TypingPosition,
+							syncReq.Since.TypingPosition, rp.Notifier.CurrentPosition().TypingPosition,
 						)
 					},
 				),
@@ -425,7 +425,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.ReceiptStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition,
+							syncReq.Since.ReceiptPosition, rp.Notifier.CurrentPosition().ReceiptPosition,
 						)
 					},
 				),
@@ -434,7 +434,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.InviteStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.InvitePosition, currentPos.InvitePosition,
+							syncReq.Since.InvitePosition, rp.Notifier.CurrentPosition().InvitePosition,
 						)
 					},
 				),
@@ -443,7 +443,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.SendToDeviceStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition,
+							syncReq.Since.SendToDevicePosition, rp.Notifier.CurrentPosition().SendToDevicePosition,
 						)
 					},
 				),
@@ -452,7 +452,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.AccountDataStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition,
+							syncReq.Since.AccountDataPosition, rp.Notifier.CurrentPosition().AccountDataPosition,
 						)
 					},
 				),
@@ -461,7 +461,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.NotificationDataStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition,
+							syncReq.Since.NotificationDataPosition, rp.Notifier.CurrentPosition().NotificationDataPosition,
 						)
 					},
 				),
@@ -470,7 +470,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.DeviceListStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition,
+							syncReq.Since.DeviceListPosition, rp.Notifier.CurrentPosition().DeviceListPosition,
 						)
 					},
 				),
@@ -479,7 +479,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.PresenceStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.PresencePosition, currentPos.PresencePosition,
+							syncReq.Since.PresencePosition, rp.Notifier.CurrentPosition().PresencePosition,
 						)
 					},
 				),

From 453b50e1d3ce4b4906e050fa8ebe7bcc4c881600 Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 7 Oct 2022 07:32:27 +0200
Subject: [PATCH 55/73] Update README

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3bb9a2350..dfef11bae 100644
--- a/README.md
+++ b/README.md
@@ -79,7 +79,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
 
 # Create an user account (add -admin for an admin user).
 # Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
-$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice
+$ ./bin/create-account --config dendrite.yaml --username alice
 ```
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
@@ -90,7 +90,7 @@ We use a script called Are We Synapse Yet which checks Sytest compliance rates.
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
 updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
-servers such as matrix.org reasonably well, although there are still some missing features (like Search).
+servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs).
 
 We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
 than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API).
@@ -112,6 +112,7 @@ This means Dendrite supports amongst others:
 - Guests
 - User Directory
 - Presence
+- Fulltext search
 
 ## Contributing
 

From b9d0e9f7ed7ce1f4be72a25c6f5185a6e809f019 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 7 Oct 2022 10:54:42 +0200
Subject: [PATCH 56/73] Add test for `QueryDeviceMessages` (#2773)

Adds tests for `QueryDeviceMessages` and also includes some
optimizations to reduce allocations in the DB layer.
---
 keyserver/internal/internal.go                |   8 +-
 keyserver/internal/internal_test.go           | 156 ++++++++++++++++++
 .../storage/postgres/device_keys_table.go     |  16 +-
 .../storage/sqlite3/device_keys_table.go      |  16 +-
 4 files changed, 172 insertions(+), 24 deletions(-)
 create mode 100644 keyserver/internal/internal_test.go

diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index a0280dff4..06fc4987c 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -212,15 +212,13 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query
 		return nil
 	}
 	maxStreamID := int64(0)
+	// remove deleted devices
+	var result []api.DeviceMessage
 	for _, m := range msgs {
 		if m.StreamID > maxStreamID {
 			maxStreamID = m.StreamID
 		}
-	}
-	// remove deleted devices
-	var result []api.DeviceMessage
-	for _, m := range msgs {
-		if m.KeyJSON == nil {
+		if m.KeyJSON == nil || len(m.KeyJSON) == 0 {
 			continue
 		}
 		result = append(result, m)
diff --git a/keyserver/internal/internal_test.go b/keyserver/internal/internal_test.go
new file mode 100644
index 000000000..8a2c9c5d9
--- /dev/null
+++ b/keyserver/internal/internal_test.go
@@ -0,0 +1,156 @@
+package internal_test
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/keyserver/internal"
+	"github.com/matrix-org/dendrite/keyserver/storage"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+)
+
+func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := storage.NewDatabase(nil, &config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	})
+	if err != nil {
+		t.Fatalf("failed to create new user db: %v", err)
+	}
+	return db, close
+}
+
+func Test_QueryDeviceMessages(t *testing.T) {
+	alice := test.NewUser(t)
+	type args struct {
+		req *api.QueryDeviceMessagesRequest
+		res *api.QueryDeviceMessagesResponse
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+		want    *api.QueryDeviceMessagesResponse
+	}{
+		{
+			name: "no existing keys",
+			args: args{
+				req: &api.QueryDeviceMessagesRequest{
+					UserID: "@doesNotExist:localhost",
+				},
+				res: &api.QueryDeviceMessagesResponse{},
+			},
+			want: &api.QueryDeviceMessagesResponse{},
+		},
+		{
+			name: "existing user returns devices",
+			args: args{
+				req: &api.QueryDeviceMessagesRequest{
+					UserID: alice.ID,
+				},
+				res: &api.QueryDeviceMessagesResponse{},
+			},
+			want: &api.QueryDeviceMessagesResponse{
+				StreamID: 6,
+				Devices: []api.DeviceMessage{
+					{
+						Type: api.TypeDeviceKeyUpdate, StreamID: 5, DeviceKeys: &api.DeviceKeys{
+							DeviceID:    "myDevice",
+							DisplayName: "first device",
+							UserID:      alice.ID,
+							KeyJSON:     []byte("ghi"),
+						},
+					},
+					{
+						Type: api.TypeDeviceKeyUpdate, StreamID: 6, DeviceKeys: &api.DeviceKeys{
+							DeviceID:    "mySecondDevice",
+							DisplayName: "second device",
+							UserID:      alice.ID,
+							KeyJSON:     []byte("jkl"),
+						}, // streamID 6
+					},
+				},
+			},
+		},
+	}
+
+	deviceMessages := []api.DeviceMessage{
+		{ // not the user we're looking for
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				UserID: "@doesNotExist:localhost",
+			},
+			// streamID 1 for this user
+		},
+		{ // empty keyJSON will be ignored
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+			}, // streamID 1
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+				KeyJSON:  []byte("abc"),
+			}, // streamID 2
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+				KeyJSON:  []byte("def"),
+			}, // streamID 3
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID: "myDevice",
+				UserID:   alice.ID,
+				KeyJSON:  []byte(""),
+			}, // streamID 4
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID:    "myDevice",
+				DisplayName: "first device",
+				UserID:      alice.ID,
+				KeyJSON:     []byte("ghi"),
+			}, // streamID 5
+		},
+		{
+			Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{
+				DeviceID:    "mySecondDevice",
+				UserID:      alice.ID,
+				KeyJSON:     []byte("jkl"),
+				DisplayName: "second device",
+			}, // streamID 6
+		},
+	}
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, closeDB := mustCreateDatabase(t, dbType)
+		defer closeDB()
+		if err := db.StoreLocalDeviceKeys(ctx, deviceMessages); err != nil {
+			t.Fatalf("failed to store local devicesKeys")
+		}
+
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				a := &internal.KeyInternalAPI{
+					DB: db,
+				}
+				if err := a.QueryDeviceMessages(ctx, tt.args.req, tt.args.res); (err != nil) != tt.wantErr {
+					t.Errorf("QueryDeviceMessages() error = %v, wantErr %v", err, tt.wantErr)
+				}
+				got := tt.args.res
+				if !reflect.DeepEqual(got, tt.want) {
+					t.Errorf("QueryDeviceMessages(): got:\n%+v, want:\n%+v", got, tt.want)
+				}
+			})
+		}
+	})
+}
diff --git a/keyserver/storage/postgres/device_keys_table.go b/keyserver/storage/postgres/device_keys_table.go
index ccd20cbd6..2aa11c520 100644
--- a/keyserver/storage/postgres/device_keys_table.go
+++ b/keyserver/storage/postgres/device_keys_table.go
@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/lib/pq"
+
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/api"
@@ -204,20 +205,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID
 		deviceIDMap[d] = true
 	}
 	var result []api.DeviceMessage
+	var displayName sql.NullString
 	for rows.Next() {
 		dk := api.DeviceMessage{
-			Type:       api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{},
+			Type: api.TypeDeviceKeyUpdate,
+			DeviceKeys: &api.DeviceKeys{
+				UserID: userID,
+			},
 		}
-		dk.UserID = userID
-		var keyJSON string
-		var streamID int64
-		var displayName sql.NullString
-		if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil {
+		if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil {
 			return nil, err
 		}
-		dk.KeyJSON = []byte(keyJSON)
-		dk.StreamID = streamID
 		if displayName.Valid {
 			dk.DisplayName = displayName.String
 		}
diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/keyserver/storage/sqlite3/device_keys_table.go
index e77b49b35..73768da5b 100644
--- a/keyserver/storage/sqlite3/device_keys_table.go
+++ b/keyserver/storage/sqlite3/device_keys_table.go
@@ -137,21 +137,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectBatchDeviceKeysStmt: rows.close() failed")
 	var result []api.DeviceMessage
+	var displayName sql.NullString
 	for rows.Next() {
 		dk := api.DeviceMessage{
-			Type:       api.TypeDeviceKeyUpdate,
-			DeviceKeys: &api.DeviceKeys{},
+			Type: api.TypeDeviceKeyUpdate,
+			DeviceKeys: &api.DeviceKeys{
+				UserID: userID,
+			},
 		}
-		dk.Type = api.TypeDeviceKeyUpdate
-		dk.UserID = userID
-		var keyJSON string
-		var streamID int64
-		var displayName sql.NullString
-		if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil {
+		if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil {
 			return nil, err
 		}
-		dk.KeyJSON = []byte(keyJSON)
-		dk.StreamID = streamID
 		if displayName.Valid {
 			dk.DisplayName = displayName.String
 		}

From 8d8f4689a0c71f79051554e69fb9113592a4b2e4 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 12:21:38 +0100
Subject: [PATCH 57/73] tDatabase transaction tweaks in roomserver

---
 roomserver/storage/shared/storage.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index d83a1ff74..8859571b7 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -598,7 +598,7 @@ func (d *Database) storeEvent(
 	}
 	// First writer is with a database-provided transaction, so that NIDs are assigned
 	// globally outside of the updater context, to help avoid races.
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+	err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error {
 		// TODO: Here we should aim to have two different code paths for new rooms
 		// vs existing ones.
 
@@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair(
 	}
 
 	if isRedactionEvent {
-		redactedEvent = d.loadEvent(ctx, info.RedactsEventID)
+		redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID)
 	} else {
-		redactionEvent = d.loadEvent(ctx, info.RedactionEventID)
+		redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID)
 	}
 
 	return redactionEvent, redactedEvent, info.Validated, nil
@@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) {
 }
 
 // loadEvent loads a single event or returns nil on any problems/missing event
-func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event {
-	nids, err := d.EventNIDs(ctx, []string{eventID})
+func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event {
+	nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter)
 	if err != nil {
 		return nil
 	}
 	if len(nids) == 0 {
 		return nil
 	}
-	evs, err := d.Events(ctx, []types.EventNID{nids[eventID]})
+	evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]})
 	if err != nil {
 		return nil
 	}
@@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget
 	}
 
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget)
+		return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget)
 	})
 }
 

From 1b5460a920d271aa03c798709f2f1b3b710e4e0e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 13:42:35 +0100
Subject: [PATCH 58/73] Ensure we only wake up a given user once (#2775)

This ensures that the sync API notifier only wakes up a given user once
for a given stream position.
---
 syncapi/notifier/notifier.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go
index db18c6b77..27f7c37ba 100644
--- a/syncapi/notifier/notifier.go
+++ b/syncapi/notifier/notifier.go
@@ -48,6 +48,7 @@ type Notifier struct {
 	lastCleanUpTime time.Time
 	// This map is reused to prevent allocations and GC pressure in SharedUsers.
 	_sharedUserMap map[string]struct{}
+	_wakeupUserMap map[string]struct{}
 }
 
 // NewNotifier creates a new notifier set to the given sync position.
@@ -61,6 +62,7 @@ func NewNotifier() *Notifier {
 		lock:                   &sync.RWMutex{},
 		lastCleanUpTime:        time.Now(),
 		_sharedUserMap:         map[string]struct{}{},
+		_wakeupUserMap:         map[string]struct{}{},
 	}
 }
 
@@ -408,12 +410,16 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P
 // specified user IDs, and also the specified peekingDevices
 func (n *Notifier) _wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) {
 	for _, userID := range userIDs {
+		n._wakeupUserMap[userID] = struct{}{}
+	}
+	for userID := range n._wakeupUserMap {
 		for _, stream := range n._fetchUserStreams(userID) {
 			if stream == nil {
 				continue
 			}
 			stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream
 		}
+		delete(n._wakeupUserMap, userID)
 	}
 
 	for _, peekingDevice := range peekingDevices {

From 8e231130e97ce716357bbb4af8f82159dc6e684e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 14:05:06 +0100
Subject: [PATCH 59/73] Revert "tDatabase transaction tweaks in roomserver"

This reverts commit 8d8f4689a0c71f79051554e69fb9113592a4b2e4.
---
 roomserver/storage/shared/storage.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index 8859571b7..d83a1ff74 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -598,7 +598,7 @@ func (d *Database) storeEvent(
 	}
 	// First writer is with a database-provided transaction, so that NIDs are assigned
 	// globally outside of the updater context, to help avoid races.
-	err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error {
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		// TODO: Here we should aim to have two different code paths for new rooms
 		// vs existing ones.
 
@@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair(
 	}
 
 	if isRedactionEvent {
-		redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID)
+		redactedEvent = d.loadEvent(ctx, info.RedactsEventID)
 	} else {
-		redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID)
+		redactionEvent = d.loadEvent(ctx, info.RedactionEventID)
 	}
 
 	return redactionEvent, redactedEvent, info.Validated, nil
@@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) {
 }
 
 // loadEvent loads a single event or returns nil on any problems/missing event
-func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event {
-	nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter)
+func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event {
+	nids, err := d.EventNIDs(ctx, []string{eventID})
 	if err != nil {
 		return nil
 	}
 	if len(nids) == 0 {
 		return nil
 	}
-	evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]})
+	evs, err := d.Events(ctx, []types.EventNID{nids[eventID]})
 	if err != nil {
 		return nil
 	}
@@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget
 	}
 
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
-		return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget)
+		return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget)
 	})
 }
 

From 1ca3f3efb512db4c1827eb571588ec31258782a5 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Fri, 7 Oct 2022 16:00:12 +0200
Subject: [PATCH 60/73] Fix issue with DMs shown as normal rooms (#2776)

Fixes #2121, test added in
https://github.com/matrix-org/complement/pull/494
---
 federationapi/api/api.go                      |  1 +
 federationapi/internal/perform.go             | 19 +++++++++---
 roomserver/api/perform.go                     |  1 +
 roomserver/internal/helpers/helpers.go        | 31 ++++++++++---------
 roomserver/internal/perform/perform_join.go   | 20 ++++++++++--
 roomserver/internal/perform/perform_leave.go  |  2 +-
 roomserver/internal/query/query.go            |  2 +-
 roomserver/storage/interface.go               |  5 +--
 roomserver/storage/postgres/invite_table.go   | 13 ++++----
 roomserver/storage/shared/storage.go          |  9 +++---
 roomserver/storage/sqlite3/invite_table.go    | 13 ++++----
 roomserver/storage/tables/interface.go        |  2 +-
 .../storage/tables/invite_table_test.go       | 11 ++++---
 13 files changed, 82 insertions(+), 47 deletions(-)

diff --git a/federationapi/api/api.go b/federationapi/api/api.go
index f25538784..362333fc9 100644
--- a/federationapi/api/api.go
+++ b/federationapi/api/api.go
@@ -159,6 +159,7 @@ type PerformJoinRequest struct {
 	// The sorted list of servers to try. Servers will be tried sequentially, after de-duplication.
 	ServerNames types.ServerNames      `json:"server_names"`
 	Content     map[string]interface{} `json:"content"`
+	Unsigned    map[string]interface{} `json:"unsigned"`
 }
 
 type PerformJoinResponse struct {
diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go
index 4cdd3a5eb..28ec48d7b 100644
--- a/federationapi/internal/perform.go
+++ b/federationapi/internal/perform.go
@@ -7,14 +7,15 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/federationapi/consumers"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/consumers"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/version"
 )
 
 // PerformLeaveRequest implements api.FederationInternalAPI
@@ -95,6 +96,7 @@ func (r *FederationInternalAPI) PerformJoin(
 			request.Content,
 			serverName,
 			supportedVersions,
+			request.Unsigned,
 		); err != nil {
 			logrus.WithError(err).WithFields(logrus.Fields{
 				"server_name": serverName,
@@ -139,6 +141,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	content map[string]interface{},
 	serverName gomatrixserverlib.ServerName,
 	supportedVersions []gomatrixserverlib.RoomVersion,
+	unsigned map[string]interface{},
 ) error {
 	// Try to perform a make_join using the information supplied in the
 	// request.
@@ -267,6 +270,14 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// If we successfully performed a send_join above then the other
 	// server now thinks we're a part of the room. Send the newly
 	// returned state to the roomserver to update our local view.
+	if unsigned != nil {
+		event, err = event.SetUnsigned(unsigned)
+		if err != nil {
+			// non-fatal, log and continue
+			logrus.WithError(err).Errorf("Failed to set unsigned content")
+		}
+	}
+
 	if err = roomserverAPI.SendEventWithState(
 		context.Background(),
 		r.rsAPI,
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 20931f807..7a362f969 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -80,6 +80,7 @@ type PerformJoinRequest struct {
 	UserID        string                         `json:"user_id"`
 	Content       map[string]interface{}         `json:"content"`
 	ServerNames   []gomatrixserverlib.ServerName `json:"server_names"`
+	Unsigned      map[string]interface{}         `json:"unsigned"`
 }
 
 type PerformJoinResponse struct {
diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go
index cbd1561f7..3b83a0a6d 100644
--- a/roomserver/internal/helpers/helpers.go
+++ b/roomserver/internal/helpers/helpers.go
@@ -7,6 +7,9 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/auth"
 	"github.com/matrix-org/dendrite/roomserver/state"
@@ -14,8 +17,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 // TODO: temporary package which has helper functions used by both internal/perform packages.
@@ -97,35 +98,35 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam
 func IsInvitePending(
 	ctx context.Context, db storage.Database,
 	roomID, userID string,
-) (bool, string, string, error) {
+) (bool, string, string, *gomatrixserverlib.Event, error) {
 	// Look up the room NID for the supplied room ID.
 	info, err := db.RoomInfo(ctx, roomID)
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.RoomInfo: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.RoomInfo: %w", err)
 	}
 	if info == nil {
-		return false, "", "", fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID)
+		return false, "", "", nil, fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID)
 	}
 
 	// Look up the state key NID for the supplied user ID.
 	targetUserNIDs, err := db.EventStateKeyNIDs(ctx, []string{userID})
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err)
 	}
 	targetUserNID, targetUserFound := targetUserNIDs[userID]
 	if !targetUserFound {
-		return false, "", "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs)
+		return false, "", "", nil, fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs)
 	}
 
 	// Let's see if we have an event active for the user in the room. If
 	// we do then it will contain a server name that we can direct the
 	// send_leave to.
-	senderUserNIDs, eventIDs, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID)
+	senderUserNIDs, eventIDs, eventJSON, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID)
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.GetInvitesForUser: %w", err)
 	}
 	if len(senderUserNIDs) == 0 {
-		return false, "", "", nil
+		return false, "", "", nil, nil
 	}
 	userNIDToEventID := make(map[types.EventStateKeyNID]string)
 	for i, nid := range senderUserNIDs {
@@ -135,18 +136,20 @@ func IsInvitePending(
 	// Look up the user ID from the NID.
 	senderUsers, err := db.EventStateKeys(ctx, senderUserNIDs)
 	if err != nil {
-		return false, "", "", fmt.Errorf("r.DB.EventStateKeys: %w", err)
+		return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeys: %w", err)
 	}
 	if len(senderUsers) == 0 {
-		return false, "", "", fmt.Errorf("no senderUsers")
+		return false, "", "", nil, fmt.Errorf("no senderUsers")
 	}
 
 	senderUser, senderUserFound := senderUsers[senderUserNIDs[0]]
 	if !senderUserFound {
-		return false, "", "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers)
+		return false, "", "", nil, fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers)
 	}
 
-	return true, senderUser, userNIDToEventID[senderUserNIDs[0]], nil
+	event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false, info.RoomVersion)
+
+	return true, senderUser, userNIDToEventID[senderUserNIDs[0]], event, err
 }
 
 // GetMembershipsAtState filters the state events to
diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go
index 167b375b7..262273ff5 100644
--- a/roomserver/internal/perform/perform_join.go
+++ b/roomserver/internal/perform/perform_join.go
@@ -22,6 +22,10 @@ import (
 	"time"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
+
 	fsAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -32,8 +36,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 )
 
 type Joiner struct {
@@ -236,7 +238,7 @@ func (r *Joiner) performJoinRoomByID(
 
 	// Force a federated join if we're dealing with a pending invite
 	// and we aren't in the room.
-	isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID)
+	isInvitePending, inviteSender, _, inviteEvent, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID)
 	if err == nil && !serverInRoom && isInvitePending {
 		_, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender)
 		if ierr != nil {
@@ -248,6 +250,17 @@ func (r *Joiner) performJoinRoomByID(
 		if inviterDomain != r.Cfg.Matrix.ServerName {
 			req.ServerNames = append(req.ServerNames, inviterDomain)
 			forceFederatedJoin = true
+			memberEvent := gjson.Parse(string(inviteEvent.JSON()))
+			// only set unsigned if we've got a content.membership, which we _should_
+			if memberEvent.Get("content.membership").Exists() {
+				req.Unsigned = map[string]interface{}{
+					"prev_sender": memberEvent.Get("sender").Str,
+					"prev_content": map[string]interface{}{
+						"is_direct":  memberEvent.Get("content.is_direct").Bool(),
+						"membership": memberEvent.Get("content.membership").Str,
+					},
+				}
+			}
 		}
 	}
 
@@ -348,6 +361,7 @@ func (r *Joiner) performFederatedJoinRoomByID(
 		UserID:      req.UserID,        // the user ID joining the room
 		ServerNames: req.ServerNames,   // the server to try joining with
 		Content:     req.Content,       // the membership event content
+		Unsigned:    req.Unsigned,      // the unsigned event content, if any
 	}
 	fedRes := fsAPI.PerformJoinResponse{}
 	r.FSAPI.PerformJoin(ctx, &fedReq, &fedRes)
diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index ada3aab06..85b659814 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -79,7 +79,7 @@ func (r *Leaver) performLeaveRoomByID(
 ) ([]api.OutputEvent, error) {
 	// If there's an invite outstanding for the room then respond to
 	// that.
-	isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
+	isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
 		_, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser)
 		if serr != nil {
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index ee8e1cfe7..7a424a334 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -872,7 +872,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
 	// but we don't specify an authorised via user, since the event auth
 	// will allow the join anyway.
 	var pending bool
-	if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
+	if pending, _, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil {
 		return fmt.Errorf("helpers.IsInvitePending: %w", err)
 	} else if pending {
 		res.Allowed = true
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 11e175f55..ee0624b21 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -17,10 +17,11 @@ package storage
 import (
 	"context"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
@@ -104,7 +105,7 @@ type Database interface {
 	// Look up the active invites targeting a user in a room and return the
 	// numeric state key IDs for the user IDs who sent them along with the event IDs for the invites.
 	// Returns an error if there was a problem talking to the database.
-	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error)
+	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error)
 	// Save a given room alias with the room ID it refers to.
 	// Returns an error if there was a problem talking to the database.
 	SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error
diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go
index 4cddfe2e9..009fd1ac1 100644
--- a/roomserver/storage/postgres/invite_table.go
+++ b/roomserver/storage/postgres/invite_table.go
@@ -61,7 +61,7 @@ const insertInviteEventSQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectInviteActiveForUserInRoomSQL = "" +
-	"SELECT invite_event_id, sender_nid FROM roomserver_invites" +
+	"SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" +
 	" WHERE target_nid = $1 AND room_nid = $2" +
 	" AND NOT retired"
 
@@ -141,25 +141,26 @@ func (s *inviteStatements) UpdateInviteRetired(
 func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	ctx context.Context, txn *sql.Tx,
 	targetUserNID types.EventStateKeyNID, roomNID types.RoomNID,
-) ([]types.EventStateKeyNID, []string, error) {
+) ([]types.EventStateKeyNID, []string, []byte, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt)
 	rows, err := stmt.QueryContext(
 		ctx, targetUserNID, roomNID,
 	)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
 	var eventIDs []string
 	var inviteEventID string
 	var senderUserNID int64
+	var eventJSON []byte
 	for rows.Next() {
-		if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil {
-			return nil, nil, err
+		if err := rows.Scan(&inviteEventID, &senderUserNID, &eventJSON); err != nil {
+			return nil, nil, nil, err
 		}
 		result = append(result, types.EventStateKeyNID(senderUserNID))
 		eventIDs = append(eventIDs, inviteEventID)
 	}
-	return result, eventIDs, rows.Err()
+	return result, eventIDs, eventJSON, rows.Err()
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index d83a1ff74..e401f17dc 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -7,13 +7,14 @@ import (
 	"fmt"
 	"sort"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/tidwall/gjson"
 )
 
 // Ideally, when we have both events we should redact the event JSON and forget about the redaction, but we currently
@@ -445,7 +446,7 @@ func (d *Database) GetInvitesForUser(
 	ctx context.Context,
 	roomNID types.RoomNID,
 	targetUserNID types.EventStateKeyNID,
-) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) {
+) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error) {
 	return d.InvitesTable.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
 }
 
diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go
index e051d63af..ca6e7c511 100644
--- a/roomserver/storage/sqlite3/invite_table.go
+++ b/roomserver/storage/sqlite3/invite_table.go
@@ -44,7 +44,7 @@ const insertInviteEventSQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectInviteActiveForUserInRoomSQL = "" +
-	"SELECT invite_event_id, sender_nid FROM roomserver_invites" +
+	"SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" +
 	" WHERE target_nid = $1 AND room_nid = $2" +
 	" AND NOT retired"
 
@@ -136,25 +136,26 @@ func (s *inviteStatements) UpdateInviteRetired(
 func (s *inviteStatements) SelectInviteActiveForUserInRoom(
 	ctx context.Context, txn *sql.Tx,
 	targetUserNID types.EventStateKeyNID, roomNID types.RoomNID,
-) ([]types.EventStateKeyNID, []string, error) {
+) ([]types.EventStateKeyNID, []string, []byte, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt)
 	rows, err := stmt.QueryContext(
 		ctx, targetUserNID, roomNID,
 	)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed")
 	var result []types.EventStateKeyNID
 	var eventIDs []string
 	var eventID string
 	var senderUserNID int64
+	var eventJSON []byte
 	for rows.Next() {
-		if err := rows.Scan(&eventID, &senderUserNID); err != nil {
-			return nil, nil, err
+		if err := rows.Scan(&eventID, &senderUserNID, &eventJSON); err != nil {
+			return nil, nil, nil, err
 		}
 		result = append(result, types.EventStateKeyNID(senderUserNID))
 		eventIDs = append(eventIDs, eventID)
 	}
-	return result, eventIDs, nil
+	return result, eventIDs, eventJSON, nil
 }
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index d7bcc95ab..8be47855f 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -116,7 +116,7 @@ type Invites interface {
 	InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte) (bool, error)
 	UpdateInviteRetired(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) ([]string, error)
 	// SelectInviteActiveForUserInRoom returns a list of sender state key NIDs and invite event IDs matching those nids.
-	SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, error)
+	SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, []byte, error)
 }
 
 type MembershipState int64
diff --git a/roomserver/storage/tables/invite_table_test.go b/roomserver/storage/tables/invite_table_test.go
index 8df3faa2d..e3eedbf14 100644
--- a/roomserver/storage/tables/invite_table_test.go
+++ b/roomserver/storage/tables/invite_table_test.go
@@ -4,6 +4,9 @@ import (
 	"context"
 	"testing"
 
+	"github.com/matrix-org/util"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
@@ -11,8 +14,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/util"
-	"github.com/stretchr/testify/assert"
 )
 
 func mustCreateInviteTable(t *testing.T, dbType test.DBType) (tables.Invites, func()) {
@@ -67,7 +68,7 @@ func TestInviteTable(t *testing.T) {
 		assert.NoError(t, err)
 		assert.True(t, newInvite)
 
-		stateKeyNIDs, eventIDs, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
+		stateKeyNIDs, eventIDs, _, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
 		assert.NoError(t, err)
 		assert.Equal(t, []string{eventID1, eventID2}, eventIDs)
 		assert.Equal(t, []types.EventStateKeyNID{2, 2}, stateKeyNIDs)
@@ -78,13 +79,13 @@ func TestInviteTable(t *testing.T) {
 		assert.Equal(t, []string{eventID1, eventID2}, retiredEventIDs)
 
 		// This should now be empty
-		stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
+		stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID)
 		assert.NoError(t, err)
 		assert.Empty(t, eventIDs)
 		assert.Empty(t, stateKeyNIDs)
 
 		// Non-existent targetUserNID
-		stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID)
+		stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID)
 		assert.NoError(t, err)
 		assert.Empty(t, stateKeyNIDs)
 		assert.Empty(t, eventIDs)

From f1b8df0f49a8a2f2c3c9c4d421b880bd2b008e08 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 7 Oct 2022 15:55:29 +0100
Subject: [PATCH 61/73] Version 0.10.2 (#2778)

Changelog and version bump.
---
 CHANGES.md          | 22 ++++++++++++++++++++++
 internal/version.go |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index dbe2ccf02..7552e41e5 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,27 @@
 # Changelog
 
+## Dendrite 0.10.2 (2022-10-07)
+
+### Features
+
+* Dendrite will now fail to start if there is an obvious problem with the configured `max_open_conns` when using PostgreSQL database backends, since this can lead to instability and performance issues
+  * More information on this is available [in the documentation](https://matrix-org.github.io/dendrite/installation/start/optimisation#postgresql-connection-limit)
+* Unnecessary/empty fields will no longer be sent in `/sync` responses
+* It is now possible to configure `old_private_keys` from previous Matrix installations on the same domain if only public key is known, to make it easier to expire old keys correctly
+  * You can configure either just the `private_key` path, or you can supply both the `public_key` and `key_id`
+
+### Fixes
+
+* The sync transaction behaviour has been modified further so that errors in one stream should not propagate to other streams unnecessarily
+* Rooms should now be classified as DM rooms correctly by passing through `is_direct` and unsigned hints
+* A bug which caused marking device lists as stale to consume lots of CPU has been fixed
+* Users accepting invites should no longer cause unnecessary federated joins if there are already other local users in the room
+* The sync API state range queries have been optimised by adding missing indexes
+* It should now be possible to configure non-English languages for full-text search in `search.language`
+* The roomserver will no longer attempt to perform federated requests to the local server when trying to fetch missing events
+* The `/keys/upload` endpoint will now always return the `one_time_keys_counts`, which may help with E2EE reliability
+* The sync API will now retrieve the latest stream position before processing each stream rather than at the beginning of the request, to hopefully reduce the number of round-trips to `/sync`
+
 ## Dendrite 0.10.1 (2022-09-30)
 
 ### Features
diff --git a/internal/version.go b/internal/version.go
index d508517be..56b83f852 100644
--- a/internal/version.go
+++ b/internal/version.go
@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 10
-	VersionPatch = 1
+	VersionPatch = 2
 	VersionTag   = "" // example: "rc1"
 )
 

From 980fa55846811eeff89f116c49b38b085143c64e Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 10:39:29 +0100
Subject: [PATCH 62/73] Stronger passwordless account checks (fixes #2780)

---
 userapi/internal/api.go           | 2 ++
 userapi/storage/shared/storage.go | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/userapi/internal/api.go b/userapi/internal/api.go
index 591faffd6..2f7795dfe 100644
--- a/userapi/internal/api.go
+++ b/userapi/internal/api.go
@@ -838,6 +838,8 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q
 		return nil
 	case bcrypt.ErrMismatchedHashAndPassword: // user exists, but password doesn't match
 		return nil
+	case bcrypt.ErrHashTooShort: // user exists, but probably a passwordless account
+		return nil
 	default:
 		res.Exists = true
 		res.Account = acc
diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index 3ff299f1b..09eeedc9f 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -75,6 +75,9 @@ func (d *Database) GetAccountByPassword(
 	if err != nil {
 		return nil, err
 	}
+	if hash == "" {
+		return nil, bcrypt.ErrHashTooShort
+	}
 	if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil {
 		return nil, err
 	}

From 04bab142901b9dc4479b2cf4e5504fbdd9dd3cf1 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 10:45:15 +0100
Subject: [PATCH 63/73] Add regression test for
 980fa55846811eeff89f116c49b38b085143c64e

---
 userapi/userapi_test.go | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index 31a69793b..984fe8854 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -151,6 +151,33 @@ func TestQueryProfile(t *testing.T) {
 	})
 }
 
+// TestPasswordlessLoginFails ensures that a passwordless account cannot
+// be logged into using an arbitrary password (effectively a regression test
+// for https://github.com/matrix-org/dendrite/issues/2780).
+func TestPasswordlessLoginFails(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType)
+		defer close()
+		_, err := accountDB.CreateAccount(ctx, "auser", "", "", api.AccountTypeAppService)
+		if err != nil {
+			t.Fatalf("failed to make account: %s", err)
+		}
+
+		userReq := &api.QueryAccountByPasswordRequest{
+			Localpart:         "auser",
+			PlaintextPassword: "apassword",
+		}
+		userRes := &api.QueryAccountByPasswordResponse{}
+		if err := userAPI.QueryAccountByPassword(ctx, userReq, userRes); err != nil {
+			t.Fatal(err)
+		}
+		if userRes.Exists || userRes.Account != nil {
+			t.Fatal("QueryAccountByPassword should not return correctly for a passwordless account")
+		}
+	})
+}
+
 func TestLoginToken(t *testing.T) {
 	ctx := context.Background()
 

From b32b6d6e8eb5d0a6c22e9215e2fccf4bdd1db8c3 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 11:03:52 +0100
Subject: [PATCH 64/73] Update issue and pull request templates

---
 .github/ISSUE_TEMPLATE/BUG_REPORT.md | 22 ++++++++++++----------
 .github/PULL_REQUEST_TEMPLATE.md     |  4 ++--
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
index 206713e04..49c9a37ef 100644
--- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md
+++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
@@ -7,24 +7,27 @@ about: Create a report to help us improve
 <!--
 All bug reports must provide the following background information
 Text between <!-- and --​> marks will be invisible in the report.
+
+IF YOUR ISSUE IS CONSIDERED A SECURITY VULNERABILITY THEN PLEASE STOP AND
+DO NOT POST IT AS A GITHUB ISSUE! Please report the issue responsibly by
+disclosing in private by email to security@matrix.org instead.
 -->
 
 ### Background information
 <!-- Please include versions of all software when known e.g database versions, docker versions, client versions -->
-- **Dendrite version or git SHA**: 
-- **Monolith or Polylith?**: 
-- **SQLite3 or Postgres?**: 
-- **Running in Docker?**: 
+- **Dendrite version or git SHA**:
+- **Monolith or Polylith?**:
+- **SQLite3 or Postgres?**:
+- **Running in Docker?**:
 - **`go version`**:
 - **Client used (if applicable)**:
 
-
 ### Description
 
- - **What** is the problem:
- - **Who** is affected:
- - **How** is this bug manifesting:
- - **When** did this first appear:
+- **What** is the problem:
+- **Who** is affected:
+- **How** is this bug manifesting:
+- **When** did this first appear:
 
 <!--
 Examples of good descriptions:
@@ -38,7 +41,6 @@ Examples of good descriptions:
 - How: "Lots of logs about device change updates"
 - When: "After my server joined Matrix HQ"
 
-
 Examples of bad descriptions:
 - What: "Can't send messages"  - This is bad because it isn't specfic enough. Which endpoint isn't working and what is the response code? Does the message send but encryption fail?
 - Who: "Me" - Who are you? Running the server or a user on a Dendrite server?
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index e0b82e2aa..d6c54ead1 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,8 +1,8 @@
 ### Pull Request Checklist
 
-<!-- Please read docs/CONTRIBUTING.md before submitting your pull request -->
+<!-- Please read https://matrix-org.github.io/dendrite/development/contributing before submitting your pull request -->
 
 * [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests.
-* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)
+* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately
 
 Signed-off-by: `Your Name <your@email.example.org>`

From 80a0ab6246aa095f428430c38b13861406dd5c78 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 11:09:40 +0100
Subject: [PATCH 65/73] Further tweak to the issue template

---
 .github/ISSUE_TEMPLATE/BUG_REPORT.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
index 49c9a37ef..f40c56609 100644
--- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md
+++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
@@ -8,9 +8,10 @@ about: Create a report to help us improve
 All bug reports must provide the following background information
 Text between <!-- and --​> marks will be invisible in the report.
 
-IF YOUR ISSUE IS CONSIDERED A SECURITY VULNERABILITY THEN PLEASE STOP AND
-DO NOT POST IT AS A GITHUB ISSUE! Please report the issue responsibly by
-disclosing in private by email to security@matrix.org instead.
+IF YOUR ISSUE IS CONSIDERED A SECURITY VULNERABILITY THEN PLEASE STOP
+AND DO NOT POST IT AS A GITHUB ISSUE! Please report the issue responsibly by
+disclosing in private by email to security@matrix.org instead. For more details, please
+see: https://www.matrix.org/security-disclosure-policy/
 -->
 
 ### Background information

From fb6cb2dbcbeb7cd7546ca4d126394720d215c310 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 11:14:16 +0100
Subject: [PATCH 66/73] Tweak `GetAccountByPassword` more

---
 clientapi/auth/password.go        | 6 ++++++
 userapi/storage/shared/storage.go | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go
index bcb4ca97b..890b18183 100644
--- a/clientapi/auth/password.go
+++ b/clientapi/auth/password.go
@@ -68,6 +68,12 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			JSON: jsonerror.BadJSON("A username must be supplied."),
 		}
 	}
+	if len(r.Password) == 0 {
+		return nil, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.BadJSON("A password must be supplied."),
+		}
+	}
 	localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName)
 	if err != nil {
 		return nil, &util.JSONResponse{
diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go
index 09eeedc9f..4e28f7b5a 100644
--- a/userapi/storage/shared/storage.go
+++ b/userapi/storage/shared/storage.go
@@ -75,7 +75,7 @@ func (d *Database) GetAccountByPassword(
 	if err != nil {
 		return nil, err
 	}
-	if hash == "" {
+	if len(hash) == 0 && len(plaintextPassword) > 0 {
 		return nil, bcrypt.ErrHashTooShort
 	}
 	if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil {

From 0f09e9d196d3375e38a490881e06668a82fb6c40 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 12:19:16 +0200
Subject: [PATCH 67/73] Move /event to the SyncAPI (#2782)

This allows us to apply history visibility without having to recalculate
it in the roomserver.
Unblocks https://github.com/matrix-org/complement/pull/495, fix missing
part of https://github.com/matrix-org/dendrite/issues/617
---
 clientapi/routing/getevent.go          | 138 -------------------------
 clientapi/routing/routing.go           |   9 --
 docs/caddy/polylith/Caddyfile          |   2 +-
 docs/hiawatha/polylith-sample.conf     |   4 +-
 docs/nginx/polylith-sample.conf        |   4 +-
 syncapi/internal/history_visibility.go |   8 +-
 syncapi/routing/getevent.go            | 102 ++++++++++++++++++
 syncapi/routing/routing.go             |  10 ++
 8 files changed, 124 insertions(+), 153 deletions(-)
 delete mode 100644 clientapi/routing/getevent.go
 create mode 100644 syncapi/routing/getevent.go

diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go
deleted file mode 100644
index 7f5842800..000000000
--- a/clientapi/routing/getevent.go
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2019 Alex Chen
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package routing
-
-import (
-	"net/http"
-
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup/config"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-)
-
-type getEventRequest struct {
-	req            *http.Request
-	device         *userapi.Device
-	roomID         string
-	eventID        string
-	cfg            *config.ClientAPI
-	requestedEvent *gomatrixserverlib.Event
-}
-
-// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}
-// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid
-func GetEvent(
-	req *http.Request,
-	device *userapi.Device,
-	roomID string,
-	eventID string,
-	cfg *config.ClientAPI,
-	rsAPI api.ClientRoomserverAPI,
-) util.JSONResponse {
-	eventsReq := api.QueryEventsByIDRequest{
-		EventIDs: []string{eventID},
-	}
-	var eventsResp api.QueryEventsByIDResponse
-	err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if len(eventsResp.Events) == 0 {
-		// Event not found locally
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
-		}
-	}
-
-	requestedEvent := eventsResp.Events[0].Event
-
-	r := getEventRequest{
-		req:            req,
-		device:         device,
-		roomID:         roomID,
-		eventID:        eventID,
-		cfg:            cfg,
-		requestedEvent: requestedEvent,
-	}
-
-	stateReq := api.QueryStateAfterEventsRequest{
-		RoomID:       r.requestedEvent.RoomID(),
-		PrevEventIDs: r.requestedEvent.PrevEventIDs(),
-		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
-			EventType: gomatrixserverlib.MRoomMember,
-			StateKey:  device.UserID,
-		}},
-	}
-	var stateResp api.QueryStateAfterEventsResponse
-	if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if !stateResp.RoomExists {
-		util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID())
-		return jsonerror.InternalServerError()
-	}
-
-	if !stateResp.PrevEventsExist {
-		// Missing some events locally; stateResp.StateEvents unavailable.
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
-		}
-	}
-
-	var appService *config.ApplicationService
-	if device.AppserviceID != "" {
-		for _, as := range cfg.Derived.ApplicationServices {
-			if as.ID == device.AppserviceID {
-				appService = &as
-				break
-			}
-		}
-	}
-
-	for _, stateEvent := range stateResp.StateEvents {
-		if appService != nil {
-			if !appService.IsInterestedInUserID(*stateEvent.StateKey()) {
-				continue
-			}
-		} else if !stateEvent.StateKeyEquals(device.UserID) {
-			continue
-		}
-		membership, err := stateEvent.Membership()
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("stateEvent.Membership failed")
-			return jsonerror.InternalServerError()
-		}
-		if membership == gomatrixserverlib.Join {
-			return util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll),
-			}
-		}
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusNotFound,
-		JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
-	}
-}
diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 7d1c434c4..f1fa66ca6 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -367,15 +367,6 @@ func Setup(
 				nil, cfg, rsAPI, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
-	v3mux.Handle("/rooms/{roomID}/event/{eventID}",
-		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
-		}),
-	).Methods(http.MethodGet, http.MethodOptions)
 
 	v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile
index 244e50e7e..906097e4e 100644
--- a/docs/caddy/polylith/Caddyfile
+++ b/docs/caddy/polylith/Caddyfile
@@ -55,7 +55,7 @@ matrix.example.com {
 	# Change the end of each reverse_proxy line to the correct
 	# address for your various services.
 	@sync_api {
-		path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$
+		path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$
 	}
 	reverse_proxy @sync_api sync_api:8073
 
diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf
index 5ed0cb5ae..036140643 100644
--- a/docs/hiawatha/polylith-sample.conf
+++ b/docs/hiawatha/polylith-sample.conf
@@ -18,8 +18,10 @@ VirtualHost {
         # /_matrix/client/.*/user/{userId}/filter/{filterID}
         # /_matrix/client/.*/keys/changes
         # /_matrix/client/.*/rooms/{roomId}/messages
+        # /_matrix/client/.*/rooms/{roomId}/context/{eventID}
+        # /_matrix/client/.*/rooms/{roomId}/event/{eventID}
         # to sync_api
-        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600
+        ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600
         ReverseProxy = /_matrix/client http://localhost:8071 600
         ReverseProxy = /_matrix/federation http://localhost:8072 600
         ReverseProxy = /_matrix/key http://localhost:8072 600
diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf
index 274d75658..345d8a6b4 100644
--- a/docs/nginx/polylith-sample.conf
+++ b/docs/nginx/polylith-sample.conf
@@ -28,8 +28,10 @@ server {
     # /_matrix/client/.*/user/{userId}/filter/{filterID}
     # /_matrix/client/.*/keys/changes
     # /_matrix/client/.*/rooms/{roomId}/messages
+    # /_matrix/client/.*/rooms/{roomId}/context/{eventID}
+    # /_matrix/client/.*/rooms/{roomId}/event/{eventID}
     # to sync_api
-    location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$  {
+    location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$  {
         proxy_pass http://sync_api:8073;
     }
 
diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go
index bbfe19f4c..71d7ddd15 100644
--- a/syncapi/internal/history_visibility.go
+++ b/syncapi/internal/history_visibility.go
@@ -19,11 +19,13 @@ import (
 	"math"
 	"time"
 
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/storage"
 )
 
 func init() {
@@ -189,7 +191,7 @@ func visibilityForEvents(
 		UserID:   userID,
 	}, membershipResp)
 	if err != nil {
-		return result, err
+		logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'")
 	}
 
 	// Create a map from eventID -> eventVisibility
diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go
new file mode 100644
index 000000000..d2cdc1b5f
--- /dev/null
+++ b/syncapi/routing/getevent.go
@@ -0,0 +1,102 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi/internal"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+// GetEvent implements
+//
+//	GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}
+//
+// https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv3roomsroomideventeventid
+func GetEvent(
+	req *http.Request,
+	device *userapi.Device,
+	roomID string,
+	eventID string,
+	cfg *config.SyncAPI,
+	syncDB storage.Database,
+	rsAPI api.SyncRoomserverAPI,
+) util.JSONResponse {
+	ctx := req.Context()
+	db, err := syncDB.NewDatabaseTransaction(ctx)
+	logger := util.GetLogger(ctx).WithFields(logrus.Fields{
+		"event_id": eventID,
+		"room_id":  roomID,
+	})
+	if err != nil {
+		logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed")
+		return jsonerror.InternalServerError()
+	}
+
+	events, err := db.Events(ctx, []string{eventID})
+	if err != nil {
+		logger.WithError(err).Error("GetEvent: syncDB.Events failed")
+		return jsonerror.InternalServerError()
+	}
+
+	// The requested event does not exist in our database
+	if len(events) == 0 {
+		logger.Debugf("GetEvent: requested event doesn't exist locally")
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
+		}
+	}
+
+	// If the request is coming from an appservice, get the user from the request
+	userID := device.UserID
+	if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" {
+		userID = asUserID
+	}
+
+	// Apply history visibility to determine if the user is allowed to view the event
+	events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event")
+	if err != nil {
+		logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalServerError(),
+		}
+	}
+
+	// We only ever expect there to be one event
+	if len(events) != 1 {
+		// 0 events -> not allowed to view event; > 1 events -> something that shouldn't happen
+		logger.WithField("event_count", len(events)).Debug("GetEvent: can't return the requested event")
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: gomatrixserverlib.HeaderedToClientEvent(events[0], gomatrixserverlib.FormatAll),
+	}
+}
diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go
index 8f84a1341..069dee81f 100644
--- a/syncapi/routing/routing.go
+++ b/syncapi/routing/routing.go
@@ -60,6 +60,16 @@ func Setup(
 		return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, rsAPI, cfg, srp, lazyLoadCache)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
+	v3mux.Handle("/rooms/{roomID}/event/{eventID}",
+		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, syncDB, rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	v3mux.Handle("/user/{userId}/filter",
 		httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))

From dcc01162874d2634738080614f02cbbeb6bc598f Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 15:38:00 +0200
Subject: [PATCH 68/73] SyTest List Maintenance

---
 sytest-blacklist | 1 -
 sytest-whitelist | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/sytest-blacklist b/sytest-blacklist
index 5b2e973a6..634c07cf3 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -47,7 +47,6 @@ Notifications can be viewed with GET /notifications
 
 # More flakey
 
-If remote user leaves room we no longer receive device updates
 Guest users can join guest_access rooms
 
 # This will fail in HTTP API mode, so blacklisted for now
diff --git a/sytest-whitelist b/sytest-whitelist
index 31940b884..9ba9df75d 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -742,3 +742,4 @@ User in private room doesn't appear in user directory
 User joining then leaving public room appears and dissappears from directory
 User in remote room doesn't appear in user directory after server left room
 User in shared private room does appear in user directory until leave
+Existing members see new member's presence
\ No newline at end of file

From 39581af3ba657032fbf66ba3719a1cb334c0519b Mon Sep 17 00:00:00 2001
From: Till Faelligen <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 15:49:56 +0200
Subject: [PATCH 69/73] CI update

---
 .github/workflows/dendrite.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml
index be3c7c173..f8019b3ea 100644
--- a/.github/workflows/dendrite.yml
+++ b/.github/workflows/dendrite.yml
@@ -342,7 +342,7 @@ jobs:
         # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
+          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
 
       - name: Run actions/checkout@v2 for dendrite
         uses: actions/checkout@v2

From b000db81ca889b350663493c27eb58654c75e295 Mon Sep 17 00:00:00 2001
From: Till <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 10 Oct 2022 17:36:26 +0200
Subject: [PATCH 70/73] Send E2EE related errors to sentry (#2784)

Only sends errors if we're not retrying them in NATS.
Not sure if those should be scoped/tagged with something like "E2EE".
---
 federationapi/consumers/keychange.go    | 17 ++++++++++++++---
 federationapi/consumers/sendtodevice.go | 14 ++++++++++----
 federationapi/queue/queue.go            |  3 +++
 federationapi/routing/send.go           |  4 ++++
 4 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go
index f3314bc98..67dfdc1d3 100644
--- a/federationapi/consumers/keychange.go
+++ b/federationapi/consumers/keychange.go
@@ -18,6 +18,11 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/types"
@@ -26,9 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	"github.com/sirupsen/logrus"
 )
 
 // KeyChangeConsumer consumes events that originate in key server.
@@ -78,6 +80,7 @@ func (t *KeyChangeConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) boo
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	var m api.DeviceMessage
 	if err := json.Unmarshal(msg.Data, &m); err != nil {
+		sentry.CaptureException(err)
 		logrus.WithError(err).Errorf("failed to read device message from key change topic")
 		return true
 	}
@@ -105,6 +108,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	// only send key change events which originated from us
 	_, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("Failed to extract domain from key change event")
 		return true
 	}
@@ -118,6 +122,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		WantMembership: "join",
 	}, &queryRes)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
 		return true
 	}
@@ -125,6 +130,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in")
 		return true
 	}
@@ -147,6 +153,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 		Keys:              m.KeyJSON,
 	}
 	if edu.Content, err = json.Marshal(event); err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("failed to marshal EDU JSON")
 		return true
 	}
@@ -160,6 +167,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 	output := m.CrossSigningKeyUpdate
 	_, host, err := gomatrixserverlib.SplitID('@', output.UserID)
 	if err != nil {
+		sentry.CaptureException(err)
 		logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure")
 		return true
 	}
@@ -176,12 +184,14 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 		WantMembership: "join",
 	}, &queryRes)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user")
 		return true
 	}
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in")
 		return true
 	}
@@ -196,6 +206,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 		Origin: string(t.serverName),
 	}
 	if edu.Content, err = json.Marshal(output); err != nil {
+		sentry.CaptureException(err)
 		logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping")
 		return true
 	}
diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go
index ffc1d8894..9aec22a3e 100644
--- a/federationapi/consumers/sendtodevice.go
+++ b/federationapi/consumers/sendtodevice.go
@@ -18,16 +18,18 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/getsentry/sentry-go"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/nats-io/nats.go"
-	log "github.com/sirupsen/logrus"
 )
 
 // OutputSendToDeviceConsumer consumes events that originate in the clientapi.
@@ -76,6 +78,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 	sender := msg.Header.Get("sender")
 	_, originServerName, err := gomatrixserverlib.SplitID('@', sender)
 	if err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender")
 		return true
 	}
@@ -85,12 +88,14 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 	// Extract the send-to-device event from msg.
 	var ote syncTypes.OutputSendToDeviceEvent
 	if err = json.Unmarshal(msg.Data, &ote); err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)")
 		return true
 	}
 
 	_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
 	if err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
 		return true
 	}
@@ -116,6 +121,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
 		},
 	}
 	if edu.Content, err = json.Marshal(tdm); err != nil {
+		sentry.CaptureException(err)
 		log.WithError(err).Error("failed to marshal EDU JSON")
 		return true
 	}
diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go
index 88664fcf9..8245aa5bd 100644
--- a/federationapi/queue/queue.go
+++ b/federationapi/queue/queue.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
@@ -307,11 +308,13 @@ func (oqs *OutgoingQueues) SendEDU(
 
 	ephemeralJSON, err := json.Marshal(e)
 	if err != nil {
+		sentry.CaptureException(err)
 		return fmt.Errorf("json.Marshal: %w", err)
 	}
 
 	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON))
 	if err != nil {
+		sentry.CaptureException(err)
 		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
 	}
 
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index 060af676d..b3bbaa394 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -22,6 +22,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -350,6 +351,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				for deviceID, message := range byUser {
 					// TODO: check that the user and the device actually exist here
 					if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil {
+						sentry.CaptureException(err)
 						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
 							"sender":    directPayload.Sender,
 							"user_id":   userID,
@@ -360,6 +362,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			}
 		case gomatrixserverlib.MDeviceListUpdate:
 			if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil {
+				sentry.CaptureException(err)
 				util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate")
 			}
 		case gomatrixserverlib.MReceipt:
@@ -395,6 +398,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			}
 		case types.MSigningKeyUpdate:
 			if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil {
+				sentry.CaptureException(err)
 				logrus.WithError(err).Errorf("Failed to process signing key update")
 			}
 		case gomatrixserverlib.MPresence:

From 6bf1912525724baa1a8bf5e0bf51524d7cee59ba Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Mon, 10 Oct 2022 16:54:04 +0100
Subject: [PATCH 71/73] Fix joined hosts with `RewritesState` (#2785)

This ensures that the joined hosts in the federation API are correct
after the state is rewritten. This might fix some races around the time
of joining federated rooms.
---
 federationapi/storage/shared/storage.go | 32 ++++++++++++-------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go
index a00d782f1..9e40f311c 100644
--- a/federationapi/storage/shared/storage.go
+++ b/federationapi/storage/shared/storage.go
@@ -70,27 +70,27 @@ func (d *Database) UpdateRoom(
 ) (joinedHosts []types.JoinedHost, err error) {
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		if purgeRoomFirst {
-			// If the event is a create event then we'll delete all of the existing
-			// data for the room. The only reason that a create event would be replayed
-			// to us in this way is if we're about to receive the entire room state.
 			if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
 				return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err)
 			}
-		}
-
-		joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID)
-		if err != nil {
-			return err
-		}
-
-		for _, add := range addHosts {
-			err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName)
-			if err != nil {
+			for _, add := range addHosts {
+				if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil {
+					return err
+				}
+				joinedHosts = append(joinedHosts, add)
+			}
+		} else {
+			if joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID); err != nil {
+				return err
+			}
+			for _, add := range addHosts {
+				if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil {
+					return err
+				}
+			}
+			if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil {
 				return err
 			}
-		}
-		if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil {
-			return err
 		}
 		return nil
 	})

From 9ed8ff6b938e0452d122c68b38cf4fedcd74c0bf Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 11 Oct 2022 10:48:36 +0100
Subject: [PATCH 72/73] Tweak federation `M_NOT_FOUND` errors

---
 federationapi/routing/events.go |  6 +++++-
 federationapi/routing/state.go  | 15 ++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go
index 23796edfa..6168912bd 100644
--- a/federationapi/routing/events.go
+++ b/federationapi/routing/events.go
@@ -20,6 +20,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -95,7 +96,10 @@ func fetchEvent(ctx context.Context, rsAPI api.FederationRoomserverAPI, eventID
 	}
 
 	if len(eventsResponse.Events) == 0 {
-		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
+		return nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Event not found"),
+		}
 	}
 
 	return eventsResponse.Events[0].Event, nil
diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go
index 5377eb88f..1d08d0a82 100644
--- a/federationapi/routing/state.go
+++ b/federationapi/routing/state.go
@@ -135,23 +135,24 @@ func getState(
 		return nil, nil, &resErr
 	}
 
-	if !response.StateKnown {
+	switch {
+	case !response.RoomExists:
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Room not found"),
+		}
+	case !response.StateKnown:
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("State not known"),
 		}
-	}
-	if response.IsRejected {
+	case response.IsRejected:
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("Event not found"),
 		}
 	}
 
-	if !response.RoomExists {
-		return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
-	}
-
 	return response.StateEvents, response.AuthChainEvents, nil
 }
 

From 3920b9f9b6155db69822d0dbcd36acb1eaa51c34 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Tue, 11 Oct 2022 10:58:34 +0100
Subject: [PATCH 73/73] Tweak `GetStateDeltas` behaviour (#2788)

Improves the control flow of `GetStateDeltas` for clarity and possibly
also fixes a bug where duplicate state delta entries could be inserted
with different memberships instead of being correctly overridden by
`join`.
---
 syncapi/storage/shared/storage_sync.go | 44 ++++++++++++++++++--------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go
index 0e19d97d2..d5b5b3121 100644
--- a/syncapi/storage/shared/storage_sync.go
+++ b/syncapi/storage/shared/storage_sync.go
@@ -360,34 +360,50 @@ func (d *DatabaseTransaction) GetStateDeltas(
 	newlyJoinedRooms := make(map[string]bool, len(state))
 	for roomID, stateStreamEvents := range state {
 		for _, ev := range stateStreamEvents {
-			if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" {
-				if membership == gomatrixserverlib.Join && prevMembership != membership {
-					// send full room state down instead of a delta
+			// Look for our membership in the state events and skip over any
+			// membership events that are not related to us.
+			membership, prevMembership := getMembershipFromEvent(ev.Event, userID)
+			if membership == "" {
+				continue
+			}
+
+			if membership == gomatrixserverlib.Join {
+				// If our membership is now join but the previous membership wasn't
+				// then this is a "join transition", so we'll insert this room.
+				if prevMembership != membership {
+					// Get the full room state, as we'll send that down for a newly
+					// joined room instead of a delta.
 					var s []types.StreamEvent
-					s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter)
-					if err != nil {
+					if s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter); err != nil {
 						if err == sql.ErrNoRows {
 							continue
 						}
 						return nil, nil, err
 					}
+
+					// Add the information for this room into the state so that
+					// it will get added with all of the rest of the joined rooms.
 					state[roomID] = s
 					newlyJoinedRooms[roomID] = true
-					continue // we'll add this room in when we do joined rooms
 				}
 
-				deltas = append(deltas, types.StateDelta{
-					Membership:    membership,
-					MembershipPos: ev.StreamPosition,
-					StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
-					RoomID:        roomID,
-				})
-				break
+				// We won't add joined rooms into the delta at this point as they
+				// are added later on.
+				continue
 			}
+
+			deltas = append(deltas, types.StateDelta{
+				Membership:    membership,
+				MembershipPos: ev.StreamPosition,
+				StateEvents:   d.StreamEventsToEvents(device, stateStreamEvents),
+				RoomID:        roomID,
+			})
+			break
 		}
 	}
 
-	// Add in currently joined rooms
+	// Finally, add in currently joined rooms, including those from the
+	// join transitions above.
 	for _, joinedRoomID := range joinedRoomIDs {
 		deltas = append(deltas, types.StateDelta{
 			Membership:  gomatrixserverlib.Join,