From 030b99563608bd86cec90793882168a410cf0edb Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 9 Mar 2022 10:19:49 +0000
Subject: [PATCH 1/9] Update to matrix-org/pinecone@cfde1f8

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 7311dd067..9957a5b6c 100644
--- a/go.mod
+++ b/go.mod
@@ -41,7 +41,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902
-	github.com/matrix-org/pinecone v0.0.0-20220223104432-0f0afd1a46aa
+	github.com/matrix-org/pinecone v0.0.0-20220308124038-cfde1f8054c5
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
 	github.com/morikuni/aec v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 74c6278e5..951c0eba9 100644
--- a/go.sum
+++ b/go.sum
@@ -985,8 +985,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902 h1:WHlrE8BYh/hzn1RKwq3YMAlhHivX47jQKAjZFtkJyPE=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo=
-github.com/matrix-org/pinecone v0.0.0-20220223104432-0f0afd1a46aa h1:rMYFNVto66gp+eWS8XAUzgp4m0qmUBid6l1HX3mHstk=
-github.com/matrix-org/pinecone v0.0.0-20220223104432-0f0afd1a46aa/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
+github.com/matrix-org/pinecone v0.0.0-20220308124038-cfde1f8054c5 h1:7viLTiLAA2MtGKY+uf14j6TjfKvvGLAMj/qdm70jJuQ=
+github.com/matrix-org/pinecone v0.0.0-20220308124038-cfde1f8054c5/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From 089d16812c8a94b39592d42d7c15edc2d6746f1d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 9 Mar 2022 10:42:24 +0000
Subject: [PATCH 2/9] Fix `GET /directory/list/room/{roomID}` (#2262)

* Let's try to work out why this endpoint lies

* Try that again

* Fix `QueryPublishedRooms`

* Remove logging

* Remove unnecessary change

* Remove unnecessary change
---
 roomserver/internal/query/query.go   | 8 ++++++++
 roomserver/storage/interface.go      | 2 ++
 roomserver/storage/shared/storage.go | 4 ++++
 3 files changed, 14 insertions(+)

diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index 70cc5d62c..471c6fb42 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -610,6 +610,14 @@ func (r *Queryer) QueryPublishedRooms(
 	req *api.QueryPublishedRoomsRequest,
 	res *api.QueryPublishedRoomsResponse,
 ) error {
+	if req.RoomID != "" {
+		visible, err := r.DB.GetPublishedRoom(ctx, req.RoomID)
+		if err == nil && visible {
+			res.RoomIDs = []string{req.RoomID}
+			return nil
+		}
+		return err
+	}
 	rooms, err := r.DB.GetPublishedRooms(ctx)
 	if err != nil {
 		return err
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index a2b22b401..a98fda073 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -139,6 +139,8 @@ type Database interface {
 	PublishRoom(ctx context.Context, roomID string, publish bool) error
 	// Returns a list of room IDs for rooms which are published.
 	GetPublishedRooms(ctx context.Context) ([]string, error)
+	// Returns whether a given room is published or not.
+	GetPublishedRoom(ctx context.Context, roomID string) (bool, error)
 
 	// TODO: factor out - from currentstateserver
 
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index dd49f35ca..68fd3867c 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -669,6 +669,10 @@ func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool)
 	})
 }
 
+func (d *Database) GetPublishedRoom(ctx context.Context, roomID string) (bool, error) {
+	return d.PublishedTable.SelectPublishedFromRoomID(ctx, nil, roomID)
+}
+
 func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) {
 	return d.PublishedTable.SelectAllPublishedRooms(ctx, nil, true)
 }

From d78e0a33e02518f7d5b901a584f3e2151759cb98 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 9 Mar 2022 16:38:50 +0000
Subject: [PATCH 3/9] Remove references to `userapi_devices` (fixes #2259)

---
 build/docker/config/dendrite.yaml  | 65 ++++++++++++++----------------
 build/docker/postgres/create_db.sh |  2 +-
 docs/INSTALL.md                    |  2 +-
 setup/config/config_test.go        |  5 ---
 4 files changed, 32 insertions(+), 42 deletions(-)

diff --git a/build/docker/config/dendrite.yaml b/build/docker/config/dendrite.yaml
index ebae50132..c01ab62e7 100644
--- a/build/docker/config/dendrite.yaml
+++ b/build/docker/config/dendrite.yaml
@@ -6,7 +6,7 @@
 #
 # At a minimum, to get started, you will need to update the settings in the
 # "global" section for your deployment, and you will need to check that the
-# database "connection_string" line in each component section is correct. 
+# database "connection_string" line in each component section is correct.
 #
 # Each component with a "database" section can accept the following formats
 # for "connection_string":
@@ -21,13 +21,13 @@
 # small number of users and likely will perform worse still with a higher volume
 # of users.
 #
-# The "max_open_conns" and "max_idle_conns" settings configure the maximum 
+# The "max_open_conns" and "max_idle_conns" settings configure the maximum
 # number of open/idle database connections. The value 0 will use the database
 # engine default, and a negative value will use unlimited connections. The
 # "conn_max_lifetime" option controls the maximum length of time a database
 # connection can be idle in seconds - a negative value is unlimited.
 
-# The version of the configuration file. 
+# The version of the configuration file.
 version: 2
 
 # Global Matrix configuration. This configuration applies to all components.
@@ -59,8 +59,8 @@ global:
   # Lists of domains that the server will trust as identity servers to verify third
   # party identifiers such as phone numbers and email addresses.
   trusted_third_party_id_servers:
-  - matrix.org
-  - vector.im
+    - matrix.org
+    - vector.im
 
   # Configuration for NATS JetStream
   jetstream:
@@ -76,7 +76,7 @@ global:
     # should not be used on a real world Dendrite deployment.
     in_memory: false
 
-    # Persistent directory to store JetStream streams in. This directory 
+    # Persistent directory to store JetStream streams in. This directory
     # should be preserved across Dendrite restarts.
     storage_path: ./
 
@@ -138,13 +138,13 @@ client_api:
   # Whether to require reCAPTCHA for registration.
   enable_registration_captcha: false
 
-  # Settings for ReCAPTCHA. 
+  # Settings for ReCAPTCHA.
   recaptcha_public_key: ""
   recaptcha_private_key: ""
   recaptcha_bypass_secret: ""
   recaptcha_siteverify_api: ""
 
-  # TURN server information that this homeserver should send to clients. 
+  # TURN server information that this homeserver should send to clients.
   turn:
     turn_user_lifetime: ""
     turn_uris: []
@@ -153,7 +153,7 @@ client_api:
     turn_password: ""
 
   # Settings for rate-limited endpoints. Rate limiting will kick in after the
-  # threshold number of "slots" have been taken by requests from a specific 
+  # threshold number of "slots" have been taken by requests from a specific
   # host. Each "slot" will be released after the cooloff time in milliseconds.
   rate_limiting:
     enabled: true
@@ -204,13 +204,13 @@ federation_api:
   # be required to satisfy key requests for servers that are no longer online when
   # joining some rooms.
   key_perspectives:
-  - server_name: matrix.org
-    keys:
-    - key_id: ed25519:auto
-      public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
-    - key_id: ed25519:a_RXGa
-      public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
-      
+    - server_name: matrix.org
+      keys:
+        - key_id: ed25519:auto
+          public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
+        - key_id: ed25519:a_RXGa
+          public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
+
   # This option will control whether Dendrite will prefer to look up keys directly
   # or whether it should try perspective servers first, using direct fetches as a
   # last resort.
@@ -255,15 +255,15 @@ media_api:
 
   # A list of thumbnail sizes to be generated for media content.
   thumbnail_sizes:
-  - width: 32
-    height: 32
-    method: crop
-  - width: 96
-    height: 96
-    method: crop
-  - width: 640
-    height: 480
-    method: scale
+    - width: 32
+      height: 32
+      method: crop
+    - width: 96
+      height: 96
+      method: crop
+    - width: 640
+      height: 480
+      method: scale
 
 # Configuration for experimental MSC's
 mscs:
@@ -295,7 +295,7 @@ sync_api:
     listen: http://0.0.0.0:7773
     connect: http://sync_api:7773
   external_api:
-      listen: http://0.0.0.0:8073
+    listen: http://0.0.0.0:8073
   database:
     connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable
     max_open_conns: 10
@@ -312,11 +312,6 @@ user_api:
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
-  device_database:
-    connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_userapi_devices?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
 
 # Configuration for the Push Server API.
 push_server:
@@ -348,7 +343,7 @@ tracing:
 # Logging configuration, in addition to the standard logging that is sent to
 # stdout by Dendrite.
 logging:
-- type: file
-  level: info
-  params:
-    path: /var/log/dendrite
+  - type: file
+    level: info
+    params:
+      path: /var/log/dendrite
diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh
index 9093738e4..27d2a4df4 100755
--- a/build/docker/postgres/create_db.sh
+++ b/build/docker/postgres/create_db.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
 
-for db in userapi_accounts userapi_devices mediaapi syncapi roomserver keyserver federationapi appservice mscs; do
+for db in userapi_accounts mediaapi syncapi roomserver keyserver federationapi appservice mscs; do
     createdb -U dendrite -O dendrite dendrite_$db
 done
diff --git a/docs/INSTALL.md b/docs/INSTALL.md
index 686ae1dbb..a7b2e67f2 100644
--- a/docs/INSTALL.md
+++ b/docs/INSTALL.md
@@ -87,7 +87,7 @@ On macOS, omit `sudo -u postgres` from the below commands.
 * If you want to run each Dendrite component with its own database:
 
   ```bash
-  for i in mediaapi syncapi roomserver federationapi appservice keyserver userapi_accounts userapi_devices; do
+  for i in mediaapi syncapi roomserver federationapi appservice keyserver userapi_accounts; do
       sudo -u postgres createdb -O dendrite dendrite_$i
   done
   ```
diff --git a/setup/config/config_test.go b/setup/config/config_test.go
index 6467b7c82..e6f0a493e 100644
--- a/setup/config/config_test.go
+++ b/setup/config/config_test.go
@@ -200,11 +200,6 @@ user_api:
     max_open_conns: 100
     max_idle_conns: 2
     conn_max_lifetime: -1
-  device_database:
-    connection_string: file:userapi_devices.db
-    max_open_conns: 100
-    max_idle_conns: 2
-    conn_max_lifetime: -1
   pusher_database:
     connection_string: file:pushserver.db
     max_open_conns: 100

From e1881627d18c541bfad0f5dc210b0786b8e55f35 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 10 Mar 2022 12:44:31 +0000
Subject: [PATCH 4/9] Implement MSC3383 (update to
 matrix-org/gomatrixserverlib#296)

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9957a5b6c..ec4646b6b 100644
--- a/go.mod
+++ b/go.mod
@@ -40,7 +40,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220310124155-116ed5cc1bfa
 	github.com/matrix-org/pinecone v0.0.0-20220308124038-cfde1f8054c5
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
diff --git a/go.sum b/go.sum
index 951c0eba9..d6086d6e7 100644
--- a/go.sum
+++ b/go.sum
@@ -983,8 +983,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902 h1:WHlrE8BYh/hzn1RKwq3YMAlhHivX47jQKAjZFtkJyPE=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220310124155-116ed5cc1bfa h1:anEGvpRn4v6akmxFWqGDobB6csEt3OWmp67pufccimE=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220310124155-116ed5cc1bfa/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo=
 github.com/matrix-org/pinecone v0.0.0-20220308124038-cfde1f8054c5 h1:7viLTiLAA2MtGKY+uf14j6TjfKvvGLAMj/qdm70jJuQ=
 github.com/matrix-org/pinecone v0.0.0-20220308124038-cfde1f8054c5/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

From e485f9c2bd15bca397229444399fa7e168eca43d Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Thu, 10 Mar 2022 13:17:28 +0000
Subject: [PATCH 5/9] 64-bit stream IDs for device list updates (#2267)

---
 federationapi/consumers/keychange.go            |  4 ++--
 keyserver/api/api.go                            |  6 +++---
 keyserver/internal/device_list_update.go        |  2 +-
 keyserver/internal/device_list_update_test.go   | 14 +++++++-------
 keyserver/internal/internal.go                  |  2 +-
 keyserver/storage/interface.go                  |  2 +-
 keyserver/storage/postgres/device_keys_table.go | 10 +++++-----
 keyserver/storage/shared/storage.go             | 12 ++++--------
 keyserver/storage/sqlite3/device_keys_table.go  | 14 +++++++-------
 keyserver/storage/storage_test.go               |  2 +-
 keyserver/storage/tables/interface.go           |  2 +-
 11 files changed, 33 insertions(+), 37 deletions(-)

diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go
index 22dbc32da..33d716d25 100644
--- a/federationapi/consumers/keychange.go
+++ b/federationapi/consumers/keychange.go
@@ -203,9 +203,9 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 	return err == nil
 }
 
-func prevID(streamID int) []int {
+func prevID(streamID int64) []int64 {
 	if streamID <= 1 {
 		return nil
 	}
-	return []int{streamID - 1}
+	return []int64{streamID - 1}
 }
diff --git a/keyserver/api/api.go b/keyserver/api/api.go
index 54eb04f8a..d361c6222 100644
--- a/keyserver/api/api.go
+++ b/keyserver/api/api.go
@@ -70,7 +70,7 @@ type DeviceMessage struct {
 	*DeviceKeys                         `json:"DeviceKeys,omitempty"`
 	*eduapi.OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"`
 	// A monotonically increasing number which represents device changes for this user.
-	StreamID       int
+	StreamID       int64
 	DeviceChangeID int64
 }
 
@@ -108,7 +108,7 @@ type DeviceKeys struct {
 }
 
 // WithStreamID returns a copy of this device message with the given stream ID
-func (k *DeviceKeys) WithStreamID(streamID int) DeviceMessage {
+func (k *DeviceKeys) WithStreamID(streamID int64) DeviceMessage {
 	return DeviceMessage{
 		DeviceKeys: k,
 		StreamID:   streamID,
@@ -281,7 +281,7 @@ type QueryDeviceMessagesRequest struct {
 
 type QueryDeviceMessagesResponse struct {
 	// The latest stream ID
-	StreamID int
+	StreamID int64
 	Devices  []DeviceMessage
 	Error    *KeyError
 }
diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go
index 974d0196b..4b2b8c187 100644
--- a/keyserver/internal/device_list_update.go
+++ b/keyserver/internal/device_list_update.go
@@ -109,7 +109,7 @@ type DeviceListUpdaterDatabase interface {
 	StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceMessage, clearUserIDs []string) error
 
 	// PrevIDsExists returns true if all prev IDs exist for this user.
-	PrevIDsExists(ctx context.Context, userID string, prevIDs []int) (bool, error)
+	PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error)
 
 	// DeviceKeysJSON populates the KeyJSON for the given keys. If any proided `keys` have a `KeyJSON` or `StreamID` already then it will be replaced.
 	DeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error
diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go
index ff939355f..0033a5086 100644
--- a/keyserver/internal/device_list_update_test.go
+++ b/keyserver/internal/device_list_update_test.go
@@ -46,7 +46,7 @@ func (p *mockKeyChangeProducer) ProduceKeyChanges(keys []api.DeviceMessage) erro
 
 type mockDeviceListUpdaterDatabase struct {
 	staleUsers   map[string]bool
-	prevIDsExist func(string, []int) bool
+	prevIDsExist func(string, []int64) bool
 	storedKeys   []api.DeviceMessage
 	mu           sync.Mutex // protect staleUsers
 }
@@ -101,7 +101,7 @@ func (d *mockDeviceListUpdaterDatabase) StoreRemoteDeviceKeys(ctx context.Contex
 }
 
 // PrevIDsExists returns true if all prev IDs exist for this user.
-func (d *mockDeviceListUpdaterDatabase) PrevIDsExists(ctx context.Context, userID string, prevIDs []int) (bool, error) {
+func (d *mockDeviceListUpdaterDatabase) PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error) {
 	return d.prevIDsExist(userID, prevIDs), nil
 }
 
@@ -139,7 +139,7 @@ func newFedClient(tripper func(*http.Request) (*http.Response, error)) *gomatrix
 func TestUpdateHavePrevID(t *testing.T) {
 	db := &mockDeviceListUpdaterDatabase{
 		staleUsers: make(map[string]bool),
-		prevIDsExist: func(string, []int) bool {
+		prevIDsExist: func(string, []int64) bool {
 			return true
 		},
 	}
@@ -151,7 +151,7 @@ func TestUpdateHavePrevID(t *testing.T) {
 		Deleted:           false,
 		DeviceID:          "FOO",
 		Keys:              []byte(`{"key":"value"}`),
-		PrevID:            []int{0},
+		PrevID:            []int64{0},
 		StreamID:          1,
 		UserID:            "@alice:localhost",
 	}
@@ -185,7 +185,7 @@ func TestUpdateHavePrevID(t *testing.T) {
 func TestUpdateNoPrevID(t *testing.T) {
 	db := &mockDeviceListUpdaterDatabase{
 		staleUsers: make(map[string]bool),
-		prevIDsExist: func(string, []int) bool {
+		prevIDsExist: func(string, []int64) bool {
 			return false
 		},
 	}
@@ -226,7 +226,7 @@ func TestUpdateNoPrevID(t *testing.T) {
 		Deleted:           false,
 		DeviceID:          "another_device_id",
 		Keys:              []byte(`{"key":"value"}`),
-		PrevID:            []int{3},
+		PrevID:            []int64{3},
 		StreamID:          4,
 		UserID:            remoteUserID,
 	}
@@ -268,7 +268,7 @@ func TestDebounce(t *testing.T) {
 	t.Skipf("panic on closed channel on GHA")
 	db := &mockDeviceListUpdaterDatabase{
 		staleUsers: make(map[string]bool),
-		prevIDsExist: func(string, []int) bool {
+		prevIDsExist: func(string, []int64) bool {
 			return true
 		},
 	}
diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go
index 0a8bef95d..cc9d3a616 100644
--- a/keyserver/internal/internal.go
+++ b/keyserver/internal/internal.go
@@ -205,7 +205,7 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query
 		}
 		return
 	}
-	maxStreamID := 0
+	maxStreamID := int64(0)
 	for _, m := range msgs {
 		if m.StreamID > maxStreamID {
 			maxStreamID = m.StreamID
diff --git a/keyserver/storage/interface.go b/keyserver/storage/interface.go
index 4dffe695c..16e034776 100644
--- a/keyserver/storage/interface.go
+++ b/keyserver/storage/interface.go
@@ -49,7 +49,7 @@ type Database interface {
 	StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceMessage, clearUserIDs []string) error
 
 	// PrevIDsExists returns true if all prev IDs exist for this user.
-	PrevIDsExists(ctx context.Context, userID string, prevIDs []int) (bool, error)
+	PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error)
 
 	// DeviceKeysForUser returns the device keys for the device IDs given. If the length of deviceIDs is 0, all devices are selected.
 	// If there are some missing keys, they are omitted from the returned slice. There is no ordering on the returned slice.
diff --git a/keyserver/storage/postgres/device_keys_table.go b/keyserver/storage/postgres/device_keys_table.go
index 628301cf7..ccd20cbd6 100644
--- a/keyserver/storage/postgres/device_keys_table.go
+++ b/keyserver/storage/postgres/device_keys_table.go
@@ -121,7 +121,7 @@ func NewPostgresDeviceKeysTable(db *sql.DB) (tables.DeviceKeys, error) {
 func (s *deviceKeysStatements) SelectDeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error {
 	for i, key := range keys {
 		var keyJSONStr string
-		var streamID int
+		var streamID int64
 		var displayName sql.NullString
 		err := s.selectDeviceKeysStmt.QueryRowContext(ctx, key.UserID, key.DeviceID).Scan(&keyJSONStr, &streamID, &displayName)
 		if err != nil && err != sql.ErrNoRows {
@@ -138,15 +138,15 @@ func (s *deviceKeysStatements) SelectDeviceKeysJSON(ctx context.Context, keys []
 	return nil
 }
 
-func (s *deviceKeysStatements) SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int32, err error) {
+func (s *deviceKeysStatements) SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int64, err error) {
 	// nullable if there are no results
-	var nullStream sql.NullInt32
+	var nullStream sql.NullInt64
 	err = sqlutil.TxStmt(txn, s.selectMaxStreamForUserStmt).QueryRowContext(ctx, userID).Scan(&nullStream)
 	if err == sql.ErrNoRows {
 		err = nil
 	}
 	if nullStream.Valid {
-		streamID = nullStream.Int32
+		streamID = nullStream.Int64
 	}
 	return
 }
@@ -211,7 +211,7 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID
 		}
 		dk.UserID = userID
 		var keyJSON string
-		var streamID int
+		var streamID int64
 		var displayName sql.NullString
 		if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil {
 			return nil, err
diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go
index f2790c8df..03215b93b 100644
--- a/keyserver/storage/shared/storage.go
+++ b/keyserver/storage/shared/storage.go
@@ -59,12 +59,8 @@ func (d *Database) DeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage)
 	return d.DeviceKeysTable.SelectDeviceKeysJSON(ctx, keys)
 }
 
-func (d *Database) PrevIDsExists(ctx context.Context, userID string, prevIDs []int) (bool, error) {
-	sids := make([]int64, len(prevIDs))
-	for i := range prevIDs {
-		sids[i] = int64(prevIDs[i])
-	}
-	count, err := d.DeviceKeysTable.CountStreamIDsForUser(ctx, userID, sids)
+func (d *Database) PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error) {
+	count, err := d.DeviceKeysTable.CountStreamIDsForUser(ctx, userID, prevIDs)
 	if err != nil {
 		return false, err
 	}
@@ -85,7 +81,7 @@ func (d *Database) StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceM
 
 func (d *Database) StoreLocalDeviceKeys(ctx context.Context, keys []api.DeviceMessage) error {
 	// work out the latest stream IDs for each user
-	userIDToStreamID := make(map[string]int)
+	userIDToStreamID := make(map[string]int64)
 	for _, k := range keys {
 		userIDToStreamID[k.UserID] = 0
 	}
@@ -95,7 +91,7 @@ func (d *Database) StoreLocalDeviceKeys(ctx context.Context, keys []api.DeviceMe
 			if err != nil {
 				return err
 			}
-			userIDToStreamID[userID] = int(streamID)
+			userIDToStreamID[userID] = streamID
 		}
 		// set the stream IDs for each key
 		for i := range keys {
diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/keyserver/storage/sqlite3/device_keys_table.go
index b461424c6..e77b49b35 100644
--- a/keyserver/storage/sqlite3/device_keys_table.go
+++ b/keyserver/storage/sqlite3/device_keys_table.go
@@ -145,7 +145,7 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID
 		dk.Type = api.TypeDeviceKeyUpdate
 		dk.UserID = userID
 		var keyJSON string
-		var streamID int
+		var streamID int64
 		var displayName sql.NullString
 		if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil {
 			return nil, err
@@ -166,7 +166,7 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID
 func (s *deviceKeysStatements) SelectDeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error {
 	for i, key := range keys {
 		var keyJSONStr string
-		var streamID int
+		var streamID int64
 		var displayName sql.NullString
 		err := s.selectDeviceKeysStmt.QueryRowContext(ctx, key.UserID, key.DeviceID).Scan(&keyJSONStr, &streamID, &displayName)
 		if err != nil && err != sql.ErrNoRows {
@@ -183,15 +183,15 @@ func (s *deviceKeysStatements) SelectDeviceKeysJSON(ctx context.Context, keys []
 	return nil
 }
 
-func (s *deviceKeysStatements) SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int32, err error) {
+func (s *deviceKeysStatements) SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int64, err error) {
 	// nullable if there are no results
-	var nullStream sql.NullInt32
+	var nullStream sql.NullInt64
 	err = sqlutil.TxStmt(txn, s.selectMaxStreamForUserStmt).QueryRowContext(ctx, userID).Scan(&nullStream)
 	if err == sql.ErrNoRows {
 		err = nil
 	}
 	if nullStream.Valid {
-		streamID = nullStream.Int32
+		streamID = nullStream.Int64
 	}
 	return
 }
@@ -204,13 +204,13 @@ func (s *deviceKeysStatements) CountStreamIDsForUser(ctx context.Context, userID
 	}
 	query := strings.Replace(countStreamIDsForUserSQL, "($2)", sqlutil.QueryVariadicOffset(len(streamIDs), 1), 1)
 	// nullable if there are no results
-	var count sql.NullInt32
+	var count sql.NullInt64
 	err := s.db.QueryRowContext(ctx, query, iStreamIDs...).Scan(&count)
 	if err != nil {
 		return 0, err
 	}
 	if count.Valid {
-		return int(count.Int32), nil
+		return int(count.Int64), nil
 	}
 	return 0, nil
 }
diff --git a/keyserver/storage/storage_test.go b/keyserver/storage/storage_test.go
index 4d5137249..84d2098ad 100644
--- a/keyserver/storage/storage_test.go
+++ b/keyserver/storage/storage_test.go
@@ -177,7 +177,7 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) {
 	if err != nil {
 		t.Fatalf("DeviceKeysForUser returned error: %s", err)
 	}
-	wantStreamIDs := map[string]int{
+	wantStreamIDs := map[string]int64{
 		"AAA":            3,
 		"another_device": 2,
 	}
diff --git a/keyserver/storage/tables/interface.go b/keyserver/storage/tables/interface.go
index cd1719598..f840cd1f3 100644
--- a/keyserver/storage/tables/interface.go
+++ b/keyserver/storage/tables/interface.go
@@ -37,7 +37,7 @@ type OneTimeKeys interface {
 type DeviceKeys interface {
 	SelectDeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error
 	InsertDeviceKeys(ctx context.Context, txn *sql.Tx, keys []api.DeviceMessage) error
-	SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int32, err error)
+	SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int64, err error)
 	CountStreamIDsForUser(ctx context.Context, userID string, streamIDs []int64) (int, error)
 	SelectBatchDeviceKeys(ctx context.Context, userID string, deviceIDs []string, includeEmpty bool) ([]api.DeviceMessage, error)
 	DeleteDeviceKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error

From 507a8e67739c2cfd89f5be48fdf96bdb0d1698d5 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 11 Mar 2022 12:48:45 +0000
Subject: [PATCH 6/9] Don't range entire state for `/sync` (#2270)

* Don't range entire state for rooms the user has no reason to care about

* Remove unnecessary db field in postgresql
---
 .../postgres/current_room_state_table.go      | 48 +++++++++++++++----
 .../postgres/output_room_events_table.go      | 17 +++----
 syncapi/storage/shared/syncserver.go          | 45 ++++++++++++-----
 .../sqlite3/current_room_state_table.go       | 48 +++++++++++++++----
 .../sqlite3/output_room_events_table.go       | 16 +++++--
 syncapi/storage/tables/interface.go           |  4 +-
 6 files changed, 137 insertions(+), 41 deletions(-)

diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go
index ee649c165..d646a0e41 100644
--- a/syncapi/storage/postgres/current_room_state_table.go
+++ b/syncapi/storage/postgres/current_room_state_table.go
@@ -77,6 +77,9 @@ const DeleteRoomStateForRoomSQL = "" +
 const selectRoomIDsWithMembershipSQL = "" +
 	"SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
 
+const selectRoomIDsWithAnyMembershipSQL = "" +
+	"SELECT DISTINCT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1"
+
 const selectCurrentStateSQL = "" +
 	"SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" +
 	" AND ( $2::text[] IS NULL OR     sender  = ANY($2)  )" +
@@ -102,14 +105,15 @@ const selectEventsWithEventIDsSQL = "" +
 	" FROM syncapi_current_room_state WHERE event_id = ANY($1)"
 
 type currentRoomStateStatements struct {
-	upsertRoomStateStmt             *sql.Stmt
-	deleteRoomStateByEventIDStmt    *sql.Stmt
-	DeleteRoomStateForRoomStmt      *sql.Stmt
-	selectRoomIDsWithMembershipStmt *sql.Stmt
-	selectCurrentStateStmt          *sql.Stmt
-	selectJoinedUsersStmt           *sql.Stmt
-	selectEventsWithEventIDsStmt    *sql.Stmt
-	selectStateEventStmt            *sql.Stmt
+	upsertRoomStateStmt                *sql.Stmt
+	deleteRoomStateByEventIDStmt       *sql.Stmt
+	DeleteRoomStateForRoomStmt         *sql.Stmt
+	selectRoomIDsWithMembershipStmt    *sql.Stmt
+	selectRoomIDsWithAnyMembershipStmt *sql.Stmt
+	selectCurrentStateStmt             *sql.Stmt
+	selectJoinedUsersStmt              *sql.Stmt
+	selectEventsWithEventIDsStmt       *sql.Stmt
+	selectStateEventStmt               *sql.Stmt
 }
 
 func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
@@ -130,6 +134,9 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
 	if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil {
 		return nil, err
 	}
+	if s.selectRoomIDsWithAnyMembershipStmt, err = db.Prepare(selectRoomIDsWithAnyMembershipSQL); err != nil {
+		return nil, err
+	}
 	if s.selectCurrentStateStmt, err = db.Prepare(selectCurrentStateSQL); err != nil {
 		return nil, err
 	}
@@ -194,6 +201,31 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 	return result, rows.Err()
 }
 
+// SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user.
+func (s *currentRoomStateStatements) SelectRoomIDsWithAnyMembership(
+	ctx context.Context,
+	txn *sql.Tx,
+	userID string,
+) (map[string]string, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithAnyMembershipStmt)
+	rows, err := stmt.QueryContext(ctx, userID)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithAnyMembership: rows.close() failed")
+
+	result := map[string]string{}
+	for rows.Next() {
+		var roomID string
+		var membership string
+		if err := rows.Scan(&roomID, &membership); err != nil {
+			return nil, err
+		}
+		result[roomID] = membership
+	}
+	return result, rows.Err()
+}
+
 // SelectCurrentState returns all the current state events for the given room.
 func (s *currentRoomStateStatements) SelectCurrentState(
 	ctx context.Context, txn *sql.Tx, roomID string,
diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go
index d4cc4f3fb..26689f447 100644
--- a/syncapi/storage/postgres/output_room_events_table.go
+++ b/syncapi/storage/postgres/output_room_events_table.go
@@ -119,13 +119,14 @@ const selectStateInRangeSQL = "" +
 	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
 	" FROM syncapi_output_room_events" +
 	" WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" +
-	" AND ( $3::text[] IS NULL OR     sender  = ANY($3)  )" +
-	" AND ( $4::text[] IS NULL OR NOT(sender  = ANY($4)) )" +
-	" AND ( $5::text[] IS NULL OR     type LIKE ANY($5)  )" +
-	" AND ( $6::text[] IS NULL OR NOT(type LIKE ANY($6)) )" +
-	" AND ( $7::bool IS NULL   OR     contains_url = $7  )" +
+	" AND room_id = ANY($3)" +
+	" AND ( $4::text[] IS NULL OR     sender  = ANY($4)  )" +
+	" AND ( $5::text[] IS NULL OR NOT(sender  = ANY($5)) )" +
+	" AND ( $6::text[] IS NULL OR     type LIKE ANY($6)  )" +
+	" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
+	" AND ( $8::bool IS NULL   OR     contains_url = $8  )" +
 	" ORDER BY id ASC" +
-	" LIMIT $8"
+	" LIMIT $9"
 
 const deleteEventsForRoomSQL = "" +
 	"DELETE FROM syncapi_output_room_events WHERE room_id = $1"
@@ -200,12 +201,12 @@ func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event
 // two positions, only the most recent state is returned.
 func (s *outputRoomEventsStatements) SelectStateInRange(
 	ctx context.Context, txn *sql.Tx, r types.Range,
-	stateFilter *gomatrixserverlib.StateFilter,
+	stateFilter *gomatrixserverlib.StateFilter, roomIDs []string,
 ) (map[string]map[string]bool, map[string]types.StreamEvent, error) {
 	stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt)
 
 	rows, err := stmt.QueryContext(
-		ctx, r.Low(), r.High(),
+		ctx, r.Low(), r.High(), pq.StringArray(roomIDs),
 		pq.StringArray(stateFilter.Senders),
 		pq.StringArray(stateFilter.NotSenders),
 		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)),
diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go
index 87d7c6df7..2c166eef7 100644
--- a/syncapi/storage/shared/syncserver.go
+++ b/syncapi/storage/shared/syncserver.go
@@ -689,10 +689,26 @@ func (d *Database) GetStateDeltas(
 	var succeeded bool
 	defer sqlutil.EndTransactionWithCheck(txn, &succeeded, &err)
 
+	// Look up all memberships for the user. We only care about rooms that a
+	// user has ever interacted with — joined to, kicked/banned from, left.
+	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	allRoomIDs := make([]string, 0, len(memberships))
+	joinedRoomIDs := make([]string, 0, len(memberships))
+	for roomID, membership := range memberships {
+		allRoomIDs = append(allRoomIDs, roomID)
+		if membership == gomatrixserverlib.Join {
+			joinedRoomIDs = append(joinedRoomIDs, roomID)
+		}
+	}
+
 	var deltas []types.StateDelta
 
 	// get all the state events ever (i.e. for all available rooms) between these two positions
-	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter)
+	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -760,10 +776,6 @@ func (d *Database) GetStateDeltas(
 	}
 
 	// Add in currently joined rooms
-	joinedRoomIDs, err := d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join)
-	if err != nil {
-		return nil, nil, err
-	}
 	for _, joinedRoomID := range joinedRoomIDs {
 		deltas = append(deltas, types.StateDelta{
 			Membership:  gomatrixserverlib.Join,
@@ -792,6 +804,22 @@ func (d *Database) GetStateDeltasForFullStateSync(
 	var succeeded bool
 	defer sqlutil.EndTransactionWithCheck(txn, &succeeded, &err)
 
+	// Look up all memberships for the user. We only care about rooms that a
+	// user has ever interacted with — joined to, kicked/banned from, left.
+	memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	allRoomIDs := make([]string, 0, len(memberships))
+	joinedRoomIDs := make([]string, 0, len(memberships))
+	for roomID, membership := range memberships {
+		allRoomIDs = append(allRoomIDs, roomID)
+		if membership == gomatrixserverlib.Join {
+			joinedRoomIDs = append(joinedRoomIDs, roomID)
+		}
+	}
+
 	// Use a reasonable initial capacity
 	deltas := make(map[string]types.StateDelta)
 
@@ -816,7 +844,7 @@ func (d *Database) GetStateDeltasForFullStateSync(
 	}
 
 	// Get all the state events ever between these two positions
-	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter)
+	stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -842,11 +870,6 @@ func (d *Database) GetStateDeltasForFullStateSync(
 		}
 	}
 
-	joinedRoomIDs, err := d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join)
-	if err != nil {
-		return nil, nil, err
-	}
-
 	// Add full states for all joined rooms
 	for _, joinedRoomID := range joinedRoomIDs {
 		s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter)
diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go
index c91ca6923..587f9d240 100644
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@@ -66,6 +66,9 @@ const DeleteRoomStateForRoomSQL = "" +
 const selectRoomIDsWithMembershipSQL = "" +
 	"SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
 
+const selectRoomIDsWithAnyMembershipSQL = "" +
+	"SELECT DISTINCT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1"
+
 const selectCurrentStateSQL = "" +
 	"SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1"
 
@@ -86,14 +89,15 @@ const selectEventsWithEventIDsSQL = "" +
 	" FROM syncapi_current_room_state WHERE event_id IN ($1)"
 
 type currentRoomStateStatements struct {
-	db                              *sql.DB
-	streamIDStatements              *streamIDStatements
-	upsertRoomStateStmt             *sql.Stmt
-	deleteRoomStateByEventIDStmt    *sql.Stmt
-	DeleteRoomStateForRoomStmt      *sql.Stmt
-	selectRoomIDsWithMembershipStmt *sql.Stmt
-	selectJoinedUsersStmt           *sql.Stmt
-	selectStateEventStmt            *sql.Stmt
+	db                                 *sql.DB
+	streamIDStatements                 *streamIDStatements
+	upsertRoomStateStmt                *sql.Stmt
+	deleteRoomStateByEventIDStmt       *sql.Stmt
+	DeleteRoomStateForRoomStmt         *sql.Stmt
+	selectRoomIDsWithMembershipStmt    *sql.Stmt
+	selectRoomIDsWithAnyMembershipStmt *sql.Stmt
+	selectJoinedUsersStmt              *sql.Stmt
+	selectStateEventStmt               *sql.Stmt
 }
 
 func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *streamIDStatements) (tables.CurrentRoomState, error) {
@@ -117,6 +121,9 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *streamIDStatements) (t
 	if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil {
 		return nil, err
 	}
+	if s.selectRoomIDsWithAnyMembershipStmt, err = db.Prepare(selectRoomIDsWithAnyMembershipSQL); err != nil {
+		return nil, err
+	}
 	if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil {
 		return nil, err
 	}
@@ -175,6 +182,31 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
 	return result, nil
 }
 
+// SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user.
+func (s *currentRoomStateStatements) SelectRoomIDsWithAnyMembership(
+	ctx context.Context,
+	txn *sql.Tx,
+	userID string,
+) (map[string]string, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithAnyMembershipStmt)
+	rows, err := stmt.QueryContext(ctx, userID)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithAnyMembership: rows.close() failed")
+
+	result := map[string]string{}
+	for rows.Next() {
+		var roomID string
+		var membership string
+		if err := rows.Scan(&roomID, &membership); err != nil {
+			return nil, err
+		}
+		result[roomID] = membership
+	}
+	return result, rows.Err()
+}
+
 // CurrentState returns all the current state events for the given room.
 func (s *currentRoomStateStatements) SelectCurrentState(
 	ctx context.Context, txn *sql.Tx, roomID string,
diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go
index 1b256f91a..b9115262e 100644
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -87,6 +88,7 @@ const selectStateInRangeSQL = "" +
 	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
 	" FROM syncapi_output_room_events" +
 	" WHERE (id > $1 AND id <= $2)" +
+	" AND room_id IN ($3)" +
 	" AND ((add_state_ids IS NOT NULL AND add_state_ids != '') OR (remove_state_ids IS NOT NULL AND remove_state_ids != ''))"
 
 // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters
@@ -155,13 +157,17 @@ func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event
 // two positions, only the most recent state is returned.
 func (s *outputRoomEventsStatements) SelectStateInRange(
 	ctx context.Context, txn *sql.Tx, r types.Range,
-	stateFilter *gomatrixserverlib.StateFilter,
+	stateFilter *gomatrixserverlib.StateFilter, roomIDs []string,
 ) (map[string]map[string]bool, map[string]types.StreamEvent, error) {
+	stmtSQL := strings.Replace(selectStateInRangeSQL, "($3)", sqlutil.QueryVariadicOffset(len(roomIDs), 2), 1)
+	inputParams := []interface{}{
+		r.Low(), r.High(),
+	}
+	for _, roomID := range roomIDs {
+		inputParams = append(inputParams, roomID)
+	}
 	stmt, params, err := prepareWithFilters(
-		s.db, txn, selectStateInRangeSQL,
-		[]interface{}{
-			r.Low(), r.High(),
-		},
+		s.db, txn, stmtSQL, inputParams,
 		stateFilter.Senders, stateFilter.NotSenders,
 		stateFilter.Types, stateFilter.NotTypes,
 		nil, stateFilter.Limit, FilterOrderAsc,
diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go
index 1ebb42651..9d1078f5f 100644
--- a/syncapi/storage/tables/interface.go
+++ b/syncapi/storage/tables/interface.go
@@ -51,7 +51,7 @@ type Peeks interface {
 }
 
 type Events interface {
-	SelectStateInRange(ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter) (map[string]map[string]bool, map[string]types.StreamEvent, error)
+	SelectStateInRange(ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, roomIDs []string) (map[string]map[string]bool, map[string]types.StreamEvent, error)
 	SelectMaxEventID(ctx context.Context, txn *sql.Tx) (id int64, err error)
 	InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error)
 	// SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high.
@@ -99,6 +99,8 @@ type CurrentRoomState interface {
 	SelectCurrentState(ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error)
 	// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
 	SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error)
+	// SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user.
+	SelectRoomIDsWithAnyMembership(ctx context.Context, txn *sql.Tx, userID string) (map[string]string, error)
 	// SelectJoinedUsers returns a map of room ID to a list of joined user IDs.
 	SelectJoinedUsers(ctx context.Context) (map[string][]string, error)
 }

From 01d71a767dbd11fd7b17b948db6d3052f577c796 Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Fri, 11 Mar 2022 17:27:12 +0000
Subject: [PATCH 7/9] Platform sanity checks (#2271)

* Platform sanity checks

* Don't yell about `root` because Docker
---
 setup/base/base.go         |  1 +
 setup/base/sanity_other.go |  8 ++++++++
 setup/base/sanity_unix.go  | 22 ++++++++++++++++++++++
 3 files changed, 31 insertions(+)
 create mode 100644 setup/base/sanity_other.go
 create mode 100644 setup/base/sanity_unix.go

diff --git a/setup/base/base.go b/setup/base/base.go
index ef3b2be29..692a77d5c 100644
--- a/setup/base/base.go
+++ b/setup/base/base.go
@@ -100,6 +100,7 @@ const (
 // The componentName is used for logging purposes, and should be a friendly name
 // of the compontent running, e.g. "SyncAPI"
 func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...BaseDendriteOptions) *BaseDendrite {
+	platformSanityChecks()
 	useHTTPAPIs := false
 	cacheMetrics := true
 	for _, opt := range options {
diff --git a/setup/base/sanity_other.go b/setup/base/sanity_other.go
new file mode 100644
index 000000000..48fe6e1f8
--- /dev/null
+++ b/setup/base/sanity_other.go
@@ -0,0 +1,8 @@
+//go:build !linux && !darwin && !netbsd && !freebsd && !openbsd && !solaris && !dragonfly && !aix
+// +build !linux,!darwin,!netbsd,!freebsd,!openbsd,!solaris,!dragonfly,!aix
+
+package base
+
+func platformSanityChecks() {
+	// Nothing to do yet.
+}
diff --git a/setup/base/sanity_unix.go b/setup/base/sanity_unix.go
new file mode 100644
index 000000000..0c1543e0b
--- /dev/null
+++ b/setup/base/sanity_unix.go
@@ -0,0 +1,22 @@
+//go:build linux || darwin || netbsd || freebsd || openbsd || solaris || dragonfly || aix
+// +build linux darwin netbsd freebsd openbsd solaris dragonfly aix
+
+package base
+
+import (
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+)
+
+func platformSanityChecks() {
+	// Dendrite needs a relatively high number of file descriptors in order
+	// to function properly, particularly when federating with lots of servers.
+	// If we run out of file descriptors, we might run into problems accessing
+	// PostgreSQL amongst other things. Complain at startup if we think the
+	// number of file descriptors is too low.
+	var rLimit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err == nil && rLimit.Cur < 65535 {
+		logrus.Warnf("IMPORTANT: Process file descriptor limit is currently %d, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues", rLimit.Cur)
+	}
+}

From a2cf1aaf4819c1548534accdc90768cc44ed6edd Mon Sep 17 00:00:00 2001
From: S7evinK <2353100+S7evinK@users.noreply.github.com>
Date: Mon, 14 Mar 2022 20:04:24 +0100
Subject: [PATCH 8/9] Fix /context with lazy_load_members (#2277)

* Add membership events to the end of the list, to ensure Sytest sees them

* Move tests to allowlist

* Append to correct list, fix logging message

* Add flakey tests to blacklist

* Remove flakey tests from whitelist
---
 syncapi/routing/context.go | 8 +++++---
 sytest-blacklist           | 4 ++--
 sytest-whitelist           | 4 +---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index d07fc0c64..2412bc2ae 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -66,7 +66,7 @@ func Context(
 	membershipRes := roomserver.QueryMembershipForUserResponse{}
 	membershipReq := roomserver.QueryMembershipForUserRequest{UserID: device.UserID, RoomID: roomID}
 	if err = rsAPI.QueryMembershipForUser(ctx, &membershipReq, &membershipRes); err != nil {
-		logrus.WithError(err).Error("unable to fo membership")
+		logrus.WithError(err).Error("unable to query membership")
 		return jsonerror.InternalServerError()
 	}
 
@@ -158,17 +158,19 @@ func applyLazyLoadMembers(filter *gomatrixserverlib.RoomEventFilter, eventsAfter
 	}
 
 	newState := []*gomatrixserverlib.HeaderedEvent{}
+	membershipEvents := []*gomatrixserverlib.HeaderedEvent{}
 	for _, event := range state {
 		if event.Type() != gomatrixserverlib.MRoomMember {
 			newState = append(newState, event)
 		} else {
 			// did the user send an event?
 			if x[event.Sender()] {
-				newState = append(newState, event)
+				membershipEvents = append(membershipEvents, event)
 			}
 		}
 	}
-	return newState
+	// Add the membershipEvents to the end of the list, to make Sytest happy
+	return append(newState, membershipEvents...)
 }
 
 func parseRoomEventFilter(req *http.Request) (*gomatrixserverlib.RoomEventFilter, error) {
diff --git a/sytest-blacklist b/sytest-blacklist
index cee2406e5..6a3b88390 100644
--- a/sytest-blacklist
+++ b/sytest-blacklist
@@ -24,7 +24,6 @@ Local device key changes get to remote servers with correct prev_id
 
 # Flakey
 Local device key changes appear in /keys/changes
-/context/ with lazy_load_members filter works
 
 # we don't support groups
 Remove group category
@@ -32,9 +31,10 @@ Remove group role
 
 # Flakey
 AS-ghosted users can use rooms themselves
-/context/ with lazy_load_members filter works
 AS-ghosted users can use rooms via AS
 Events in rooms with AS-hosted room aliases are sent to AS server
+Inviting an AS-hosted user asks the AS server
+Accesing an AS-hosted room alias asks the AS server
 
 # Flakey, need additional investigation
 Messages that notify from another user increment notification_count
diff --git a/sytest-whitelist b/sytest-whitelist
index 6c4745b32..c3a4ad92f 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -515,7 +515,6 @@ AS can create a user with inhibit_login
 AS can set avatar for ghosted users
 AS can set displayname for ghosted users
 Ghost user must register before joining room
-Inviting an AS-hosted user asks the AS server
 Can generate a openid access_token that can be exchanged for information about a user
 Invalid openid access tokens are rejected
 Requests to userinfo without access tokens are rejected
@@ -661,6 +660,5 @@ Multiple calls to /sync should not cause 500 errors
 Canonical alias can be set
 Canonical alias can include alt_aliases
 Can delete canonical alias
-Multiple calls to /sync should not cause 500 errors
 AS can make room aliases
-Accesing an AS-hosted room alias asks the AS server
+/context/ with lazy_load_members filter works

From 69268fc48e7ba6327378e8bcd8eb2a677a3a2c77 Mon Sep 17 00:00:00 2001
From: S7evinK <2353100+S7evinK@users.noreply.github.com>
Date: Tue, 15 Mar 2022 12:13:33 +0100
Subject: [PATCH 9/9] Use accountdb with provided config (#2273)

* Fix #2272

* Ensure we only allow lowercase usernames using `create-account`

Co-authored-by: kegsay <kegan@matrix.org>
---
 cmd/create-account/main.go | 40 +++++++++++++++++---------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index 307fa17e6..4d01a9f42 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -21,16 +21,15 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/crypto/bcrypt"
 	"golang.org/x/term"
 
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	userdb "github.com/matrix-org/dendrite/userapi/storage"
 )
 
 const usage = `Usage: %s
@@ -56,13 +55,14 @@ Arguments:
 `
 
 var (
-	username      = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
-	password      = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)")
-	pwdFile       = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
-	pwdStdin      = flag.Bool("passwordstdin", false, "Reads the password from stdin")
-	askPass       = flag.Bool("ask-pass", false, "Ask for the password to use")
-	isAdmin       = flag.Bool("admin", false, "Create an admin account")
-	resetPassword = flag.Bool("reset-password", false, "Resets the password for the given username")
+	username           = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
+	password           = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)")
+	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
+	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
+	askPass            = flag.Bool("ask-pass", false, "Ask for the password to use")
+	isAdmin            = flag.Bool("admin", false, "Create an admin account")
+	resetPassword      = flag.Bool("reset-password", false, "Resets the password for the given username")
+	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
 )
 
 func main() {
@@ -78,25 +78,21 @@ func main() {
 		os.Exit(1)
 	}
 
+	if !validUsernameRegex.MatchString(*username) {
+		logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
+		os.Exit(1)
+	}
+
 	pass := getPassword(password, pwdFile, pwdStdin, askPass, os.Stdin)
 
-	accountDB, err := userdb.NewDatabase(
-		&config.DatabaseOptions{
-			ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
-		},
-		cfg.Global.ServerName, bcrypt.DefaultCost,
-		cfg.UserAPI.OpenIDTokenLifetimeMS,
-		api.DefaultLoginTokenLifetime,
-	)
-	if err != nil {
-		logrus.Fatalln("Failed to connect to the database:", err.Error())
-	}
+	b := base.NewBaseDendrite(cfg, "create-account")
+	accountDB := b.CreateAccountsDB()
 
 	accType := api.AccountTypeUser
 	if *isAdmin {
 		accType = api.AccountTypeAdmin
 	}
-
+	var err error
 	if *resetPassword {
 		err = accountDB.SetPassword(context.Background(), *username, pass)
 		if err != nil {