Merge branch 'main' into dont-ignore-config-flag-for-yggdrasil-demo

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,6 +2,6 @@
 
 <!-- Please read docs/CONTRIBUTING.md before submitting your pull request -->
 
-* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/master/docs/CONTRIBUTING.md#sign-off)
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)
 
 Signed-off-by: `Your Name <your@email.example.org>`

.github/workflows/codeql-analysis.yml

@@ -2,9 +2,9 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   analyze:
@@ -14,21 +14,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ['go']
+        language: ["go"]
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
 
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
+      - run: git checkout HEAD^2
+        if: ${{ github.event_name == 'pull_request' }}
 
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+        with:
+          languages: ${{ matrix.language }}
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1

.github/workflows/tests.yml

@@ -2,7 +2,7 @@ name: Tests
 
 on:
   push:
-    branches: [ 'master' ]
+    branches: ["main"]
   pull_request:
 
 concurrency:
@@ -33,7 +33,7 @@ jobs:
           path: dendrite
 
       # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to master.
+      # doesn't exist, fallback to main.
       - name: Checkout complement
         shell: bash
         run: |
@@ -63,9 +63,9 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -p 1 -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
         env:
           COMPLEMENT_BASE_IMAGE: complement-dendrite:latest
-        working-directory: complement
+        working-directory: complement

.gitignore

@@ -23,6 +23,7 @@
 /vendor/bin
 /docker/build
 /logs
+/jetstream
 
 # Architecture specific extensions/prefixes
 *.[568vq]

CHANGES.md

@@ -1,5 +1,79 @@
 # Changelog
 
+## Dendrite 0.6.3 (2022-02-10)
+
+### Features
+
+* Initial support for `m.login.token`
+* A number of regressions from earlier v0.6.x versions should now be corrected
+
+### Fixes
+
+* Missing state is now correctly retrieved in cases where a gap in the timeline was closed but some of those events were missing state snapshots, which should help to unstick slow or broken rooms
+* Fixed a transaction issue where inserting events into the database could deadlock, which should stop rooms from getting stuck
+* Fixed a problem where rejected events could result in rolled back database transactions
+* Avoided a potential race condition on fetching latest events by using the room updater instead
+* Processing events from `/get_missing_events` will no longer result in potential recursion
+* Federation events are now correctly generated for updated self-signing keys and signed devices
+* Rejected events can now be un-rejected if they are reprocessed and all of the correct conditions are met
+* Fetching missing auth events will no longer error as long as all needed events for auth were satisfied
+* Users can now correctly forget rooms if they were not a member of the room
+
+## Dendrite 0.6.2 (2022-02-04)
+
+### Fixes
+
+* Resolves an issue where the key change consumer in the keyserver could consume extreme amounts of CPU
+
+## Dendrite 0.6.1 (2022-02-04)
+
+### Features
+
+* Roomserver inputs now take place with full transactional isolation in PostgreSQL deployments
+* Pull consumers are now used instead of push consumers when retrieving messages from NATS to better guarantee ordering and to reduce redelivery of duplicate messages
+* Further logging tweaks, particularly when joining rooms
+* Improved calculation of servers in the room, when checking for missing auth/prev events or state
+* Dendrite will now skip dead servers more quickly when federating by reducing the TCP dial timeout
+* The key change consumers have now been converted to use native NATS code rather than a wrapper
+* Go 1.16 is now the minimum supported version for Dendrite
+
+### Fixes
+
+* Local clients should now be notified correctly of invites
+* The roomserver input API now has more time to process events, particularly when fetching missing events or state, which should fix a number of errors from expired contexts
+* Fixed a panic that could happen due to a closed channel in the roomserver input API
+* Logging in with uppercase usernames from old installations is now supported again (contributed by [hoernschen](https://github.com/hoernschen))
+* Federated room joins now have more time to complete and should not fail due to expired contexts
+* Events that were sent to the roomserver along with a complete state snapshot are now persisted with the correct state, even if they were rejected or soft-failed
+
+## Dendrite 0.6.0 (2022-01-28)
+
+### Features
+
+* NATS JetStream is now used instead of Kafka and Naffka
+  * For monolith deployments, a built-in NATS Server is embedded into Dendrite or a standalone NATS Server deployment can be optionally used instead
+  * For polylith deployments, a standalone NATS Server deployment is required
+  * Requires the version 2 configuration file — please see the new `dendrite-config.yaml` sample config file
+  * Kafka and Naffka are no longer supported as of this release
+* The roomserver is now responsible for fetching missing events and state instead of the federation API
+  * Removes a number of race conditions between the federation API and roomserver, which reduces duplicate work and overall lowers CPU usage
+* The roomserver input API is now strictly ordered with support for asynchronous requests, smoothing out incoming federation significantly
+* Consolidated the federation API, federation sender and signing key server into a single component
+  * If multiple databases are used, tables for the federation sender and signing key server should be merged into the federation API database (table names have not changed)
+* Device list synchronisation is now database-backed rather than using the now-removed Kafka logs
+
+### Fixes
+
+* The code for fetching missing events and state now correctly identifies when gaps in history have been closed, so federation traffic will consume less CPU and memory than before
+* The stream position is now correctly advanced when typing notifications time out in the sync API
+* Event NIDs are now correctly returned when persisting events in the roomserver in SQLite mode
+  * The built-in SQLite was updated to version 3.37.0 as a result
+* The `/event_auth` endpoint now strictly returns the auth chain for the requested event without loading the room state, which should reduce spikes in memory usage
+* Filters are now correctly sent when using federated public room directories (contributed by [S7evinK](https://github.com/S7evinK))
+* Login usernames are now squashed to lower-case (contributed by [BernardZhao](https://github.com/BernardZhao))
+* The logs should no longer be flooded with `Failed to get server ACLs for room` warnings at startup
+* Backfilling will now attempt federation as a last resort when trying to retrieve missing events from the database fails
+
 ## Dendrite 0.5.1 (2021-11-16)
 
 ### Features

README.md

@@ -31,7 +31,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 
 ## Requirements
 
-To build Dendrite, you will need Go 1.15 or later. 
+To build Dendrite, you will need Go 1.16 or later. 
 
 For a usable federating Dendrite deployment, you will also need:
 - A domain name (or subdomain) 

appservice/api/query.go

@@ -23,7 +23,7 @@ import (
 	"errors"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -85,7 +85,7 @@ func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceQueryAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 ) (*authtypes.Profile, error) {
 	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {

appservice/appservice.go

@@ -22,6 +22,8 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/inthttp"
@@ -34,7 +36,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/sirupsen/logrus"
 )
 
 // AddInternalRoutes registers HTTP handlers for internal API calls
@@ -58,7 +59,7 @@ func NewInternalAPI(
 			},
 		},
 	}
-	js, _, _ := jetstream.Prepare(&base.Cfg.Global.JetStream)
+	js := jetstream.Prepare(&base.Cfg.Global.JetStream)
 
 	// Create a connection to the appservice postgres DB
 	appserviceDB, err := storage.NewDatabase(&base.Cfg.AppServiceAPI.Database)
@@ -121,7 +122,7 @@ func generateAppServiceAccount(
 ) error {
 	var accRes userapi.PerformAccountCreationResponse
 	err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{
-		AccountType:  userapi.AccountTypeUser,
+		AccountType:  userapi.AccountTypeAppService,
 		Localpart:    as.SenderLocalpart,
 		AppServiceID: as.ID,
 		OnConflict:   userapi.ConflictUpdate,

appservice/consumers/roomserver.go

@@ -34,7 +34,7 @@ import (
 type OutputRoomEventConsumer struct {
 	ctx          context.Context
 	jetstream    nats.JetStreamContext
-	durable      nats.SubOpt
+	durable      string
 	topic        string
 	asDB         storage.Database
 	rsAPI        api.RoomserverInternalAPI
@@ -66,37 +66,37 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	_, err := s.jetstream.Subscribe(s.topic, s.onMessage, s.durable)
-	return err
+	return jetstream.JetStreamConsumer(
+		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
+		nats.DeliverAll(), nats.ManualAck(),
+	)
 }
 
 // onMessage is called when the appservice component receives a new event from
 // the room server output log.
-func (s *OutputRoomEventConsumer) onMessage(msg *nats.Msg) {
-	jetstream.WithJetStreamMessage(msg, func(msg *nats.Msg) bool {
-		// Parse out the event JSON
-		var output api.OutputEvent
-		if err := json.Unmarshal(msg.Data, &output); err != nil {
-			// If the message was invalid, log it and move on to the next message in the stream
-			log.WithError(err).Errorf("roomserver output log: message parse failure")
-			return true
-		}
-
-		if output.Type != api.OutputTypeNewRoomEvent {
-			return true
-		}
-
-		events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
-		events = append(events, output.NewRoomEvent.AddStateEvents...)
-
-		// Send event to any relevant application services
-		if err := s.filterRoomserverEvents(context.TODO(), events); err != nil {
-			log.WithError(err).Errorf("roomserver output log: filter error")
-			return true
-		}
-
+func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	// Parse out the event JSON
+	var output api.OutputEvent
+	if err := json.Unmarshal(msg.Data, &output); err != nil {
+		// If the message was invalid, log it and move on to the next message in the stream
+		log.WithError(err).Errorf("roomserver output log: message parse failure")
 		return true
-	})
+	}
+
+	if output.Type != api.OutputTypeNewRoomEvent {
+		return true
+	}
+
+	events := []*gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event}
+	events = append(events, output.NewRoomEvent.AddStateEvents...)
+
+	// Send event to any relevant application services
+	if err := s.filterRoomserverEvents(context.TODO(), events); err != nil {
+		log.WithError(err).Errorf("roomserver output log: filter error")
+		return true
+	}
+
+	return true
 }
 
 // filterRoomserverEvents takes in events and decides whether any of them need

build.sh

@@ -7,7 +7,7 @@ if [ -d ".git" ]
 then
     export BUILD=`git rev-parse --short HEAD || ""`
     export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""`
-    if [ "$BRANCH" = master ]
+    if [ "$BRANCH" = main ]
     then
         export BRANCH=""
     fi

build/docker/DendriteJS.Dockerfile

@@ -9,9 +9,9 @@ FROM golang:1.14-alpine AS gobuild
 
 # Download and build dendrite
 WORKDIR /build
-ADD https://github.com/matrix-org/dendrite/archive/master.tar.gz /build/master.tar.gz
-RUN tar xvfz master.tar.gz
-WORKDIR /build/dendrite-master
+ADD https://github.com/matrix-org/dendrite/archive/main.tar.gz /build/main.tar.gz
+RUN tar xvfz main.tar.gz
+WORKDIR /build/dendrite-main
 RUN GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs
 
 
@@ -21,7 +21,7 @@ RUN apt-get update && apt-get -y install python
 
 # Download riot-web and libp2p repos
 WORKDIR /build
-ADD https://github.com/matrix-org/go-http-js-libp2p/archive/master.tar.gz /build/libp2p.tar.gz
+ADD https://github.com/matrix-org/go-http-js-libp2p/archive/main.tar.gz /build/libp2p.tar.gz
 RUN tar xvfz libp2p.tar.gz
 ADD https://github.com/vector-im/element-web/archive/matthew/p2p.tar.gz /build/p2p.tar.gz
 RUN tar xvfz p2p.tar.gz
@@ -31,21 +31,21 @@ WORKDIR /build/element-web-matthew-p2p
 RUN yarn install
 RUN ln -s /build/go-http-js-libp2p-master /build/element-web-matthew-p2p/node_modules/go-http-js-libp2p
 RUN (cd node_modules/go-http-js-libp2p && yarn install)
-COPY --from=gobuild /build/dendrite-master/main.wasm ./src/vector/dendrite.wasm
+COPY --from=gobuild /build/dendrite-main/main.wasm ./src/vector/dendrite.wasm
 # build it all
 RUN yarn build:p2p
 
 SHELL ["/bin/bash", "-c"]
 RUN echo $'\ 
-{ \n\
+    { \n\
     "default_server_config": { \n\
-        "m.homeserver": { \n\
-            "base_url": "https://p2p.riot.im", \n\
-            "server_name": "p2p.riot.im" \n\
-        }, \n\
-        "m.identity_server": { \n\
-            "base_url": "https://vector.im" \n\
-        } \n\
+    "m.homeserver": { \n\
+    "base_url": "https://p2p.riot.im", \n\
+    "server_name": "p2p.riot.im" \n\
+    }, \n\
+    "m.identity_server": { \n\
+    "base_url": "https://vector.im" \n\
+    } \n\
     }, \n\
     "disable_custom_urls": false, \n\
     "disable_guests": true, \n\
@@ -55,57 +55,57 @@ RUN echo $'\
     "integrations_ui_url": "https://scalar.vector.im/", \n\
     "integrations_rest_url": "https://scalar.vector.im/api", \n\
     "integrations_widgets_urls": [ \n\
-        "https://scalar.vector.im/_matrix/integrations/v1", \n\
-        "https://scalar.vector.im/api", \n\
-        "https://scalar-staging.vector.im/_matrix/integrations/v1", \n\
-        "https://scalar-staging.vector.im/api", \n\
-        "https://scalar-staging.riot.im/scalar/api" \n\
+    "https://scalar.vector.im/_matrix/integrations/v1", \n\
+    "https://scalar.vector.im/api", \n\
+    "https://scalar-staging.vector.im/_matrix/integrations/v1", \n\
+    "https://scalar-staging.vector.im/api", \n\
+    "https://scalar-staging.riot.im/scalar/api" \n\
     ], \n\
     "integrations_jitsi_widget_url": "https://scalar.vector.im/api/widgets/jitsi.html", \n\
     "bug_report_endpoint_url": "https://riot.im/bugreports/submit", \n\
     "defaultCountryCode": "GB", \n\
     "showLabsSettings": false, \n\
     "features": { \n\
-        "feature_pinning": "labs", \n\
-        "feature_custom_status": "labs", \n\
-        "feature_custom_tags": "labs", \n\
-        "feature_state_counters": "labs" \n\
+    "feature_pinning": "labs", \n\
+    "feature_custom_status": "labs", \n\
+    "feature_custom_tags": "labs", \n\
+    "feature_state_counters": "labs" \n\
     }, \n\
     "default_federate": true, \n\
     "default_theme": "light", \n\
     "roomDirectory": { \n\
-        "servers": [ \n\
-            "matrix.org" \n\
-        ] \n\
+    "servers": [ \n\
+    "matrix.org" \n\
+    ] \n\
     }, \n\
     "welcomeUserId": "", \n\
     "piwik": { \n\
-        "url": "https://piwik.riot.im/", \n\
-        "whitelistedHSUrls": ["https://matrix.org"], \n\
-        "whitelistedISUrls": ["https://vector.im", "https://matrix.org"], \n\
-        "siteId": 1 \n\
+    "url": "https://piwik.riot.im/", \n\
+    "whitelistedHSUrls": ["https://matrix.org"], \n\
+    "whitelistedISUrls": ["https://vector.im", "https://matrix.org"], \n\
+    "siteId": 1 \n\
     }, \n\
     "enable_presence_by_hs_url": { \n\
-        "https://matrix.org": false, \n\
-        "https://matrix-client.matrix.org": false \n\
+    "https://matrix.org": false, \n\
+    "https://matrix-client.matrix.org": false \n\
     }, \n\
     "settingDefaults": { \n\
-        "breadcrumbs": true \n\
+    "breadcrumbs": true \n\
     } \n\
-}' > webapp/config.json
+    }' > webapp/config.json
 
 FROM nginx
 # Add "Service-Worker-Allowed: /" header so the worker can sniff traffic on this domain rather
 # than just the path this gets hosted under. NB this newline echo syntax only works on bash.
 SHELL ["/bin/bash", "-c"]
 RUN echo $'\
-server { \n\
+    server { \n\
     listen 80; \n\
     add_header \'Service-Worker-Allowed\' \'/\'; \n\
     location / { \n\
-        root   /usr/share/nginx/html; \n\
-        index  index.html index.htm; \n\
+    root   /usr/share/nginx/html; \n\
+    index  index.html index.htm; \n\
     } \n\
-}' > /etc/nginx/conf.d/default.conf
+    }' > /etc/nginx/conf.d/default.conf
 RUN sed -i 's/}/    application\/wasm  wasm;\n}/g' /etc/nginx/mime.types
 COPY --from=jsbuild /build/element-web-matthew-p2p/webapp /usr/share/nginx/html

build/gobind-pinecone/monolith.go

@@ -281,10 +281,9 @@ func (m *DendriteMonolith) Start() {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("file:%s/%s", m.StorageDirectory, prefix))
+	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix))
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-device.db", m.StorageDirectory, prefix))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-mediaapi.db", m.CacheDirectory, prefix))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix))
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix))

build/gobind-yggdrasil/monolith.go

@@ -86,9 +86,8 @@ func (m *DendriteMonolith) Start() {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
 	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("file:%s/", m.StorageDirectory))
+	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-account.db", m.StorageDirectory))
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-device.db", m.StorageDirectory))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory))

build/scripts/Complement.Dockerfile

@@ -2,6 +2,10 @@ FROM golang:1.16-stretch as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
 
+# we will dump the binaries and config file to this location to ensure any local untracked files
+# that come from the COPY . . file don't contaminate the build
+RUN mkdir /dendrite
+
 # Utilise Docker caching when downloading dependencies, this stops us needlessly
 # downloading dependencies every time.
 COPY go.mod .
@@ -9,13 +13,19 @@ COPY go.sum .
 RUN go mod download
 
 COPY . .
-RUN go build ./cmd/dendrite-monolith-server
-RUN go build ./cmd/generate-keys
-RUN go build ./cmd/generate-config
-RUN ./generate-config --ci > dendrite.yaml
-RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
+RUN go build -o /dendrite ./cmd/dendrite-monolith-server
+RUN go build -o /dendrite ./cmd/generate-keys
+RUN go build -o /dendrite ./cmd/generate-config
+
+WORKDIR /dendrite
+RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
 EXPOSE 8008 8448
 
-CMD sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml && ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
+# At runtime, generate TLS cert based on the CA now mounted at /ca
+# At runtime, replace the SERVER_NAME with what we are told
+CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
+ ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml

build/scripts/ComplementLocal.Dockerfile

@@ -0,0 +1,53 @@
+# A local development Complement dockerfile, to be used with host mounts
+# /cache -> Contains the entire dendrite code at Dockerfile build time. Builds binaries but only keeps the generate-* ones. Pre-compilation saves time.
+# /dendrite -> Host-mounted sources
+# /runtime -> Binaries and config go here and are run at runtime
+# At runtime, dendrite is built from /dendrite and run in /runtime.
+#
+# Use these mounts to make use of this dockerfile:
+# COMPLEMENT_HOST_MOUNTS='/your/local/dendrite:/dendrite:ro;/your/go/path:/go:ro'
+FROM golang:1.16-stretch
+RUN apt-get update && apt-get install -y sqlite3
+
+WORKDIR /runtime
+
+ENV SERVER_NAME=localhost
+EXPOSE 8008 8448
+
+# This script compiles Dendrite for us.
+RUN echo '\
+#!/bin/bash -eux \n\
+if test -f "/runtime/dendrite-monolith-server"; then \n\
+    echo "Skipping compilation; binaries exist" \n\
+    exit 0 \n\
+fi \n\
+cd /dendrite \n\
+go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
+' > compile.sh && chmod +x compile.sh
+
+# This script runs Dendrite for us. Must be run in the /runtime directory.
+RUN echo '\
+#!/bin/bash -eu \n\
+./generate-keys --private-key matrix_key.pem \n\
+./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
+./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
+cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
+./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+' > run.sh && chmod +x run.sh
+
+
+WORKDIR /cache
+# Pre-download deps; we don't need to do this if the GOPATH is mounted.
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+# Build the monolith in /cache - we won't actually use this but will rely on build artifacts to speed
+# up the real compilation. Build the generate-* binaries in the true /runtime locations.
+# If the generate-* source is changed, this dockerfile needs re-running.
+COPY . .
+RUN go build ./cmd/dendrite-monolith-server && go build -o /runtime ./cmd/generate-keys && go build -o /runtime ./cmd/generate-config
+
+
+WORKDIR /runtime
+CMD /runtime/compile.sh && /runtime/run.sh

build/scripts/ComplementPostgres.Dockerfile

@@ -0,0 +1,53 @@
+FROM golang:1.16-stretch as build
+RUN apt-get update && apt-get install -y postgresql
+WORKDIR /build
+
+# No password when connecting over localhost
+RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \
+    # Bump up max conns for moar concurrency
+    sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
+
+# This entry script starts postgres, waits for it to be up then starts dendrite
+RUN echo '\
+#!/bin/bash -eu \n\
+pg_lsclusters \n\
+pg_ctlcluster 9.6 main start \n\
+ \n\
+until pg_isready \n\
+do \n\
+  echo "Waiting for postgres"; \n\
+  sleep 1; \n\
+done \n\
+' > run_postgres.sh && chmod +x run_postgres.sh
+
+# we will dump the binaries and config file to this location to ensure any local untracked files
+# that come from the COPY . . file don't contaminate the build
+RUN mkdir /dendrite
+
+# Utilise Docker caching when downloading dependencies, this stops us needlessly
+# downloading dependencies every time.
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY . .
+RUN go build -o /dendrite ./cmd/dendrite-monolith-server
+RUN go build -o /dendrite ./cmd/generate-keys
+RUN go build -o /dendrite ./cmd/generate-config
+
+WORKDIR /dendrite
+RUN ./generate-keys --private-key matrix_key.pem
+
+ENV SERVER_NAME=localhost
+EXPOSE 8008 8448
+
+
+# At runtime, generate TLS cert based on the CA now mounted at /ca
+# At runtime, replace the SERVER_NAME with what we are told
+CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
+ ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+ # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
+ sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
+ sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
+ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml

build/scripts/find-lint.sh

@@ -33,7 +33,7 @@ echo "Looking for lint..."
 # Capture exit code to ensure go.{mod,sum} is restored before exiting
 exit_code=0
 
-golangci-lint run $args || exit_code=1
+PATH="$PATH:${GOPATH:-~/go}/bin" golangci-lint run $args || exit_code=1
 
 # Restore go.{mod,sum}
 mv go.mod.bak go.mod && mv go.sum.bak go.sum

clientapi/auth/auth.go

@@ -42,6 +42,7 @@ type DeviceDatabase interface {
 type AccountDatabase interface {
 	// Look up the account matching the given localpart.
 	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
+	GetAccountByPassword(ctx context.Context, localpart, password string) (*api.Account, error)
 }
 
 // VerifyUserFromRequest authenticates the HTTP request,

clientapi/auth/authtypes/logintypes.go

@@ -10,4 +10,5 @@ const (
 	LoginTypeSharedSecret       = "org.matrix.login.shared_secret"
 	LoginTypeRecaptcha          = "m.login.recaptcha"
 	LoginTypeApplicationService = "m.login.application_service"
+	LoginTypeToken              = "m.login.token"
 )

clientapi/auth/login.go

@@ -0,0 +1,83 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+// LoginFromJSONReader performs authentication given a login request body reader and
+// some context. It returns the basic login information and a cleanup function to be
+// called after authorization has completed, with the result of the authorization.
+// If the final return value is non-nil, an error occurred and the cleanup function
+// is nil.
+func LoginFromJSONReader(ctx context.Context, r io.Reader, accountDB AccountDatabase, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	reqBytes, err := ioutil.ReadAll(r)
+	if err != nil {
+		err := &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Reading request body failed: " + err.Error()),
+		}
+		return nil, nil, err
+	}
+
+	var header struct {
+		Type string `json:"type"`
+	}
+	if err := json.Unmarshal(reqBytes, &header); err != nil {
+		err := &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Reading request body failed: " + err.Error()),
+		}
+		return nil, nil, err
+	}
+
+	var typ Type
+	switch header.Type {
+	case authtypes.LoginTypePassword:
+		typ = &LoginTypePassword{
+			GetAccountByPassword: accountDB.GetAccountByPassword,
+			Config:               cfg,
+		}
+	case authtypes.LoginTypeToken:
+		typ = &LoginTypeToken{
+			UserAPI: userAPI,
+			Config:  cfg,
+		}
+	default:
+		err := util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue("unhandled login type: " + header.Type),
+		}
+		return nil, nil, &err
+	}
+
+	return typ.LoginFromJSON(ctx, reqBytes)
+}
+
+// UserInternalAPIForLogin contains the aspects of UserAPI required for logging in.
+type UserInternalAPIForLogin interface {
+	uapi.LoginTokenInternalAPI
+}

clientapi/auth/login_test.go

@@ -0,0 +1,194 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+	"database/sql"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+func TestLoginFromJSONReader(t *testing.T) {
+	ctx := context.Background()
+
+	tsts := []struct {
+		Name string
+		Body string
+
+		WantUsername      string
+		WantDeviceID      string
+		WantDeletedTokens []string
+	}{
+		{
+			Name: "passwordWorks",
+			Body: `{
+				"type": "m.login.password",
+				"identifier": { "type": "m.id.user", "user": "alice" },
+				"password": "herpassword",
+				"device_id": "adevice"
+            }`,
+			WantUsername: "alice",
+			WantDeviceID: "adevice",
+		},
+		{
+			Name: "tokenWorks",
+			Body: `{
+				"type": "m.login.token",
+				"token": "atoken",
+				"device_id": "adevice"
+            }`,
+			WantUsername:      "@auser:example.com",
+			WantDeviceID:      "adevice",
+			WantDeletedTokens: []string{"atoken"},
+		},
+	}
+	for _, tst := range tsts {
+		t.Run(tst.Name, func(t *testing.T) {
+			var accountDB fakeAccountDB
+			var userAPI fakeUserInternalAPI
+			cfg := &config.ClientAPI{
+				Matrix: &config.Global{
+					ServerName: serverName,
+				},
+			}
+			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &accountDB, &userAPI, cfg)
+			if err != nil {
+				t.Fatalf("LoginFromJSONReader failed: %+v", err)
+			}
+			cleanup(ctx, &util.JSONResponse{Code: http.StatusOK})
+
+			if login.Username() != tst.WantUsername {
+				t.Errorf("Username: got %q, want %q", login.Username(), tst.WantUsername)
+			}
+
+			if login.DeviceID == nil {
+				if tst.WantDeviceID != "" {
+					t.Errorf("DeviceID: got %v, want %q", login.DeviceID, tst.WantDeviceID)
+				}
+			} else {
+				if *login.DeviceID != tst.WantDeviceID {
+					t.Errorf("DeviceID: got %q, want %q", *login.DeviceID, tst.WantDeviceID)
+				}
+			}
+
+			if !reflect.DeepEqual(userAPI.DeletedTokens, tst.WantDeletedTokens) {
+				t.Errorf("DeletedTokens: got %+v, want %+v", userAPI.DeletedTokens, tst.WantDeletedTokens)
+			}
+		})
+	}
+}
+
+func TestBadLoginFromJSONReader(t *testing.T) {
+	ctx := context.Background()
+
+	tsts := []struct {
+		Name string
+		Body string
+
+		WantErrCode string
+	}{
+		{Name: "empty", WantErrCode: "M_BAD_JSON"},
+		{
+			Name:        "badUnmarshal",
+			Body:        `badsyntaxJSON`,
+			WantErrCode: "M_BAD_JSON",
+		},
+		{
+			Name: "badPassword",
+			Body: `{
+				"type": "m.login.password",
+				"identifier": { "type": "m.id.user", "user": "alice" },
+				"password": "invalidpassword",
+				"device_id": "adevice"
+            }`,
+			WantErrCode: "M_FORBIDDEN",
+		},
+		{
+			Name: "badToken",
+			Body: `{
+				"type": "m.login.token",
+				"token": "invalidtoken",
+				"device_id": "adevice"
+            }`,
+			WantErrCode: "M_FORBIDDEN",
+		},
+		{
+			Name: "badType",
+			Body: `{
+				"type": "m.login.invalid",
+				"device_id": "adevice"
+            }`,
+			WantErrCode: "M_INVALID_ARGUMENT_VALUE",
+		},
+	}
+	for _, tst := range tsts {
+		t.Run(tst.Name, func(t *testing.T) {
+			var accountDB fakeAccountDB
+			var userAPI fakeUserInternalAPI
+			cfg := &config.ClientAPI{
+				Matrix: &config.Global{
+					ServerName: serverName,
+				},
+			}
+			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &accountDB, &userAPI, cfg)
+			if errRes == nil {
+				cleanup(ctx, nil)
+				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
+			} else if merr, ok := errRes.JSON.(*jsonerror.MatrixError); ok && merr.ErrCode != tst.WantErrCode {
+				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
+			}
+		})
+	}
+}
+
+type fakeAccountDB struct {
+	AccountDatabase
+}
+
+func (*fakeAccountDB) GetAccountByPassword(ctx context.Context, localpart, password string) (*uapi.Account, error) {
+	if password == "invalidpassword" {
+		return nil, sql.ErrNoRows
+	}
+
+	return &uapi.Account{}, nil
+}
+
+type fakeUserInternalAPI struct {
+	UserInternalAPIForLogin
+
+	DeletedTokens []string
+}
+
+func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, req *uapi.PerformLoginTokenDeletionRequest, res *uapi.PerformLoginTokenDeletionResponse) error {
+	ua.DeletedTokens = append(ua.DeletedTokens, req.Token)
+	return nil
+}
+
+func (*fakeUserInternalAPI) QueryLoginToken(ctx context.Context, req *uapi.QueryLoginTokenRequest, res *uapi.QueryLoginTokenResponse) error {
+	if req.Token == "invalidtoken" {
+		return nil
+	}
+
+	res.Data = &uapi.LoginTokenData{UserID: "@auser:example.com"}
+	return nil
+}

clientapi/auth/login_token.go

@@ -0,0 +1,83 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/setup/config"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+// LoginTypeToken describes how to authenticate with a login token.
+type LoginTypeToken struct {
+	UserAPI uapi.LoginTokenInternalAPI
+	Config  *config.ClientAPI
+}
+
+// Name implements Type.
+func (t *LoginTypeToken) Name() string {
+	return authtypes.LoginTypeToken
+}
+
+// LoginFromJSON implements Type. The cleanup function deletes the token from
+// the database on success.
+func (t *LoginTypeToken) LoginFromJSON(ctx context.Context, reqBytes []byte) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	var r loginTokenRequest
+	if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
+		return nil, nil, err
+	}
+
+	var res uapi.QueryLoginTokenResponse
+	if err := t.UserAPI.QueryLoginToken(ctx, &uapi.QueryLoginTokenRequest{Token: r.Token}, &res); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("UserAPI.QueryLoginToken failed")
+		jsonErr := jsonerror.InternalServerError()
+		return nil, nil, &jsonErr
+	}
+	if res.Data == nil {
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("invalid login token"),
+		}
+	}
+
+	r.Login.Identifier.Type = "m.id.user"
+	r.Login.Identifier.User = res.Data.UserID
+
+	cleanup := func(ctx context.Context, authRes *util.JSONResponse) {
+		if authRes == nil {
+			util.GetLogger(ctx).Error("No JSONResponse provided to LoginTokenType cleanup function")
+			return
+		}
+		if authRes.Code == http.StatusOK {
+			var res uapi.PerformLoginTokenDeletionResponse
+			if err := t.UserAPI.PerformLoginTokenDeletion(ctx, &uapi.PerformLoginTokenDeletionRequest{Token: r.Token}, &res); err != nil {
+				util.GetLogger(ctx).WithError(err).Error("UserAPI.PerformLoginTokenDeletion failed")
+			}
+		}
+	}
+	return &r.Login, cleanup, nil
+}
+
+// loginTokenRequest struct to hold the possible parameters from an HTTP request.
+type loginTokenRequest struct {
+	Login
+	Token string `json:"token"`
+}

clientapi/auth/password.go

@@ -16,9 +16,12 @@ package auth
 
 import (
 	"context"
+	"database/sql"
 	"net/http"
 	"strings"
 
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -40,17 +43,26 @@ type LoginTypePassword struct {
 }
 
 func (t *LoginTypePassword) Name() string {
-	return "m.login.password"
+	return authtypes.LoginTypePassword
 }
 
-func (t *LoginTypePassword) Request() interface{} {
-	return &PasswordRequest{}
+func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	var r PasswordRequest
+	if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
+		return nil, nil, err
+	}
+
+	login, err := t.Login(ctx, &r)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return login, func(context.Context, *util.JSONResponse) {}, nil
 }
 
 func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
 	r := req.(*PasswordRequest)
-	// Squash username to all lowercase letters
-	username := strings.ToLower(r.Username())
+  username := strings.ToLower(r.Username())
 	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
@@ -64,8 +76,15 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			JSON: jsonerror.InvalidUsername(err.Error()),
 		}
 	}
-	_, err = t.GetAccountByPassword(ctx, localpart, r.Password)
+	// Squash username to all lowercase letters
+	_, err = t.GetAccountByPassword(ctx, strings.ToLower(localpart), r.Password)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			_, err = t.GetAccountByPassword(ctx, localpart, r.Password)
+			if err == nil {
+				return &r.Login, nil
+			}
+		}
 		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
 		// but that would leak the existence of the user.
 		return nil, &util.JSONResponse{

clientapi/auth/user_interactive.go

@@ -32,22 +32,24 @@ import (
 type Type interface {
 	// Name returns the name of the auth type e.g `m.login.password`
 	Name() string
-	// Request returns a pointer to a new request body struct to unmarshal into.
-	Request() interface{}
 	// Login with the auth type, returning an error response on failure.
 	// Not all types support login, only m.login.password and m.login.token
 	// See https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-login
-	// `req` is guaranteed to be the type returned from Request()
 	// This function will be called when doing login and when doing 'sudo' style
 	// actions e.g deleting devices. The response must be a 401 as per:
 	// "If the homeserver decides that an attempt on a stage was unsuccessful, but the
 	// client may make a second attempt, it returns the same HTTP status 401 response as above,
 	// with the addition of the standard errcode and error fields describing the error."
-	Login(ctx context.Context, req interface{}) (login *Login, errRes *util.JSONResponse)
+	//
+	// The returned cleanup function must be non-nil on success, and will be called after
+	// authorization has been completed. Its argument is the final result of authorization.
+	LoginFromJSON(ctx context.Context, reqBytes []byte) (login *Login, cleanup LoginCleanupFunc, errRes *util.JSONResponse)
 	// TODO: Extend to support Register() flow
 	// Register(ctx context.Context, sessionID string, req interface{})
 }
 
+type LoginCleanupFunc func(context.Context, *util.JSONResponse)
+
 // LoginIdentifier represents identifier types
 // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types
 type LoginIdentifier struct {
@@ -61,11 +63,8 @@ type LoginIdentifier struct {
 
 // Login represents the shared fields used in all forms of login/sudo endpoints.
 type Login struct {
-	Type       string          `json:"type"`
-	Identifier LoginIdentifier `json:"identifier"`
-	User       string          `json:"user"`    // deprecated in favour of identifier
-	Medium     string          `json:"medium"`  // deprecated in favour of identifier
-	Address    string          `json:"address"` // deprecated in favour of identifier
+	LoginIdentifier                 // Flat fields deprecated in favour of `identifier`.
+	Identifier      LoginIdentifier `json:"identifier"`
 
 	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
 	// Thus a pointer is needed to differentiate between the two
@@ -111,12 +110,11 @@ type UserInteractive struct {
 	Sessions map[string][]string
 }
 
-func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.ClientAPI) *UserInteractive {
+func NewUserInteractive(accountDB AccountDatabase, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		GetAccountByPassword: getAccByPass,
+		GetAccountByPassword: accountDB.GetAccountByPassword,
 		Config:               cfg,
 	}
-	// TODO: Add SSO login
 	return &UserInteractive{
 		Completed: []string{},
 		Flows: []userInteractiveFlow{
@@ -236,18 +234,13 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 		}
 	}
 
-	r := loginType.Request()
-	if err := json.Unmarshal([]byte(gjson.GetBytes(bodyBytes, "auth").Raw), r); err != nil {
-		return nil, &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
-		}
+	login, cleanup, resErr := loginType.LoginFromJSON(ctx, []byte(gjson.GetBytes(bodyBytes, "auth").Raw))
+	if resErr != nil {
+		return nil, u.ResponseWithChallenge(sessionID, resErr.JSON)
 	}
-	login, resErr := loginType.Login(ctx, r)
-	if resErr == nil {
-		u.AddCompletedStage(sessionID, authType)
-		// TODO: Check if there's more stages to go and return an error
-		return login, nil
-	}
-	return nil, u.ResponseWithChallenge(sessionID, resErr.JSON)
+
+	u.AddCompletedStage(sessionID, authType)
+	cleanup(ctx, nil)
+	// TODO: Check if there's more stages to go and return an error
+	return login, nil
 }

clientapi/auth/user_interactive_test.go

@@ -24,7 +24,11 @@ var (
 	}
 )
 
-func getAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) {
+type fakeAccountDatabase struct {
+	AccountDatabase
+}
+
+func (*fakeAccountDatabase) GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) {
 	acc, ok := lookup[localpart+" "+plaintextPassword]
 	if !ok {
 		return nil, fmt.Errorf("unknown user/password")
@@ -38,7 +42,7 @@ func setup() *UserInteractive {
 			ServerName: serverName,
 		},
 	}
-	return NewUserInteractive(getAccountByPassword, cfg)
+	return NewUserInteractive(&fakeAccountDatabase{}, cfg)
 }
 
 func TestUserInteractiveChallenge(t *testing.T) {

clientapi/clientapi.go

@@ -28,7 +28,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -37,7 +37,7 @@ func AddPublicRoutes(
 	router *mux.Router,
 	synapseAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
-	accountsDB accounts.Database,
+	accountsDB userdb.Database,
 	federation *gomatrixserverlib.FederationClient,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	eduInputAPI eduServerAPI.EDUServerInputAPI,
@@ -49,7 +49,7 @@ func AddPublicRoutes(
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs,
 ) {
-	js, _, _ := jetstream.Prepare(&cfg.Matrix.JetStream)
+	js := jetstream.Prepare(&cfg.Matrix.JetStream)
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream: js,

clientapi/httputil/httputil.go

@@ -36,6 +36,10 @@ func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONRespon
 		return &resp
 	}
 
+	return UnmarshalJSON(body, iface)
+}
+
+func UnmarshalJSON(body []byte, iface interface{}) *util.JSONResponse {
 	if !utf8.Valid(body) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,

clientapi/jsonerror/jsonerror.go

@@ -149,6 +149,15 @@ func MissingParam(msg string) *MatrixError {
 	return &MatrixError{"M_MISSING_PARAM", msg}
 }
 
+// LeaveServerNoticeError is an error returned when trying to reject an invite
+// for a server notice room.
+func LeaveServerNoticeError() *MatrixError {
+	return &MatrixError{
+		ErrCode: "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM",
+		Err:     "You cannot reject this invite",
+	}
+}
+
 type IncompatibleRoomVersionError struct {
 	RoomVersion string `json:"room_version"`
 	Error       string `json:"error"`

clientapi/producers/syncapi.go

@@ -51,7 +51,7 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string
 		"user_id":   userID,
 		"room_id":   roomID,
 		"data_type": dataType,
-	}).Infof("Producing to topic '%s'", p.Topic)
+	}).Tracef("Producing to topic '%s'", p.Topic)
 
 	_, err = p.JetStream.PublishMsg(m)
 	return err

clientapi/routing/admin_whois.go

@@ -47,8 +47,8 @@ func GetAdminWhois(
 	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
 	userID string,
 ) util.JSONResponse {
-	if userID != device.UserID {
-		// TODO: Still allow if user is admin
+	allowed := device.AccountType == api.AccountTypeAdmin || userID == device.UserID
+	if !allowed {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("userID does not match the current user"),

clientapi/routing/createroom.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -30,7 +31,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	log "github.com/sirupsen/logrus"
@@ -137,36 +138,17 @@ type fledglingEvent struct {
 func CreateRoom(
 	req *http.Request, device *api.Device,
 	cfg *config.ClientAPI,
-	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
+	accountDB userdb.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	// TODO (#267): Check room ID doesn't clash with an existing one, and we
-	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
-	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
-	return createRoom(req, device, cfg, roomID, accountDB, rsAPI, asAPI)
-}
-
-// createRoom implements /createRoom
-// nolint: gocyclo
-func createRoom(
-	req *http.Request, device *api.Device,
-	cfg *config.ClientAPI, roomID string,
-	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
-	asAPI appserviceAPI.AppServiceQueryAPI,
-) util.JSONResponse {
-	logger := util.GetLogger(req.Context())
-	userID := device.UserID
 	var r createRoomRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
 		return *resErr
 	}
-	// TODO: apply rate-limit
-
 	if resErr = r.Validate(); resErr != nil {
 		return *resErr
 	}
-
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{
@@ -174,6 +156,25 @@ func createRoom(
 			JSON: jsonerror.InvalidArgumentValue(err.Error()),
 		}
 	}
+	return createRoom(req.Context(), r, device, cfg, accountDB, rsAPI, asAPI, evTime)
+}
+
+// createRoom implements /createRoom
+// nolint: gocyclo
+func createRoom(
+	ctx context.Context,
+	r createRoomRequest, device *api.Device,
+	cfg *config.ClientAPI,
+	accountDB userdb.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+	evTime time.Time,
+) util.JSONResponse {
+	// TODO (#267): Check room ID doesn't clash with an existing one, and we
+	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
+	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
+
+	logger := util.GetLogger(ctx)
+	userID := device.UserID
 
 	// Clobber keys: creator, room_version
 
@@ -200,16 +201,16 @@ func createRoom(
 		"roomVersion": roomVersion,
 	}).Info("Creating new room")
 
-	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed")
+		util.GetLogger(ctx).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed")
 		return jsonerror.InternalServerError()
 	}
 
 	createContent := map[string]interface{}{}
 	if len(r.CreationContent) > 0 {
 		if err = json.Unmarshal(r.CreationContent, &createContent); err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for creation_content failed")
+			util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for creation_content failed")
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.BadJSON("invalid create content"),
@@ -230,7 +231,7 @@ func createRoom(
 		// Merge powerLevelContentOverride fields by unmarshalling it atop the defaults
 		err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent)
 		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for power_level_content_override failed")
+			util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for power_level_content_override failed")
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.BadJSON("malformed power_level_content_override"),
@@ -319,9 +320,9 @@ func createRoom(
 		}
 
 		var aliasResp roomserverAPI.GetRoomIDForAliasResponse
-		err = rsAPI.GetRoomIDForAlias(req.Context(), &hasAliasReq, &aliasResp)
+		err = rsAPI.GetRoomIDForAlias(ctx, &hasAliasReq, &aliasResp)
 		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed")
+			util.GetLogger(ctx).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed")
 			return jsonerror.InternalServerError()
 		}
 		if aliasResp.RoomID != "" {
@@ -426,7 +427,7 @@ func createRoom(
 		}
 		err = builder.SetContent(e.Content)
 		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("builder.SetContent failed")
+			util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed")
 			return jsonerror.InternalServerError()
 		}
 		if i > 0 {
@@ -435,12 +436,12 @@ func createRoom(
 		var ev *gomatrixserverlib.Event
 		ev, err = buildEvent(&builder, &authEvents, cfg, evTime, roomVersion)
 		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("buildEvent failed")
+			util.GetLogger(ctx).WithError(err).Error("buildEvent failed")
 			return jsonerror.InternalServerError()
 		}
 
 		if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.Allowed failed")
+			util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.Allowed failed")
 			return jsonerror.InternalServerError()
 		}
 
@@ -448,7 +449,7 @@ func createRoom(
 		builtEvents = append(builtEvents, ev.Headered(roomVersion))
 		err = authEvents.AddEvent(ev)
 		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("authEvents.AddEvent failed")
+			util.GetLogger(ctx).WithError(err).Error("authEvents.AddEvent failed")
 			return jsonerror.InternalServerError()
 		}
 	}
@@ -462,8 +463,8 @@ func createRoom(
 			SendAsServer: roomserverAPI.DoNotSendToOtherServers,
 		})
 	}
-	if err = roomserverAPI.SendInputRoomEvents(req.Context(), rsAPI, inputs, false); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed")
+	if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, inputs, false); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -478,9 +479,9 @@ func createRoom(
 		}
 
 		var aliasResp roomserverAPI.SetRoomAliasResponse
-		err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
+		err = rsAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp)
 		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
+			util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 			return jsonerror.InternalServerError()
 		}
 
@@ -519,11 +520,11 @@ func createRoom(
 		for _, invitee := range r.Invite {
 			// Build the invite event.
 			inviteEvent, err := buildMembershipEvent(
-				req.Context(), invitee, "", accountDB, device, gomatrixserverlib.Invite,
+				ctx, invitee, "", accountDB, device, gomatrixserverlib.Invite,
 				roomID, true, cfg, evTime, rsAPI, asAPI,
 			)
 			if err != nil {
-				util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvent failed")
+				util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
 				continue
 			}
 			inviteStrippedState := append(
@@ -532,7 +533,7 @@ func createRoom(
 			)
 			// Send the invite event to the roomserver.
 			err = roomserverAPI.SendInvite(
-				req.Context(),
+				ctx,
 				rsAPI,
 				inviteEvent.Headered(roomVersion),
 				inviteStrippedState,   // invite room state
@@ -544,7 +545,7 @@ func createRoom(
 				return e.JSONResponse()
 			case nil:
 			default:
-				util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.SendInvite failed")
+				util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInvite failed")
 				return util.JSONResponse{
 					Code: http.StatusInternalServerError,
 					JSON: jsonerror.InternalServerError(),
@@ -556,13 +557,13 @@ func createRoom(
 	if r.Visibility == "public" {
 		// expose this room in the published room list
 		var pubRes roomserverAPI.PerformPublishResponse
-		rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
+		rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
 			RoomID:     roomID,
 			Visibility: "public",
 		}, &pubRes)
 		if pubRes.Error != nil {
 			// treat as non-fatal since the room is already made by this point
-			util.GetLogger(req.Context()).WithError(pubRes.Error).Error("failed to visibility:public")
+			util.GetLogger(ctx).WithError(pubRes.Error).Error("failed to visibility:public")
 		}
 	}
 

clientapi/routing/directory_public.go

@@ -63,7 +63,12 @@ func GetPostPublicRooms(
 	serverName := gomatrixserverlib.ServerName(request.Server)
 
 	if serverName != "" && serverName != cfg.Matrix.ServerName {
-		res, err := federation.GetPublicRooms(req.Context(), serverName, int(request.Limit), request.Since, false, "")
+		res, err := federation.GetPublicRoomsFiltered(
+			req.Context(), serverName,
+			int(request.Limit), request.Since,
+			request.Filter.SearchTerms, false,
+			"",
+		)
 		if err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("failed to get public rooms")
 			return jsonerror.InternalServerError()

clientapi/routing/joinroom.go

@@ -23,7 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -32,7 +32,7 @@ func JoinRoomByIDOrAlias(
 	req *http.Request,
 	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	roomIDOrAlias string,
 ) util.JSONResponse {
 	// Prepare to ask the roomserver to perform the room join.

clientapi/routing/key_crosssigning.go

@@ -24,7 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/util"
 )
 
@@ -36,7 +36,7 @@ type crossSigningRequest struct {
 func UploadCrossSigningDeviceKeys(
 	req *http.Request, userInteractiveAuth *auth.UserInteractive,
 	keyserverAPI api.KeyInternalAPI, device *userapi.Device,
-	accountDB accounts.Database, cfg *config.ClientAPI,
+	accountDB userdb.Database, cfg *config.ClientAPI,
 ) util.JSONResponse {
 	uploadReq := &crossSigningRequest{}
 	uploadRes := &api.PerformUploadDeviceKeysResponse{}

clientapi/routing/leaveroom.go

@@ -38,6 +38,12 @@ func LeaveRoomByID(
 
 	// Ask the roomserver to perform the leave.
 	if err := rsAPI.PerformLeave(req.Context(), &leaveReq, &leaveRes); err != nil {
+		if leaveRes.Code != 0 {
+			return util.JSONResponse{
+				Code: leaveRes.Code,
+				JSON: jsonerror.LeaveServerNoticeError(),
+			}
+		}
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.Unknown(err.Error()),

clientapi/routing/login.go

@@ -19,12 +19,11 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
-	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -55,7 +54,7 @@ func passwordLogin() flows {
 
 // Login implements GET and POST /login
 func Login(
-	req *http.Request, accountDB accounts.Database, userAPI userapi.UserInternalAPI,
+	req *http.Request, accountDB userdb.Database, userAPI userapi.UserInternalAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
@@ -65,21 +64,14 @@ func Login(
 			JSON: passwordLogin(),
 		}
 	} else if req.Method == http.MethodPost {
-		typePassword := auth.LoginTypePassword{
-			GetAccountByPassword: accountDB.GetAccountByPassword,
-			Config:               cfg,
-		}
-		r := typePassword.Request()
-		resErr := httputil.UnmarshalJSONRequest(req, r)
-		if resErr != nil {
-			return *resErr
-		}
-		login, authErr := typePassword.Login(req.Context(), r)
+		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, accountDB, userAPI, cfg)
 		if authErr != nil {
 			return *authErr
 		}
 		// make a device/access token
-		return completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login, req.RemoteAddr, req.UserAgent())
+		authErr2 := completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login, req.RemoteAddr, req.UserAgent())
+		cleanup(req.Context(), &authErr2)
+		return authErr2
 	}
 	return util.JSONResponse{
 		Code: http.StatusMethodNotAllowed,

clientapi/routing/membership.go

@@ -17,6 +17,7 @@ package routing
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 	"time"
 
@@ -29,7 +30,7 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
@@ -38,7 +39,7 @@ import (
 var errMissingUserID = errors.New("'user_id' must be supplied")
 
 func SendBan(
-	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	req *http.Request, accountDB userdb.Database, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -80,7 +81,7 @@ func SendBan(
 	return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
 }
 
-func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device,
+func sendMembership(ctx context.Context, accountDB userdb.Database, device *userapi.Device,
 	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
 	roomVer gomatrixserverlib.RoomVersion,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
@@ -124,7 +125,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us
 }
 
 func SendKick(
-	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	req *http.Request, accountDB userdb.Database, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -164,7 +165,7 @@ func SendKick(
 }
 
 func SendUnban(
-	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	req *http.Request, accountDB userdb.Database, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -199,7 +200,7 @@ func SendUnban(
 }
 
 func SendInvite(
-	req *http.Request, accountDB accounts.Database, device *userapi.Device,
+	req *http.Request, accountDB userdb.Database, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -225,27 +226,42 @@ func SendInvite(
 		}
 	}
 
+	// We already received the return value, so no need to check for an error here.
+	response, _ := sendInvite(req.Context(), accountDB, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
+	return response
+}
+
+// sendInvite sends an invitation to a user. Returns a JSONResponse and an error
+func sendInvite(
+	ctx context.Context,
+	accountDB userdb.Database,
+	device *userapi.Device,
+	roomID, userID, reason string,
+	cfg *config.ClientAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI, evTime time.Time,
+) (util.JSONResponse, error) {
 	event, err := buildMembershipEvent(
-		req.Context(), body.UserID, body.Reason, accountDB, device, "invite",
+		ctx, userID, reason, accountDB, device, "invite",
 		roomID, false, cfg, evTime, rsAPI, asAPI,
 	)
 	if err == errMissingUserID {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.BadJSON(err.Error()),
-		}
+		}, err
 	} else if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound(err.Error()),
-		}
+		}, err
 	} else if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvent failed")
-		return jsonerror.InternalServerError()
+		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
+		return jsonerror.InternalServerError(), err
 	}
 
 	err = roomserverAPI.SendInvite(
-		req.Context(), rsAPI,
+		ctx, rsAPI,
 		event,
 		nil, // ask the roomserver to draw up invite room state for us
 		cfg.Matrix.ServerName,
@@ -253,24 +269,24 @@ func SendInvite(
 	)
 	switch e := err.(type) {
 	case *roomserverAPI.PerformError:
-		return e.JSONResponse()
+		return e.JSONResponse(), err
 	case nil:
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: struct{}{},
-		}
+		}, nil
 	default:
-		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.SendInvite failed")
+		util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInvite failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: jsonerror.InternalServerError(),
-		}
+		}, err
 	}
 }
 
 func buildMembershipEvent(
 	ctx context.Context,
-	targetUserID, reason string, accountDB accounts.Database,
+	targetUserID, reason string, accountDB userdb.Database,
 	device *userapi.Device,
 	membership, roomID string, isDirect bool,
 	cfg *config.ClientAPI, evTime time.Time,
@@ -311,7 +327,7 @@ func loadProfile(
 	ctx context.Context,
 	userID string,
 	cfg *config.ClientAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*authtypes.Profile, error) {
 	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
@@ -365,7 +381,7 @@ func checkAndProcessThreepid(
 	body *threepid.MembershipRequest,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	roomID string,
 	evTime time.Time,
 ) (inviteStored bool, errRes *util.JSONResponse) {
@@ -459,13 +475,7 @@ func SendForget(
 	if membershipRes.IsInRoom {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.Forbidden("user is still a member of the room"),
-		}
-	}
-	if !membershipRes.HasBeenInRoom {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.Forbidden("user did not belong to room"),
+			JSON: jsonerror.Unknown(fmt.Sprintf("User %s is in room %s", device.UserID, roomID)),
 		}
 	}
 

clientapi/routing/password.go

@@ -9,7 +9,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -29,7 +29,7 @@ type newPasswordAuth struct {
 func Password(
 	req *http.Request,
 	userAPI api.UserInternalAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	device *api.Device,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {

clientapi/routing/peekroom.go

@@ -19,7 +19,7 @@ import (
 
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -28,7 +28,7 @@ func PeekRoomByIDOrAlias(
 	req *http.Request,
 	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	roomIDOrAlias string,
 ) util.JSONResponse {
 	// if this is a remote roomIDOrAlias, we have to ask the roomserver (or federation sender?) to
@@ -82,7 +82,7 @@ func UnpeekRoomByID(
 	req *http.Request,
 	device *api.Device,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	roomID string,
 ) util.JSONResponse {
 	unpeekReq := roomserverAPI.PerformUnpeekRequest{

clientapi/routing/profile.go

@@ -27,7 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/gomatrix"
@@ -36,7 +36,7 @@ import (
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI,
+	req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
@@ -65,7 +65,7 @@ func GetProfile(
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI,
+	req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -92,7 +92,7 @@ func GetAvatarURL(
 
 // SetAvatarURL implements PUT /profile/{userID}/avatar_url
 func SetAvatarURL(
-	req *http.Request, accountDB accounts.Database,
+	req *http.Request, accountDB userdb.Database,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -182,7 +182,7 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI,
+	req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -209,7 +209,7 @@ func GetDisplayName(
 
 // SetDisplayName implements PUT /profile/{userID}/displayname
 func SetDisplayName(
-	req *http.Request, accountDB accounts.Database,
+	req *http.Request, accountDB userdb.Database,
 	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
@@ -302,7 +302,7 @@ func SetDisplayName(
 // Returns an error when something goes wrong or specifically
 // eventutil.ErrProfileNoExists when the profile doesn't exist.
 func getProfile(
-	ctx context.Context, accountDB accounts.Database, cfg *config.ClientAPI,
+	ctx context.Context, accountDB userdb.Database, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,

clientapi/routing/register.go

@@ -32,18 +32,19 @@ import (
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/config"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/tokens"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/tokens"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 )
 
 var (
@@ -447,7 +448,7 @@ func validateApplicationService(
 func Register(
 	req *http.Request,
 	userAPI userapi.UserInternalAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var r registerRequest
@@ -531,6 +532,13 @@ func handleGuestRegistration(
 	cfg *config.ClientAPI,
 	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
+	if cfg.RegistrationDisabled || cfg.GuestsDisabled {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Guest registration is disabled"),
+		}
+	}
+
 	var res userapi.PerformAccountCreationResponse
 	err := userAPI.PerformAccountCreation(req.Context(), &userapi.PerformAccountCreationRequest{
 		AccountType: userapi.AccountTypeGuest,
@@ -701,7 +709,7 @@ func handleApplicationServiceRegistration(
 	// application service registration is entirely separate.
 	return completeRegistration(
 		req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(),
-		r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
+		r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService,
 	)
 }
 
@@ -720,7 +728,7 @@ func checkAndCompleteFlow(
 		// This flow was completed, registration can continue
 		return completeRegistration(
 			req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(),
-			r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
+			r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser,
 		)
 	}
 
@@ -745,6 +753,7 @@ func completeRegistration(
 	username, password, appserviceID, ipAddr, userAgent string,
 	inhibitLogin eventutil.WeakBoolean,
 	displayName, deviceID *string,
+	accType userapi.AccountType,
 ) util.JSONResponse {
 	if username == "" {
 		return util.JSONResponse{
@@ -759,13 +768,12 @@ func completeRegistration(
 			JSON: jsonerror.BadJSON("missing password"),
 		}
 	}
-
 	var accRes userapi.PerformAccountCreationResponse
 	err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
 		AppServiceID: appserviceID,
 		Localpart:    username,
 		Password:     password,
-		AccountType:  userapi.AccountTypeUser,
+		AccountType:  accType,
 		OnConflict:   userapi.ConflictAbort,
 	}, &accRes)
 	if err != nil {
@@ -891,7 +899,7 @@ type availableResponse struct {
 func RegisterAvailable(
 	req *http.Request,
 	cfg *config.ClientAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 ) util.JSONResponse {
 	username := req.URL.Query().Get("username")
 
@@ -963,5 +971,10 @@ func handleSharedSecretRegistration(userAPI userapi.UserInternalAPI, sr *SharedS
 		return *resErr
 	}
 	deviceID := "shared_secret_registration"
-	return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), false, &ssrr.User, &deviceID)
+
+	accType := userapi.AccountTypeUser
+	if ssrr.Admin {
+		accType = userapi.AccountTypeAdmin
+	}
+	return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), false, &ssrr.User, &deviceID, accType)
 }

clientapi/routing/routing.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 	"strings"
@@ -34,7 +35,7 @@ import (
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
@@ -51,7 +52,7 @@ func Setup(
 	eduAPI eduServerAPI.EDUServerInputAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
-	accountDB accounts.Database,
+	accountDB userdb.Database,
 	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	syncProducer *producers.SyncAPIProducer,
@@ -62,7 +63,7 @@ func Setup(
 	mscCfg *config.MSCs,
 ) {
 	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
-	userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg)
+	userInteractiveAuth := auth.NewUserInteractive(accountDB, cfg)
 
 	unstableFeatures := map[string]bool{
 		"org.matrix.e2e_cross_signing": true,
@@ -117,15 +118,66 @@ func Setup(
 		).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 	}
 
-	r0mux := publicAPIMux.PathPrefix("/r0").Subrouter()
+	// server notifications
+	if cfg.Matrix.ServerNotices.Enabled {
+		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
+		serverNotificationSender, err := getSenderDevice(context.Background(), userAPI, accountDB, cfg)
+		if err != nil {
+			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+		}
+
+		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
+			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+				// not specced, but ensure we're rate limiting requests to this endpoint
+				if r := rateLimits.Limit(req); r != nil {
+					return *r
+				}
+				vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+				if err != nil {
+					return util.ErrorResponse(err)
+				}
+				txnID := vars["txnID"]
+				return SendServerNotice(
+					req, &cfg.Matrix.ServerNotices,
+					cfg, userAPI, rsAPI, accountDB, asAPI,
+					device, serverNotificationSender,
+					&txnID, transactionsCache,
+				)
+			}),
+		).Methods(http.MethodPut, http.MethodOptions)
+
+		synapseAdminRouter.Handle("/admin/v1/send_server_notice",
+			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+				// not specced, but ensure we're rate limiting requests to this endpoint
+				if r := rateLimits.Limit(req); r != nil {
+					return *r
+				}
+				return SendServerNotice(
+					req, &cfg.Matrix.ServerNotices,
+					cfg, userAPI, rsAPI, accountDB, asAPI,
+					device, serverNotificationSender,
+					nil, transactionsCache,
+				)
+			}),
+		).Methods(http.MethodPost, http.MethodOptions)
+	}
+
+	// You can't just do PathPrefix("/(r0|v3)") because regexps only apply when inside named path variables.
+	// So make a named path variable called 'apiversion' (which we will never read in handlers) and then do
+	// (r0|v3) - BUT this is a captured group, which makes no sense because you cannot extract this group
+	// from a match (gorilla/mux exposes no way to do this) so it demands you make it a non-capturing group
+	// using ?: so the final regexp becomes what is below. We also need a trailing slash to stop 'v33333' matching.
+	// Note that 'apiversion' is chosen because it must not collide with a variable used in any of the routing!
+	v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
+
 	unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter()
 
-	r0mux.Handle("/createRoom",
+	v3mux.Handle("/createRoom",
 		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/join/{roomIDOrAlias}",
+	v3mux.Handle("/join/{roomIDOrAlias}",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -141,7 +193,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	if mscCfg.Enabled("msc2753") {
-		r0mux.Handle("/peek/{roomIDOrAlias}",
+		v3mux.Handle("/peek/{roomIDOrAlias}",
 			httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 				if r := rateLimits.Limit(req); r != nil {
 					return *r
@@ -156,12 +208,12 @@ func Setup(
 			}),
 		).Methods(http.MethodPost, http.MethodOptions)
 	}
-	r0mux.Handle("/joined_rooms",
+	v3mux.Handle("/joined_rooms",
 		httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetJoinedRooms(req, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/join",
+	v3mux.Handle("/rooms/{roomID}/join",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -175,7 +227,7 @@ func Setup(
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/leave",
+	v3mux.Handle("/rooms/{roomID}/leave",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -189,7 +241,7 @@ func Setup(
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/unpeek",
+	v3mux.Handle("/rooms/{roomID}/unpeek",
 		httputil.MakeAuthAPI("unpeek", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -200,7 +252,7 @@ func Setup(
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/ban",
+	v3mux.Handle("/rooms/{roomID}/ban",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -209,7 +261,7 @@ func Setup(
 			return SendBan(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/invite",
+	v3mux.Handle("/rooms/{roomID}/invite",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -221,7 +273,7 @@ func Setup(
 			return SendInvite(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/kick",
+	v3mux.Handle("/rooms/{roomID}/kick",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -230,7 +282,7 @@ func Setup(
 			return SendKick(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/unban",
+	v3mux.Handle("/rooms/{roomID}/unban",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -239,7 +291,7 @@ func Setup(
 			return SendUnban(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
+	v3mux.Handle("/rooms/{roomID}/send/{eventType}",
 		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -248,7 +300,7 @@ func Setup(
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
+	v3mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
 		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -259,7 +311,7 @@ func Setup(
 				nil, cfg, rsAPI, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
+	v3mux.Handle("/rooms/{roomID}/event/{eventID}",
 		httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -269,7 +321,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+	v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -277,7 +329,7 @@ func Setup(
 		return OnIncomingStateRequest(req.Context(), device, rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/aliases", httputil.MakeAuthAPI("aliases", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+	v3mux.Handle("/rooms/{roomID}/aliases", httputil.MakeAuthAPI("aliases", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -285,7 +337,7 @@ func Setup(
 		return GetAliases(req, rsAPI, device, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+	v3mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -296,7 +348,7 @@ func Setup(
 		return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], eventType, "", eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+	v3mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {
 			return util.ErrorResponse(err)
@@ -305,7 +357,7 @@ func Setup(
 		return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], vars["type"], vars["stateKey"], eventFormat)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
+	v3mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
 		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -317,7 +369,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
+	v3mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
 		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -328,21 +380,21 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
+	v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
 		if r := rateLimits.Limit(req); r != nil {
 			return *r
 		}
 		return Register(req, userAPI, accountDB, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
+	v3mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
 		if r := rateLimits.Limit(req); r != nil {
 			return *r
 		}
 		return RegisterAvailable(req, cfg, accountDB)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/directory/room/{roomAlias}",
+	v3mux.Handle("/directory/room/{roomAlias}",
 		httputil.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -352,7 +404,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/directory/room/{roomAlias}",
+	v3mux.Handle("/directory/room/{roomAlias}",
 		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -362,7 +414,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/directory/room/{roomAlias}",
+	v3mux.Handle("/directory/room/{roomAlias}",
 		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -371,7 +423,7 @@ func Setup(
 			return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
-	r0mux.Handle("/directory/list/room/{roomID}",
+	v3mux.Handle("/directory/list/room/{roomID}",
 		httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -381,7 +433,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 	// TODO: Add AS support
-	r0mux.Handle("/directory/list/room/{roomID}",
+	v3mux.Handle("/directory/list/room/{roomID}",
 		httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -390,25 +442,25 @@ func Setup(
 			return SetVisibility(req, rsAPI, device, vars["roomID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
-	r0mux.Handle("/publicRooms",
+	v3mux.Handle("/publicRooms",
 		httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
 			return GetPostPublicRooms(req, rsAPI, extRoomsProvider, federation, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/logout",
+	v3mux.Handle("/logout",
 		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return Logout(req, userAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/logout/all",
+	v3mux.Handle("/logout/all",
 		httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return LogoutAll(req, userAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
+	v3mux.Handle("/rooms/{roomID}/typing/{userID}",
 		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -420,7 +472,7 @@ func Setup(
 			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI, rsAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/redact/{eventID}",
+	v3mux.Handle("/rooms/{roomID}/redact/{eventID}",
 		httputil.MakeAuthAPI("rooms_redact", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -429,7 +481,7 @@ func Setup(
 			return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomID}/redact/{eventID}/{txnId}",
+	v3mux.Handle("/rooms/{roomID}/redact/{eventID}/{txnId}",
 		httputil.MakeAuthAPI("rooms_redact", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -439,7 +491,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
+	v3mux.Handle("/sendToDevice/{eventType}/{txnID}",
 		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -464,7 +516,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/account/whoami",
+	v3mux.Handle("/account/whoami",
 		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -473,7 +525,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/account/password",
+	v3mux.Handle("/account/password",
 		httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -482,7 +534,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/account/deactivate",
+	v3mux.Handle("/account/deactivate",
 		httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -493,7 +545,7 @@ func Setup(
 
 	// Stub endpoints required by Element
 
-	r0mux.Handle("/login",
+	v3mux.Handle("/login",
 		httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -502,14 +554,14 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/auth/{authType}/fallback/web",
+	v3mux.Handle("/auth/{authType}/fallback/web",
 		httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
 			vars := mux.Vars(req)
 			return AuthFallback(w, req, vars["authType"], cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/pushrules/",
+	v3mux.Handle("/pushrules/",
 		httputil.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse {
 			// TODO: Implement push rules API
 			res := json.RawMessage(`{
@@ -530,7 +582,7 @@ func Setup(
 
 	// Element user settings
 
-	r0mux.Handle("/profile/{userID}",
+	v3mux.Handle("/profile/{userID}",
 		httputil.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -540,7 +592,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/profile/{userID}/avatar_url",
+	v3mux.Handle("/profile/{userID}/avatar_url",
 		httputil.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -550,7 +602,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/profile/{userID}/avatar_url",
+	v3mux.Handle("/profile/{userID}/avatar_url",
 		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -565,7 +617,7 @@ func Setup(
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
 	// PUT requests, so we need to allow this method
 
-	r0mux.Handle("/profile/{userID}/displayname",
+	v3mux.Handle("/profile/{userID}/displayname",
 		httputil.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -575,7 +627,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/profile/{userID}/displayname",
+	v3mux.Handle("/profile/{userID}/displayname",
 		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -590,13 +642,13 @@ func Setup(
 	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
 	// PUT requests, so we need to allow this method
 
-	r0mux.Handle("/account/3pid",
+	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, accountDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/account/3pid",
+	v3mux.Handle("/account/3pid",
 		httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
 		}),
@@ -608,14 +660,14 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
+	v3mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
 		httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
 			return RequestEmailToken(req, accountDB, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// Element logs get flooded unless this is handled
-	r0mux.Handle("/presence/{userID}/status",
+	v3mux.Handle("/presence/{userID}/status",
 		httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -628,7 +680,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/voip/turnServer",
+	v3mux.Handle("/voip/turnServer",
 		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -637,7 +689,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/thirdparty/protocols",
+	v3mux.Handle("/thirdparty/protocols",
 		httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
 			// TODO: Return the third party protcols
 			return util.JSONResponse{
@@ -647,7 +699,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/initialSync",
+	v3mux.Handle("/rooms/{roomID}/initialSync",
 		httputil.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse {
 			// TODO: Allow people to peek into rooms.
 			return util.JSONResponse{
@@ -657,7 +709,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/user/{userID}/account_data/{type}",
+	v3mux.Handle("/user/{userID}/account_data/{type}",
 		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -667,7 +719,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
+	v3mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
 		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -677,7 +729,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/user/{userID}/account_data/{type}",
+	v3mux.Handle("/user/{userID}/account_data/{type}",
 		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -687,7 +739,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet)
 
-	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
+	v3mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
 		httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -697,7 +749,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet)
 
-	r0mux.Handle("/admin/whois/{userID}",
+	v3mux.Handle("/admin/whois/{userID}",
 		httputil.MakeAuthAPI("admin_whois", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -707,7 +759,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet)
 
-	r0mux.Handle("/user/{userID}/openid/request_token",
+	v3mux.Handle("/user/{userID}/openid/request_token",
 		httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -720,7 +772,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/user_directory/search",
+	v3mux.Handle("/user_directory/search",
 		httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -745,7 +797,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/members",
+	v3mux.Handle("/rooms/{roomID}/members",
 		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -755,7 +807,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/joined_members",
+	v3mux.Handle("/rooms/{roomID}/joined_members",
 		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -765,7 +817,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/read_markers",
+	v3mux.Handle("/rooms/{roomID}/read_markers",
 		httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -778,7 +830,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/forget",
+	v3mux.Handle("/rooms/{roomID}/forget",
 		httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -791,13 +843,13 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/devices",
+	v3mux.Handle("/devices",
 		httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, userAPI, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/devices/{deviceID}",
+	v3mux.Handle("/devices/{deviceID}",
 		httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -807,7 +859,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/devices/{deviceID}",
+	v3mux.Handle("/devices/{deviceID}",
 		httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -817,7 +869,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/devices/{deviceID}",
+	v3mux.Handle("/devices/{deviceID}",
 		httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -827,14 +879,14 @@ func Setup(
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
-	r0mux.Handle("/delete_devices",
+	v3mux.Handle("/delete_devices",
 		httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return DeleteDevices(req, userAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// Stub implementations for sytest
-	r0mux.Handle("/events",
+	v3mux.Handle("/events",
 		httputil.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
 				"chunk": []interface{}{},
@@ -844,7 +896,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/initialSync",
+	v3mux.Handle("/initialSync",
 		httputil.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
 				"end": "",
@@ -852,7 +904,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags",
+	v3mux.Handle("/user/{userId}/rooms/{roomId}/tags",
 		httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -862,7 +914,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
+	v3mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
 		httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -872,7 +924,7 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
+	v3mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
 		httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 			if err != nil {
@@ -882,7 +934,7 @@ func Setup(
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
-	r0mux.Handle("/capabilities",
+	v3mux.Handle("/capabilities",
 		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r
@@ -925,11 +977,11 @@ func Setup(
 		return CreateKeyBackupVersion(req, userAPI, device)
 	})
 
-	r0mux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
-	r0mux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
-	r0mux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut)
-	r0mux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete)
-	r0mux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions)
+	v3mux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	v3mux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	v3mux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut)
+	v3mux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete)
+	v3mux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
 	unstableMux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
@@ -1021,9 +1073,9 @@ func Setup(
 		return UploadBackupKeys(req, userAPI, device, version, &keyReq)
 	})
 
-	r0mux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut)
-	r0mux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut)
-	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut)
+	v3mux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut)
+	v3mux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut)
+	v3mux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut)
 
 	unstableMux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut)
 	unstableMux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut)
@@ -1051,9 +1103,9 @@ func Setup(
 		return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], vars["sessionID"])
 	})
 
-	r0mux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions)
-	r0mux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions)
-	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions)
+	v3mux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions)
+	v3mux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions)
+	v3mux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions)
 
 	unstableMux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions)
 	unstableMux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions)
@@ -1071,34 +1123,34 @@ func Setup(
 		return UploadCrossSigningDeviceSignatures(req, keyAPI, device)
 	})
 
-	r0mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions)
+	v3mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
+	v3mux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
 	unstableMux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions)
 
 	// Supplying a device ID is deprecated.
-	r0mux.Handle("/keys/upload/{deviceID}",
+	v3mux.Handle("/keys/upload/{deviceID}",
 		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return UploadKeys(req, keyAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/keys/upload",
+	v3mux.Handle("/keys/upload",
 		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return UploadKeys(req, keyAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/keys/query",
+	v3mux.Handle("/keys/query",
 		httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return QueryKeys(req, keyAPI, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/keys/claim",
+	v3mux.Handle("/keys/claim",
 		httputil.MakeAuthAPI("keys_claim", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return ClaimKeys(req, keyAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	r0mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
+	v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req); r != nil {
 				return *r

clientapi/routing/sendevent.go

@@ -15,10 +15,16 @@
 package routing
 
 import (
+	"context"
 	"net/http"
 	"sync"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
@@ -26,10 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 )
 
 // http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid
@@ -97,7 +99,22 @@ func SendEvent(
 	defer mutex.(*sync.Mutex).Unlock()
 
 	startedGeneratingEvent := time.Now()
-	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI)
+
+	var r map[string]interface{} // must be a JSON object
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return *resErr
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, cfg, rsAPI, evTime)
 	if resErr != nil {
 		return *resErr
 	}
@@ -153,27 +170,16 @@ func SendEvent(
 }
 
 func generateSendEvent(
-	req *http.Request,
+	ctx context.Context,
+	r map[string]interface{},
 	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
 	cfg *config.ClientAPI,
 	rsAPI api.RoomserverInternalAPI,
+	evTime time.Time,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	// parse the incoming http request
 	userID := device.UserID
-	var r map[string]interface{} // must be a JSON object
-	resErr := httputil.UnmarshalJSONRequest(req, &r)
-	if resErr != nil {
-		return nil, resErr
-	}
-
-	evTime, err := httputil.ParseTSParam(req)
-	if err != nil {
-		return nil, &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidArgumentValue(err.Error()),
-		}
-	}
 
 	// create the new event and set all the fields we can
 	builder := gomatrixserverlib.EventBuilder{
@@ -182,15 +188,15 @@ func generateSendEvent(
 		Type:     eventType,
 		StateKey: stateKey,
 	}
-	err = builder.SetContent(r)
+	err := builder.SetContent(r)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("builder.SetContent failed")
+		util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed")
 		resErr := jsonerror.InternalServerError()
 		return nil, &resErr
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := eventutil.QueryAndBuildEvent(req.Context(), &builder, cfg.Matrix, evTime, rsAPI, &queryRes)
+	e, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, &queryRes)
 	if err == eventutil.ErrRoomNoExists {
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -213,7 +219,7 @@ func generateSendEvent(
 			JSON: jsonerror.BadJSON(e.Error()),
 		}
 	} else if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed")
+		util.GetLogger(ctx).WithError(err).Error("eventutil.BuildEvent failed")
 		resErr := jsonerror.InternalServerError()
 		return nil, &resErr
 	}

clientapi/routing/sendtyping.go

@@ -20,7 +20,7 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/util"
 )
 
@@ -33,7 +33,7 @@ type typingContentJSON struct {
 // sends the typing events to client API typingProducer
 func SendTyping(
 	req *http.Request, device *userapi.Device, roomID string,
-	userID string, accountDB accounts.Database,
+	userID string, accountDB userdb.Database,
 	eduAPI api.EDUServerInputAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {

clientapi/routing/server_notices.go

@@ -0,0 +1,343 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/tokens"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/internal/transactions"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+// Unspecced server notice request
+// https://github.com/matrix-org/synapse/blob/develop/docs/admin_api/server_notices.md
+type sendServerNoticeRequest struct {
+	UserID  string `json:"user_id,omitempty"`
+	Content struct {
+		MsgType string `json:"msgtype,omitempty"`
+		Body    string `json:"body,omitempty"`
+	} `json:"content,omitempty"`
+	Type     string `json:"type,omitempty"`
+	StateKey string `json:"state_key,omitempty"`
+}
+
+// SendServerNotice sends a message to a specific user. It can only be invoked by an admin.
+func SendServerNotice(
+	req *http.Request,
+	cfgNotices *config.ServerNotices,
+	cfgClient *config.ClientAPI,
+	userAPI userapi.UserInternalAPI,
+	rsAPI api.RoomserverInternalAPI,
+	accountsDB userdb.Database,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+	device *userapi.Device,
+	senderDevice *userapi.Device,
+	txnID *string,
+	txnCache *transactions.Cache,
+) util.JSONResponse {
+	if device.AccountType != userapi.AccountTypeAdmin {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
+		}
+	}
+
+	if txnID != nil {
+		// Try to fetch response from transactionsCache
+		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
+			return *res
+		}
+	}
+
+	ctx := req.Context()
+	var r sendServerNoticeRequest
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return *resErr
+	}
+
+	// check that all required fields are set
+	if !r.valid() {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Invalid request"),
+		}
+	}
+
+	// get rooms for specified user
+	allUserRooms := []string{}
+	userRooms := api.QueryRoomsForUserResponse{}
+	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+		UserID:         r.UserID,
+		WantMembership: "join",
+	}, &userRooms); err != nil {
+		return util.ErrorResponse(err)
+	}
+	allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
+	// get invites for specified user
+	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+		UserID:         r.UserID,
+		WantMembership: "invite",
+	}, &userRooms); err != nil {
+		return util.ErrorResponse(err)
+	}
+	allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
+	// get left rooms for specified user
+	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+		UserID:         r.UserID,
+		WantMembership: "leave",
+	}, &userRooms); err != nil {
+		return util.ErrorResponse(err)
+	}
+	allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
+
+	// get rooms of the sender
+	senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName)
+	senderRooms := api.QueryRoomsForUserResponse{}
+	if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
+		UserID:         senderUserID,
+		WantMembership: "join",
+	}, &senderRooms); err != nil {
+		return util.ErrorResponse(err)
+	}
+
+	// check if we have rooms in common
+	commonRooms := []string{}
+	for _, userRoomID := range allUserRooms {
+		for _, senderRoomID := range senderRooms.RoomIDs {
+			if userRoomID == senderRoomID {
+				commonRooms = append(commonRooms, senderRoomID)
+			}
+		}
+	}
+
+	if len(commonRooms) > 1 {
+		return util.ErrorResponse(fmt.Errorf("expected to find one room, but got %d", len(commonRooms)))
+	}
+
+	var (
+		roomID      string
+		roomVersion = gomatrixserverlib.RoomVersionV6
+	)
+
+	// create a new room for the user
+	if len(commonRooms) == 0 {
+		powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID)
+		powerLevelContent.Users[r.UserID] = -10 // taken from Synapse
+		pl, err := json.Marshal(powerLevelContent)
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		createContent := map[string]interface{}{}
+		createContent["m.federate"] = false
+		cc, err := json.Marshal(createContent)
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		crReq := createRoomRequest{
+			Invite:                    []string{r.UserID},
+			Name:                      cfgNotices.RoomName,
+			Visibility:                "private",
+			Preset:                    presetPrivateChat,
+			CreationContent:           cc,
+			GuestCanJoin:              false,
+			RoomVersion:               roomVersion,
+			PowerLevelContentOverride: pl,
+		}
+
+		roomRes := createRoom(ctx, crReq, senderDevice, cfgClient, accountsDB, rsAPI, asAPI, time.Now())
+
+		switch data := roomRes.JSON.(type) {
+		case createRoomResponse:
+			roomID = data.RoomID
+
+			// tag the room, so we can later check if the user tries to reject an invite
+			serverAlertTag := gomatrix.TagContent{Tags: map[string]gomatrix.TagProperties{
+				"m.server_notice": {
+					Order: 1.0,
+				},
+			}}
+			if err = saveTagData(req, r.UserID, roomID, userAPI, serverAlertTag); err != nil {
+				util.GetLogger(ctx).WithError(err).Error("saveTagData failed")
+				return jsonerror.InternalServerError()
+			}
+
+		default:
+			// if we didn't get a createRoomResponse, we probably received an error, so return that.
+			return roomRes
+		}
+
+	} else {
+		// we've found a room in common, check the membership
+		roomID = commonRooms[0]
+		// re-invite the user
+		res, err := sendInvite(ctx, accountsDB, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
+		if err != nil {
+			return res
+		}
+	}
+
+	startedGeneratingEvent := time.Now()
+
+	request := map[string]interface{}{
+		"body":    r.Content.Body,
+		"msgtype": r.Content.MsgType,
+	}
+	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, cfgClient, rsAPI, time.Now())
+	if resErr != nil {
+		logrus.Errorf("failed to send message: %+v", resErr)
+		return *resErr
+	}
+	timeToGenerateEvent := time.Since(startedGeneratingEvent)
+
+	var txnAndSessionID *api.TransactionID
+	if txnID != nil {
+		txnAndSessionID = &api.TransactionID{
+			TransactionID: *txnID,
+			SessionID:     device.SessionID,
+		}
+	}
+
+	// pass the new event to the roomserver and receive the correct event ID
+	// event ID in case of duplicate transaction is discarded
+	startedSubmittingEvent := time.Now()
+	if err := api.SendEvents(
+		ctx, rsAPI,
+		api.KindNew,
+		[]*gomatrixserverlib.HeaderedEvent{
+			e.Headered(roomVersion),
+		},
+		cfgClient.Matrix.ServerName,
+		cfgClient.Matrix.ServerName,
+		txnAndSessionID,
+		false,
+	); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
+		return jsonerror.InternalServerError()
+	}
+	util.GetLogger(ctx).WithFields(logrus.Fields{
+		"event_id":     e.EventID(),
+		"room_id":      roomID,
+		"room_version": roomVersion,
+	}).Info("Sent event to roomserver")
+	timeToSubmitEvent := time.Since(startedSubmittingEvent)
+
+	res := util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: sendEventResponse{e.EventID()},
+	}
+	// Add response to transactionsCache
+	if txnID != nil {
+		txnCache.AddTransaction(device.AccessToken, *txnID, &res)
+	}
+
+	// Take a note of how long it took to generate the event vs submit
+	// it to the roomserver.
+	sendEventDuration.With(prometheus.Labels{"action": "build"}).Observe(float64(timeToGenerateEvent.Milliseconds()))
+	sendEventDuration.With(prometheus.Labels{"action": "submit"}).Observe(float64(timeToSubmitEvent.Milliseconds()))
+
+	return res
+}
+
+func (r sendServerNoticeRequest) valid() (ok bool) {
+	if r.UserID == "" {
+		return false
+	}
+	if r.Content.MsgType == "" || r.Content.Body == "" {
+		return false
+	}
+	return true
+}
+
+// getSenderDevice creates a user account to be used when sending server notices.
+// It returns an userapi.Device, which is used for building the event
+func getSenderDevice(
+	ctx context.Context,
+	userAPI userapi.UserInternalAPI,
+	accountDB userdb.Database,
+	cfg *config.ClientAPI,
+) (*userapi.Device, error) {
+	var accRes userapi.PerformAccountCreationResponse
+	// create account if it doesn't exist
+	err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
+		AccountType: userapi.AccountTypeUser,
+		Localpart:   cfg.Matrix.ServerNotices.LocalPart,
+		OnConflict:  userapi.ConflictUpdate,
+	}, &accRes)
+	if err != nil {
+		return nil, err
+	}
+
+	// set the avatarurl for the user
+	if err = accountDB.SetAvatarURL(ctx, cfg.Matrix.ServerNotices.LocalPart, cfg.Matrix.ServerNotices.AvatarURL); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("accountDB.SetAvatarURL failed")
+		return nil, err
+	}
+
+	// Check if we got existing devices
+	deviceRes := &userapi.QueryDevicesResponse{}
+	err = userAPI.QueryDevices(ctx, &userapi.QueryDevicesRequest{
+		UserID: accRes.Account.UserID,
+	}, deviceRes)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(deviceRes.Devices) > 0 {
+		return &deviceRes.Devices[0], nil
+	}
+
+	// create an AccessToken
+	token, err := tokens.GenerateLoginToken(tokens.TokenOptions{
+		ServerPrivateKey: cfg.Matrix.PrivateKey.Seed(),
+		ServerName:       string(cfg.Matrix.ServerName),
+		UserID:           accRes.Account.UserID,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// create a new device, if we didn't find any
+	var devRes userapi.PerformDeviceCreationResponse
+	err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
+		Localpart:          cfg.Matrix.ServerNotices.LocalPart,
+		DeviceDisplayName:  &cfg.Matrix.ServerNotices.LocalPart,
+		AccessToken:        token,
+		NoDeviceListUpdate: true,
+	}, &devRes)
+
+	if err != nil {
+		return nil, err
+	}
+	return devRes.Device, nil
+}

clientapi/routing/server_notices_test.go

@@ -0,0 +1,83 @@
+package routing
+
+import (
+	"testing"
+)
+
+func Test_sendServerNoticeRequest_validate(t *testing.T) {
+	type fields struct {
+		UserID  string `json:"user_id,omitempty"`
+		Content struct {
+			MsgType string `json:"msgtype,omitempty"`
+			Body    string `json:"body,omitempty"`
+		} `json:"content,omitempty"`
+		Type     string `json:"type,omitempty"`
+		StateKey string `json:"state_key,omitempty"`
+	}
+
+	content := struct {
+		MsgType string `json:"msgtype,omitempty"`
+		Body    string `json:"body,omitempty"`
+	}{
+		MsgType: "m.text",
+		Body:    "Hello world!",
+	}
+
+	tests := []struct {
+		name   string
+		fields fields
+		wantOk bool
+	}{
+		{
+			name:   "empty request",
+			fields: fields{},
+		},
+		{
+			name: "msgtype empty",
+			fields: fields{
+				UserID: "@alice:localhost",
+				Content: struct {
+					MsgType string `json:"msgtype,omitempty"`
+					Body    string `json:"body,omitempty"`
+				}{
+					Body: "Hello world!",
+				},
+			},
+		},
+		{
+			name: "msg body empty",
+			fields: fields{
+				UserID: "@alice:localhost",
+			},
+		},
+		{
+			name: "statekey empty",
+			fields: fields{
+				UserID:  "@alice:localhost",
+				Content: content,
+			},
+			wantOk: true,
+		},
+		{
+			name: "type empty",
+			fields: fields{
+				UserID:  "@alice:localhost",
+				Content: content,
+			},
+			wantOk: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := sendServerNoticeRequest{
+				UserID:   tt.fields.UserID,
+				Content:  tt.fields.Content,
+				Type:     tt.fields.Type,
+				StateKey: tt.fields.StateKey,
+			}
+			if gotOk := r.valid(); gotOk != tt.wantOk {
+				t.Errorf("valid() = %v, want %v", gotOk, tt.wantOk)
+			}
+		})
+	}
+}

clientapi/routing/threepid.go

@@ -23,7 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -40,7 +40,7 @@ type threePIDsResponse struct {
 // RequestEmailToken implements:
 //     POST /account/3pid/email/requestToken
 //     POST /register/email/requestToken
-func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI) util.JSONResponse {
+func RequestEmailToken(req *http.Request, accountDB userdb.Database, cfg *config.ClientAPI) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
@@ -61,7 +61,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.MatrixError{
 				ErrCode: "M_THREEPID_IN_USE",
-				Err:     accounts.Err3PIDInUse.Error(),
+				Err:     userdb.Err3PIDInUse.Error(),
 			},
 		}
 	}
@@ -85,7 +85,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf
 
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
-	req *http.Request, accountDB accounts.Database, device *api.Device,
+	req *http.Request, accountDB userdb.Database, device *api.Device,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
@@ -149,7 +149,7 @@ func CheckAndSave3PIDAssociation(
 
 // GetAssociated3PIDs implements GET /account/3pid
 func GetAssociated3PIDs(
-	req *http.Request, accountDB accounts.Database, device *api.Device,
+	req *http.Request, accountDB userdb.Database, device *api.Device,
 ) util.JSONResponse {
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
@@ -170,7 +170,7 @@ func GetAssociated3PIDs(
 }
 
 // Forget3PID implements POST /account/3pid/delete
-func Forget3PID(req *http.Request, accountDB accounts.Database) util.JSONResponse {
+func Forget3PID(req *http.Request, accountDB userdb.Database) util.JSONResponse {
 	var body authtypes.ThreePID
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr

clientapi/routing/whoami.go

@@ -21,7 +21,9 @@ import (
 
 // whoamiResponse represents an response for a `whoami` request
 type whoamiResponse struct {
-	UserID string `json:"user_id"`
+	UserID   string `json:"user_id"`
+	DeviceID string `json:"device_id"`
+	IsGuest  bool   `json:"is_guest"`
 }
 
 // Whoami implements `/account/whoami` which enables client to query their account user id.
@@ -29,6 +31,10 @@ type whoamiResponse struct {
 func Whoami(req *http.Request, device *api.Device) util.JSONResponse {
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: whoamiResponse{UserID: device.UserID},
+		JSON: whoamiResponse{
+			UserID:   device.UserID,
+			DeviceID: device.ID,
+			IsGuest:  device.AccountType == api.AccountTypeGuest,
+		},
 	}
 }

clientapi/threepid/invites.go

@@ -29,7 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -87,7 +87,7 @@ var (
 func CheckAndProcessInvite(
 	ctx context.Context,
 	device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI,
-	rsAPI api.RoomserverInternalAPI, db accounts.Database,
+	rsAPI api.RoomserverInternalAPI, db userdb.Database,
 	roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
@@ -137,7 +137,7 @@ func CheckAndProcessInvite(
 // Returns an error if a check or a request failed.
 func queryIDServer(
 	ctx context.Context,
-	db accounts.Database, cfg *config.ClientAPI, device *userapi.Device,
+	db userdb.Database, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
 	if err = isTrusted(body.IDServer, cfg); err != nil {
@@ -206,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
 // Returns an error if the request failed to send or if the response couldn't be parsed.
 func queryIDServerStoreInvite(
 	ctx context.Context,
-	db accounts.Database, cfg *config.ClientAPI, device *userapi.Device,
+	db userdb.Database, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name

cmd/create-account/main.go

@@ -23,12 +23,14 @@ import (
 	"os"
 	"strings"
 
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/bcrypt"
 	"golang.org/x/term"
+
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi/api"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 )
 
 const usage = `Usage: %s
@@ -57,6 +59,7 @@ var (
 	pwdFile  = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
 	pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin")
 	askPass  = flag.Bool("ask-pass", false, "Ask for the password to use")
+	isAdmin  = flag.Bool("admin", false, "Create an admin account")
 )
 
 func main() {
@@ -74,14 +77,23 @@ func main() {
 
 	pass := getPassword(password, pwdFile, pwdStdin, askPass, os.Stdin)
 
-	accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{
-		ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
-	}, cfg.Global.ServerName, bcrypt.DefaultCost, cfg.UserAPI.OpenIDTokenLifetimeMS)
+	accountDB, err := userdb.NewDatabase(
+		&config.DatabaseOptions{
+			ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
+		},
+		cfg.Global.ServerName, bcrypt.DefaultCost,
+		cfg.UserAPI.OpenIDTokenLifetimeMS,
+		api.DefaultLoginTokenLifetime,
+	)
 	if err != nil {
 		logrus.Fatalln("Failed to connect to the database:", err.Error())
 	}
 
-	_, err = accountDB.CreateAccount(context.Background(), *username, pass, "")
+	accType := api.AccountTypeUser
+	if *isAdmin {
+		accType = api.AccountTypeAdmin
+	}
+	_, err = accountDB.CreateAccount(context.Background(), *username, pass, "", accType)
 	if err != nil {
 		logrus.Fatalln("Failed to create the account:", err.Error())
 	}

cmd/dendrite-demo-libp2p/main.go

@@ -126,7 +126,6 @@ func main() {
 	cfg.FederationAPI.FederationMaxRetries = 6
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))

cmd/dendrite-demo-pinecone/main.go

@@ -160,7 +160,6 @@ func main() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))

cmd/dendrite-demo-yggdrasil/README.md

@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.15 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.16 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run:
 

cmd/dendrite-demo-yggdrasil/main.go

@@ -64,14 +64,6 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	/*
-		ygg.SetMulticastEnabled(true)
-		if instancePeer != nil && *instancePeer != "" {
-			if err = ygg.SetStaticPeer(*instancePeer); err != nil {
-				logrus.WithError(err).Error("Failed to set static peer")
-			}
-		}
-	*/
 
 	// iterate through the cli args and check if the config flag was set
 	configFlagSet := false
@@ -89,16 +81,14 @@ func main() {
 		cfg = setup.ParseFlags(true)
 	} else {
 		cfg.Defaults(true)
-  	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
+		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-		cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
 		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
 		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
 		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
 		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
 		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
 		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-		cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
 		cfg.MSCs.MSCs = []string{"msc2836"}
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
 		if err = cfg.Derive(); err != nil {

cmd/dendrite-monolith-server/main.go

@@ -132,6 +132,7 @@ func main() {
 	// dependency. Other components also need updating after their dependencies are up.
 	rsImpl.SetFederationAPI(fsAPI, keyRing)
 	rsImpl.SetAppserviceAPI(asAPI)
+	rsImpl.SetUserAPI(userAPI)
 	keyImpl.SetUserAPI(userAPI)
 
 	eduInputAPI := eduserver.NewInternalAPI(

cmd/dendritejs-pinecone/main.go

@@ -164,7 +164,6 @@ func startup() {
 	cfg.Defaults(true)
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
 	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
-	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
 	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
 	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
 	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"

cmd/dendritejs/main.go

@@ -167,7 +167,6 @@ func main() {
 	cfg.Defaults(true)
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
 	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
-	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
 	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
 	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
 	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"

cmd/generate-config/main.go

@@ -32,7 +32,6 @@ func main() {
 		cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI)
 		cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI)
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI)
-		cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(*dbURI)
 	}
 	cfg.Global.TrustedIDServers = []string{
 		"matrix.org",
@@ -83,7 +82,7 @@ func main() {
 	if *defaultsForCI {
 		cfg.AppServiceAPI.DisableTLSValidation = true
 		cfg.ClientAPI.RateLimiting.Enabled = false
-		cfg.FederationAPI.DisableTLSValidation = true
+		cfg.FederationAPI.DisableTLSValidation = false
 		// don't hit matrix.org when running tests!!!
 		cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
 		cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"}
@@ -91,6 +90,7 @@ func main() {
 		cfg.Logging[0].Type = "std"
 		cfg.UserAPI.BCryptCost = bcrypt.MinCost
 		cfg.Global.JetStream.InMemory = true
+		cfg.ClientAPI.RegistrationSharedSecret = "complement"
 	}
 
 	j, err := yaml.Marshal(cfg)

cmd/generate-keys/main.go

@@ -32,9 +32,12 @@ Arguments:
 `
 
 var (
-	tlsCertFile    = flag.String("tls-cert", "", "An X509 certificate file to generate for use for TLS")
-	tlsKeyFile     = flag.String("tls-key", "", "An RSA private key file to generate for use for TLS")
-	privateKeyFile = flag.String("private-key", "", "An Ed25519 private key to generate for use for object signing")
+	tlsCertFile       = flag.String("tls-cert", "", "An X509 certificate file to generate for use for TLS")
+	tlsKeyFile        = flag.String("tls-key", "", "An RSA private key file to generate for use for TLS")
+	privateKeyFile    = flag.String("private-key", "", "An Ed25519 private key to generate for use for object signing")
+	authorityCertFile = flag.String("tls-authority-cert", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
+	authorityKeyFile  = flag.String("tls-authority-key", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
+	serverName        = flag.String("server", "", "Optional: Create TLS certificate/keys with this domain name set. Useful for integration testing.")
 )
 
 func main() {
@@ -54,8 +57,15 @@ func main() {
 		if *tlsCertFile == "" || *tlsKeyFile == "" {
 			log.Fatal("Zero or both of --tls-key and --tls-cert must be supplied")
 		}
-		if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile); err != nil {
-			panic(err)
+		if *authorityCertFile == "" && *authorityKeyFile == "" {
+			if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile); err != nil {
+				panic(err)
+			}
+		} else {
+			// generate the TLS cert/key based on the authority given.
+			if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile); err != nil {
+				panic(err)
+			}
 		}
 		fmt.Printf("Created TLS cert file:    %s\n", *tlsCertFile)
 		fmt.Printf("Created TLS key file:     %s\n", *tlsKeyFile)

cmd/goose/main.go

@@ -8,12 +8,11 @@ import (
 	"log"
 	"os"
 
-	pgaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas"
-	slaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas"
-	pgdevices "github.com/matrix-org/dendrite/userapi/storage/devices/postgres/deltas"
-	sldevices "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas"
 	"github.com/pressly/goose"
 
+	pgusers "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
+	slusers "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
+
 	_ "github.com/lib/pq"
 	_ "github.com/mattn/go-sqlite3"
 )
@@ -26,8 +25,7 @@ const (
 	RoomServer       = "roomserver"
 	SigningKeyServer = "signingkeyserver"
 	SyncAPI          = "syncapi"
-	UserAPIAccounts  = "userapi_accounts"
-	UserAPIDevices   = "userapi_devices"
+	UserAPI          = "userapi"
 )
 
 var (
@@ -35,7 +33,7 @@ var (
 	flags     = flag.NewFlagSet("goose", flag.ExitOnError)
 	component = flags.String("component", "", "dendrite component name")
 	knownDBs  = []string{
-		AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPIAccounts, UserAPIDevices,
+		AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPI,
 	}
 )
 
@@ -143,18 +141,14 @@ Commands:
 
 func loadSQLiteDeltas(component string) {
 	switch component {
-	case UserAPIAccounts:
-		slaccounts.LoadFromGoose()
-	case UserAPIDevices:
-		sldevices.LoadFromGoose()
+	case UserAPI:
+		slusers.LoadFromGoose()
 	}
 }
 
 func loadPostgresDeltas(component string) {
 	switch component {
-	case UserAPIAccounts:
-		pgaccounts.LoadFromGoose()
-	case UserAPIDevices:
-		pgdevices.LoadFromGoose()
+	case UserAPI:
+		pgusers.LoadFromGoose()
 	}
 }

dendrite-config.yaml

@@ -68,6 +68,18 @@ global:
   # to other servers and the federation API will not be exposed.
   disable_federation: false
 
+  # Server notices allows server admins to send messages to all users.
+  server_notices:
+    enabled: false
+    # The server localpart to be used when sending notices, ensure this is not yet taken
+    local_part: "_server"
+    # The displayname to be used when sending notices
+    display_name: "Server alerts"
+    # The mxid of the avatar to use
+    avatar_url: ""
+    # The roomname to be used when creating messages
+    room_name: "Server Alerts"
+
   # Configuration for NATS JetStream
   jetstream:
     # A list of NATS Server addresses to connect to. If none are specified, an
@@ -142,6 +154,10 @@ client_api:
   # using the registration shared secret below.
   registration_disabled: false
 
+  # Prevents new guest accounts from being created. Guest registration is also
+  # disabled implicitly by setting 'registration_disabled' above.
+  guests_disabled: true
+
   # If set, allows registration by anyone who knows the shared secret, regardless of
   # whether registration is otherwise disabled.
   registration_shared_secret: ""
@@ -204,13 +220,6 @@ federation_api:
   # enable this option in production as it presents a security risk!
   disable_tls_validation: false
 
-  # Use the following proxy server for outbound federation traffic.
-  proxy_outbound:
-    enabled: false
-    protocol: http
-    host: localhost
-    port: 8080
-
   # Perspective keyservers to use as a backup when direct key fetches fail. This may
   # be required to satisfy key requests for servers that are no longer online when
   # joining some rooms.

docs/CONTRIBUTING.md

@@ -37,7 +37,7 @@ If a job fails, click the "details" button and you should be taken to the job's
 logs.
 
 ![Click the details button on the failing build
-step](https://raw.githubusercontent.com/matrix-org/dendrite/master/docs/images/details-button-location.jpg)
+step](https://raw.githubusercontent.com/matrix-org/dendrite/main/docs/images/details-button-location.jpg)
 
 Scroll down to the failing step and you should see some log output. Scan the
 logs until you find what it's complaining about, fix it, submit a new commit,
@@ -57,7 +57,7 @@ significant amount of CPU and RAM.
 
 Once the code builds, run [Sytest](https://github.com/matrix-org/sytest)
 according to the guide in
-[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-a-sytest-docker-image)
+[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/main/docs/sytest.md#using-a-sytest-docker-image)
 so you can see whether something is being broken and whether there are newly
 passing tests.
 
@@ -94,4 +94,4 @@ For more general questions please use
 
 We ask that everyone who contributes to the project signs off their
 contributions, in accordance with the
-[DCO](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst#sign-off).
+[DCO](https://github.com/matrix-org/matrix-doc/blob/main/CONTRIBUTING.rst#sign-off).

docs/INSTALL.md

@@ -27,7 +27,7 @@ use in production environments just yet!
 
 Dendrite requires:
 
-* Go 1.15 or higher
+* Go 1.16 or higher
 * PostgreSQL 12 or higher (if using PostgreSQL databases, not needed for SQLite)
 
 If you want to run a polylith deployment, you also need:

docs/p2p.md

@@ -6,7 +6,7 @@ These are the instructions for setting up P2P Dendrite, current as of May 2020.
 
 #### Build
 
-- The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
+- The `main` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
 - Build it and copy assets to riot-web.
 ```
 $ ./build-dendritejs.sh

eduserver/api/input.go

@@ -100,10 +100,4 @@ type EDUServerInputAPI interface {
 		request *InputReceiptEventRequest,
 		response *InputReceiptEventResponse,
 	) error
-
-	InputCrossSigningKeyUpdate(
-		ctx context.Context,
-		request *InputCrossSigningKeyUpdateRequest,
-		response *InputCrossSigningKeyUpdateResponse,
-	) error
 }

eduserver/eduserver.go

@@ -42,7 +42,7 @@ func NewInternalAPI(
 ) api.EDUServerInputAPI {
 	cfg := &base.Cfg.EDUServer
 
-	js, _, _ := jetstream.Prepare(&cfg.Matrix.JetStream)
+	js := jetstream.Prepare(&cfg.Matrix.JetStream)
 
 	return &input.EDUServerInputAPI{
 		Cache:                        eduCache,
@@ -51,7 +51,6 @@ func NewInternalAPI(
 		OutputTypingEventTopic:       cfg.Matrix.JetStream.TopicFor(jetstream.OutputTypingEvent),
 		OutputSendToDeviceEventTopic: cfg.Matrix.JetStream.TopicFor(jetstream.OutputSendToDeviceEvent),
 		OutputReceiptEventTopic:      cfg.Matrix.JetStream.TopicFor(jetstream.OutputReceiptEvent),
-		OutputKeyChangeEventTopic:    cfg.Matrix.JetStream.TopicFor(jetstream.OutputKeyChangeEvent),
 		ServerName:                   cfg.Matrix.ServerName,
 	}
 }

eduserver/input/input.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/eduserver/cache"
-	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
@@ -40,8 +39,6 @@ type EDUServerInputAPI struct {
 	OutputSendToDeviceEventTopic string
 	// The kafka topic to output new receipt events to
 	OutputReceiptEventTopic string
-	// The kafka topic to output new key change events to
-	OutputKeyChangeEventTopic string
 	// kafka producer
 	JetStream nats.JetStreamContext
 	// Internal user query API
@@ -80,34 +77,6 @@ func (t *EDUServerInputAPI) InputSendToDeviceEvent(
 	return t.sendToDeviceEvent(ise)
 }
 
-// InputCrossSigningKeyUpdate implements api.EDUServerInputAPI
-func (t *EDUServerInputAPI) InputCrossSigningKeyUpdate(
-	ctx context.Context,
-	request *api.InputCrossSigningKeyUpdateRequest,
-	response *api.InputCrossSigningKeyUpdateResponse,
-) error {
-	eventJSON, err := json.Marshal(&keyapi.DeviceMessage{
-		Type: keyapi.TypeCrossSigningUpdate,
-		OutputCrossSigningKeyUpdate: &api.OutputCrossSigningKeyUpdate{
-			CrossSigningKeyUpdate: request.CrossSigningKeyUpdate,
-		},
-	})
-	if err != nil {
-		return err
-	}
-
-	logrus.WithFields(logrus.Fields{
-		"user_id": request.UserID,
-	}).Infof("Producing to topic '%s'", t.OutputKeyChangeEventTopic)
-
-	_, err = t.JetStream.PublishMsg(&nats.Msg{
-		Subject: t.OutputKeyChangeEventTopic,
-		Header:  nats.Header{},
-		Data:    eventJSON,
-	})
-	return err
-}
-
 func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
 	ev := &api.TypingEvent{
 		Type:   gomatrixserverlib.MTyping,
@@ -134,7 +103,7 @@ func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error {
 		"room_id": ite.RoomID,
 		"user_id": ite.UserID,
 		"typing":  ite.Typing,
-	}).Infof("Producing to topic '%s'", t.OutputTypingEventTopic)
+	}).Tracef("Producing to topic '%s'", t.OutputTypingEventTopic)
 
 	_, err = t.JetStream.PublishMsg(&nats.Msg{
 		Subject: t.OutputTypingEventTopic,
@@ -175,7 +144,7 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e
 		"user_id":     ise.UserID,
 		"num_devices": len(devices),
 		"type":        ise.Type,
-	}).Infof("Producing to topic '%s'", t.OutputSendToDeviceEventTopic)
+	}).Tracef("Producing to topic '%s'", t.OutputSendToDeviceEventTopic)
 	for _, device := range devices {
 		ote := &api.OutputSendToDeviceEvent{
 			UserID:            ise.UserID,
@@ -208,7 +177,7 @@ func (t *EDUServerInputAPI) InputReceiptEvent(
 	request *api.InputReceiptEventRequest,
 	response *api.InputReceiptEventResponse,
 ) error {
-	logrus.WithFields(logrus.Fields{}).Infof("Producing to topic '%s'", t.OutputReceiptEventTopic)
+	logrus.WithFields(logrus.Fields{}).Tracef("Producing to topic '%s'", t.OutputReceiptEventTopic)
 	output := &api.OutputReceiptEvent{
 		UserID:    request.InputReceiptEvent.UserID,
 		RoomID:    request.InputReceiptEvent.RoomID,

eduserver/inthttp/client.go

@@ -12,10 +12,9 @@ import (
 
 // HTTP paths for the internal HTTP APIs
 const (
-	EDUServerInputTypingEventPath           = "/eduserver/input"
-	EDUServerInputSendToDeviceEventPath     = "/eduserver/sendToDevice"
-	EDUServerInputReceiptEventPath          = "/eduserver/receipt"
-	EDUServerInputCrossSigningKeyUpdatePath = "/eduserver/crossSigningKeyUpdate"
+	EDUServerInputTypingEventPath       = "/eduserver/input"
+	EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice"
+	EDUServerInputReceiptEventPath      = "/eduserver/receipt"
 )
 
 // NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API.
@@ -69,16 +68,3 @@ func (h *httpEDUServerInputAPI) InputReceiptEvent(
 	apiURL := h.eduServerURL + EDUServerInputReceiptEventPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
-
-// InputCrossSigningKeyUpdate implements EDUServerInputAPI
-func (h *httpEDUServerInputAPI) InputCrossSigningKeyUpdate(
-	ctx context.Context,
-	request *api.InputCrossSigningKeyUpdateRequest,
-	response *api.InputCrossSigningKeyUpdateResponse,
-) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputCrossSigningKeyUpdate")
-	defer span.Finish()
-
-	apiURL := h.eduServerURL + EDUServerInputCrossSigningKeyUpdatePath
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-}

eduserver/inthttp/server.go

@@ -51,17 +51,4 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
-	internalAPIMux.Handle(EDUServerInputCrossSigningKeyUpdatePath,
-		httputil.MakeInternalAPI("inputCrossSigningKeyUpdate", func(req *http.Request) util.JSONResponse {
-			var request api.InputCrossSigningKeyUpdateRequest
-			var response api.InputCrossSigningKeyUpdateResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := t.InputCrossSigningKeyUpdate(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
 }

federationapi/consumers/eduserver.go

@@ -34,7 +34,7 @@ import (
 type OutputEDUConsumer struct {
 	ctx               context.Context
 	jetstream         nats.JetStreamContext
-	durable           nats.SubOpt
+	durable           string
 	db                storage.Database
 	queues            *queue.OutgoingQueues
 	ServerName        gomatrixserverlib.ServerName
@@ -66,13 +66,22 @@ func NewOutputEDUConsumer(
 
 // Start consuming from EDU servers
 func (t *OutputEDUConsumer) Start() error {
-	if _, err := t.jetstream.Subscribe(t.typingTopic, t.onTypingEvent, t.durable); err != nil {
+	if err := jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.typingTopic, t.durable, t.onTypingEvent,
+		nats.DeliverAll(), nats.ManualAck(),
+	); err != nil {
 		return err
 	}
-	if _, err := t.jetstream.Subscribe(t.sendToDeviceTopic, t.onSendToDeviceEvent, t.durable); err != nil {
+	if err := jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.sendToDeviceTopic, t.durable, t.onSendToDeviceEvent,
+		nats.DeliverAll(), nats.ManualAck(),
+	); err != nil {
 		return err
 	}
-	if _, err := t.jetstream.Subscribe(t.receiptTopic, t.onReceiptEvent, t.durable); err != nil {
+	if err := jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.receiptTopic, t.durable, t.onReceiptEvent,
+		nats.DeliverAll(), nats.ManualAck(),
+	); err != nil {
 		return err
 	}
 	return nil
@@ -80,175 +89,169 @@ func (t *OutputEDUConsumer) Start() error {
 
 // onSendToDeviceEvent is called in response to a message received on the
 // send-to-device events topic from the EDU server.
-func (t *OutputEDUConsumer) onSendToDeviceEvent(msg *nats.Msg) {
+func (t *OutputEDUConsumer) onSendToDeviceEvent(ctx context.Context, msg *nats.Msg) bool {
 	// Extract the send-to-device event from msg.
-	jetstream.WithJetStreamMessage(msg, func(msg *nats.Msg) bool {
-		var ote api.OutputSendToDeviceEvent
-		if err := json.Unmarshal(msg.Data, &ote); err != nil {
-			log.WithError(err).Errorf("eduserver output log: message parse failed (expected send-to-device)")
-			return true
-		}
-
-		// only send send-to-device events which originated from us
-		_, originServerName, err := gomatrixserverlib.SplitID('@', ote.Sender)
-		if err != nil {
-			log.WithError(err).WithField("user_id", ote.Sender).Error("Failed to extract domain from send-to-device sender")
-			return true
-		}
-		if originServerName != t.ServerName {
-			log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere")
-			return true
-		}
-
-		_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
-		if err != nil {
-			log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
-			return true
-		}
-
-		// Pack the EDU and marshal it
-		edu := &gomatrixserverlib.EDU{
-			Type:   gomatrixserverlib.MDirectToDevice,
-			Origin: string(t.ServerName),
-		}
-		tdm := gomatrixserverlib.ToDeviceMessage{
-			Sender:    ote.Sender,
-			Type:      ote.Type,
-			MessageID: util.RandomString(32),
-			Messages: map[string]map[string]json.RawMessage{
-				ote.UserID: {
-					ote.DeviceID: ote.Content,
-				},
-			},
-		}
-		if edu.Content, err = json.Marshal(tdm); err != nil {
-			log.WithError(err).Error("failed to marshal EDU JSON")
-			return true
-		}
-
-		log.Infof("Sending send-to-device message into %q destination queue", destServerName)
-		if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil {
-			log.WithError(err).Error("failed to send EDU")
-			return false
-		}
-
+	var ote api.OutputSendToDeviceEvent
+	if err := json.Unmarshal(msg.Data, &ote); err != nil {
+		log.WithError(err).Errorf("eduserver output log: message parse failed (expected send-to-device)")
 		return true
-	})
+	}
+
+	// only send send-to-device events which originated from us
+	_, originServerName, err := gomatrixserverlib.SplitID('@', ote.Sender)
+	if err != nil {
+		log.WithError(err).WithField("user_id", ote.Sender).Error("Failed to extract domain from send-to-device sender")
+		return true
+	}
+	if originServerName != t.ServerName {
+		log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere")
+		return true
+	}
+
+	_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
+		return true
+	}
+
+	// Pack the EDU and marshal it
+	edu := &gomatrixserverlib.EDU{
+		Type:   gomatrixserverlib.MDirectToDevice,
+		Origin: string(t.ServerName),
+	}
+	tdm := gomatrixserverlib.ToDeviceMessage{
+		Sender:    ote.Sender,
+		Type:      ote.Type,
+		MessageID: util.RandomString(32),
+		Messages: map[string]map[string]json.RawMessage{
+			ote.UserID: {
+				ote.DeviceID: ote.Content,
+			},
+		},
+	}
+	if edu.Content, err = json.Marshal(tdm); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+
+	log.Debugf("Sending send-to-device message into %q destination queue", destServerName)
+	if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
 }
 
 // onTypingEvent is called in response to a message received on the typing
 // events topic from the EDU server.
-func (t *OutputEDUConsumer) onTypingEvent(msg *nats.Msg) {
-	jetstream.WithJetStreamMessage(msg, func(msg *nats.Msg) bool {
-		// Extract the typing event from msg.
-		var ote api.OutputTypingEvent
-		if err := json.Unmarshal(msg.Data, &ote); err != nil {
-			// Skip this msg but continue processing messages.
-			log.WithError(err).Errorf("eduserver output log: message parse failed (expected typing)")
-			_ = msg.Ack()
-			return true
-		}
-
-		// only send typing events which originated from us
-		_, typingServerName, err := gomatrixserverlib.SplitID('@', ote.Event.UserID)
-		if err != nil {
-			log.WithError(err).WithField("user_id", ote.Event.UserID).Error("Failed to extract domain from typing sender")
-			_ = msg.Ack()
-			return true
-		}
-		if typingServerName != t.ServerName {
-			return true
-		}
-
-		joined, err := t.db.GetJoinedHosts(t.ctx, ote.Event.RoomID)
-		if err != nil {
-			log.WithError(err).WithField("room_id", ote.Event.RoomID).Error("failed to get joined hosts for room")
-			return false
-		}
-
-		names := make([]gomatrixserverlib.ServerName, len(joined))
-		for i := range joined {
-			names[i] = joined[i].ServerName
-		}
-
-		edu := &gomatrixserverlib.EDU{Type: ote.Event.Type}
-		if edu.Content, err = json.Marshal(map[string]interface{}{
-			"room_id": ote.Event.RoomID,
-			"user_id": ote.Event.UserID,
-			"typing":  ote.Event.Typing,
-		}); err != nil {
-			log.WithError(err).Error("failed to marshal EDU JSON")
-			return true
-		}
-
-		if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
-			log.WithError(err).Error("failed to send EDU")
-			return false
-		}
-
+func (t *OutputEDUConsumer) onTypingEvent(ctx context.Context, msg *nats.Msg) bool {
+	// Extract the typing event from msg.
+	var ote api.OutputTypingEvent
+	if err := json.Unmarshal(msg.Data, &ote); err != nil {
+		// Skip this msg but continue processing messages.
+		log.WithError(err).Errorf("eduserver output log: message parse failed (expected typing)")
+		_ = msg.Ack()
 		return true
-	})
+	}
+
+	// only send typing events which originated from us
+	_, typingServerName, err := gomatrixserverlib.SplitID('@', ote.Event.UserID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", ote.Event.UserID).Error("Failed to extract domain from typing sender")
+		_ = msg.Ack()
+		return true
+	}
+	if typingServerName != t.ServerName {
+		return true
+	}
+
+	joined, err := t.db.GetJoinedHosts(ctx, ote.Event.RoomID)
+	if err != nil {
+		log.WithError(err).WithField("room_id", ote.Event.RoomID).Error("failed to get joined hosts for room")
+		return false
+	}
+
+	names := make([]gomatrixserverlib.ServerName, len(joined))
+	for i := range joined {
+		names[i] = joined[i].ServerName
+	}
+
+	edu := &gomatrixserverlib.EDU{Type: ote.Event.Type}
+	if edu.Content, err = json.Marshal(map[string]interface{}{
+		"room_id": ote.Event.RoomID,
+		"user_id": ote.Event.UserID,
+		"typing":  ote.Event.Typing,
+	}); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+
+	if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
 }
 
 // onReceiptEvent is called in response to a message received on the receipt
 // events topic from the EDU server.
-func (t *OutputEDUConsumer) onReceiptEvent(msg *nats.Msg) {
-	jetstream.WithJetStreamMessage(msg, func(msg *nats.Msg) bool {
-		// Extract the typing event from msg.
-		var receipt api.OutputReceiptEvent
-		if err := json.Unmarshal(msg.Data, &receipt); err != nil {
-			// Skip this msg but continue processing messages.
-			log.WithError(err).Errorf("eduserver output log: message parse failed (expected receipt)")
-			return true
-		}
-
-		// only send receipt events which originated from us
-		_, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID)
-		if err != nil {
-			log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender")
-			return true
-		}
-		if receiptServerName != t.ServerName {
-			return true
-		}
-
-		joined, err := t.db.GetJoinedHosts(t.ctx, receipt.RoomID)
-		if err != nil {
-			log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room")
-			return false
-		}
-
-		names := make([]gomatrixserverlib.ServerName, len(joined))
-		for i := range joined {
-			names[i] = joined[i].ServerName
-		}
-
-		content := map[string]api.FederationReceiptMRead{}
-		content[receipt.RoomID] = api.FederationReceiptMRead{
-			User: map[string]api.FederationReceiptData{
-				receipt.UserID: {
-					Data: api.ReceiptTS{
-						TS: receipt.Timestamp,
-					},
-					EventIDs: []string{receipt.EventID},
-				},
-			},
-		}
-
-		edu := &gomatrixserverlib.EDU{
-			Type:   gomatrixserverlib.MReceipt,
-			Origin: string(t.ServerName),
-		}
-		if edu.Content, err = json.Marshal(content); err != nil {
-			log.WithError(err).Error("failed to marshal EDU JSON")
-			return true
-		}
-
-		if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
-			log.WithError(err).Error("failed to send EDU")
-			return false
-		}
-
+func (t *OutputEDUConsumer) onReceiptEvent(ctx context.Context, msg *nats.Msg) bool {
+	// Extract the typing event from msg.
+	var receipt api.OutputReceiptEvent
+	if err := json.Unmarshal(msg.Data, &receipt); err != nil {
+		// Skip this msg but continue processing messages.
+		log.WithError(err).Errorf("eduserver output log: message parse failed (expected receipt)")
 		return true
-	})
+	}
+
+	// only send receipt events which originated from us
+	_, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID)
+	if err != nil {
+		log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender")
+		return true
+	}
+	if receiptServerName != t.ServerName {
+		return true
+	}
+
+	joined, err := t.db.GetJoinedHosts(ctx, receipt.RoomID)
+	if err != nil {
+		log.WithError(err).WithField("room_id", receipt.RoomID).Error("failed to get joined hosts for room")
+		return false
+	}
+
+	names := make([]gomatrixserverlib.ServerName, len(joined))
+	for i := range joined {
+		names[i] = joined[i].ServerName
+	}
+
+	content := map[string]api.FederationReceiptMRead{}
+	content[receipt.RoomID] = api.FederationReceiptMRead{
+		User: map[string]api.FederationReceiptData{
+			receipt.UserID: {
+				Data: api.ReceiptTS{
+					TS: receipt.Timestamp,
+				},
+				EventIDs: []string{receipt.EventID},
+			},
+		},
+	}
+
+	edu := &gomatrixserverlib.EDU{
+		Type:   gomatrixserverlib.MReceipt,
+		Origin: string(t.ServerName),
+	}
+	if edu.Content, err = json.Marshal(content); err != nil {
+		log.WithError(err).Error("failed to marshal EDU JSON")
+		return true
+	}
+
+	if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
+		log.WithError(err).Error("failed to send EDU")
+		return false
+	}
+
+	return true
 }

federationapi/consumers/keychange.go

@@ -17,80 +17,73 @@ package consumers
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 
-	"github.com/Shopify/sarama"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
 	"github.com/sirupsen/logrus"
 )
 
 // KeyChangeConsumer consumes events that originate in key server.
 type KeyChangeConsumer struct {
 	ctx        context.Context
-	consumer   *internal.ContinualConsumer
+	jetstream  nats.JetStreamContext
+	durable    string
 	db         storage.Database
 	queues     *queue.OutgoingQueues
 	serverName gomatrixserverlib.ServerName
 	rsAPI      roomserverAPI.RoomserverInternalAPI
+	topic      string
 }
 
 // NewKeyChangeConsumer creates a new KeyChangeConsumer. Call Start() to begin consuming from key servers.
 func NewKeyChangeConsumer(
 	process *process.ProcessContext,
 	cfg *config.KeyServer,
-	kafkaConsumer sarama.Consumer,
+	js nats.JetStreamContext,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) *KeyChangeConsumer {
-	c := &KeyChangeConsumer{
-		ctx: process.Context(),
-		consumer: &internal.ContinualConsumer{
-			Process:        process,
-			ComponentName:  "federationapi/keychange",
-			Topic:          string(cfg.Matrix.JetStream.TopicFor(jetstream.OutputKeyChangeEvent)),
-			Consumer:       kafkaConsumer,
-			PartitionStore: store,
-		},
+	return &KeyChangeConsumer{
+		ctx:        process.Context(),
+		jetstream:  js,
+		durable:    cfg.Matrix.JetStream.TopicFor("FederationAPIKeyChangeConsumer"),
+		topic:      cfg.Matrix.JetStream.TopicFor(jetstream.OutputKeyChangeEvent),
 		queues:     queues,
 		db:         store,
 		serverName: cfg.Matrix.ServerName,
 		rsAPI:      rsAPI,
 	}
-	c.consumer.ProcessMessage = c.onMessage
-
-	return c
 }
 
 // Start consuming from key servers
 func (t *KeyChangeConsumer) Start() error {
-	if err := t.consumer.Start(); err != nil {
-		return fmt.Errorf("t.consumer.Start: %w", err)
-	}
-	return nil
+	return jetstream.JetStreamConsumer(
+		t.ctx, t.jetstream, t.topic, t.durable, t.onMessage,
+		nats.DeliverAll(), nats.ManualAck(),
+	)
 }
 
 // onMessage is called in response to a message received on the
 // key change events topic from the key server.
-func (t *KeyChangeConsumer) onMessage(msg *sarama.ConsumerMessage) error {
+func (t *KeyChangeConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
 	var m api.DeviceMessage
-	if err := json.Unmarshal(msg.Value, &m); err != nil {
+	if err := json.Unmarshal(msg.Data, &m); err != nil {
 		logrus.WithError(err).Errorf("failed to read device message from key change topic")
-		return nil
+		return true
 	}
 	if m.DeviceKeys == nil && m.OutputCrossSigningKeyUpdate == nil {
 		// This probably shouldn't happen but stops us from panicking if we come
 		// across an update that doesn't satisfy either types.
-		return nil
+		return true
 	}
 	switch m.Type {
 	case api.TypeCrossSigningUpdate:
@@ -102,9 +95,9 @@ func (t *KeyChangeConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 	}
 }
 
-func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) error {
+func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
 	if m.DeviceKeys == nil {
-		return nil
+		return true
 	}
 	logger := logrus.WithField("user_id", m.UserID)
 
@@ -112,10 +105,10 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) error {
 	_, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID)
 	if err != nil {
 		logger.WithError(err).Error("Failed to extract domain from key change event")
-		return nil
+		return true
 	}
 	if originServerName != t.serverName {
-		return nil
+		return true
 	}
 
 	var queryRes roomserverAPI.QueryRoomsForUserResponse
@@ -125,15 +118,18 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) error {
 	}, &queryRes)
 	if err != nil {
 		logger.WithError(err).Error("failed to calculate joined rooms for user")
-		return nil
+		return true
 	}
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
 		logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in")
-		return nil
+		return true
 	}
 
+	if len(destinations) == 0 {
+		return true
+	}
 	// Pack the EDU and marshal it
 	edu := &gomatrixserverlib.EDU{
 		Type:   gomatrixserverlib.MDeviceListUpdate,
@@ -149,24 +145,26 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) error {
 		Keys:              m.KeyJSON,
 	}
 	if edu.Content, err = json.Marshal(event); err != nil {
-		return err
+		logger.WithError(err).Error("failed to marshal EDU JSON")
+		return true
 	}
 
-	logrus.Infof("Sending device list update message to %q", destinations)
-	return t.queues.SendEDU(edu, t.serverName, destinations)
+	logger.Debugf("Sending device list update message to %q", destinations)
+	err = t.queues.SendEDU(edu, t.serverName, destinations)
+	return err == nil
 }
 
-func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) error {
+func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
 	output := m.CrossSigningKeyUpdate
 	_, host, err := gomatrixserverlib.SplitID('@', output.UserID)
 	if err != nil {
 		logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure")
-		return nil
+		return true
 	}
 	if host != gomatrixserverlib.ServerName(t.serverName) {
 		// Ignore any messages that didn't originate locally, otherwise we'll
 		// end up parroting information we received from other servers.
-		return nil
+		return true
 	}
 	logger := logrus.WithField("user_id", output.UserID)
 
@@ -177,13 +175,17 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) error {
 	}, &queryRes)
 	if err != nil {
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user")
-		return nil
+		return true
 	}
 	// send this key change to all servers who share rooms with this user.
 	destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
 	if err != nil {
 		logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in")
-		return nil
+		return true
+	}
+
+	if len(destinations) == 0 {
+		return true
 	}
 
 	// Pack the EDU and marshal it
@@ -193,11 +195,12 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) error {
 	}
 	if edu.Content, err = json.Marshal(output); err != nil {
 		logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping")
-		return nil
+		return true
 	}
 
-	logger.Infof("Sending cross-signing update message to %q", destinations)
-	return t.queues.SendEDU(edu, t.serverName, destinations)
+	logger.Debugf("Sending cross-signing update message to %q", destinations)
+	err = t.queues.SendEDU(edu, t.serverName, destinations)
+	return err == nil
 }
 
 func prevID(streamID int) []int {

federationapi/consumers/roomserver.go

@@ -19,6 +19,10 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/types"
@@ -26,9 +30,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/nats-io/nats.go"
-	log "github.com/sirupsen/logrus"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
@@ -37,7 +38,7 @@ type OutputRoomEventConsumer struct {
 	cfg       *config.FederationAPI
 	rsAPI     api.RoomserverInternalAPI
 	jetstream nats.JetStreamContext
-	durable   nats.SubOpt
+	durable   string
 	db        storage.Database
 	queues    *queue.OutgoingQueues
 	topic     string
@@ -66,74 +67,63 @@ func NewOutputRoomEventConsumer(
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
-	_, err := s.jetstream.Subscribe(
-		s.topic, s.onMessage, s.durable,
-		nats.DeliverAll(),
-		nats.ManualAck(),
+	return jetstream.JetStreamConsumer(
+		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
+		nats.DeliverAll(), nats.ManualAck(),
 	)
-	return err
 }
 
 // onMessage is called when the federation server receives a new event from the room server output log.
 // It is unsafe to call this with messages for the same room in multiple gorountines
 // because updates it will likely fail with a types.EventIDMismatchError when it
 // realises that it cannot update the room state using the deltas.
-func (s *OutputRoomEventConsumer) onMessage(msg *nats.Msg) {
-	jetstream.WithJetStreamMessage(msg, func(msg *nats.Msg) bool {
-		// Parse out the event JSON
-		var output api.OutputEvent
-		if err := json.Unmarshal(msg.Data, &output); err != nil {
-			// If the message was invalid, log it and move on to the next message in the stream
-			log.WithError(err).Errorf("roomserver output log: message parse failure")
-			return true
-		}
+func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
+	// Parse out the event JSON
+	var output api.OutputEvent
+	if err := json.Unmarshal(msg.Data, &output); err != nil {
+		// If the message was invalid, log it and move on to the next message in the stream
+		log.WithError(err).Errorf("roomserver output log: message parse failure")
+		return true
+	}
 
-		switch output.Type {
-		case api.OutputTypeNewRoomEvent:
-			ev := output.NewRoomEvent.Event
+	switch output.Type {
+	case api.OutputTypeNewRoomEvent:
+		ev := output.NewRoomEvent.Event
 
-			if output.NewRoomEvent.RewritesState {
-				if err := s.db.PurgeRoomState(s.ctx, ev.RoomID()); err != nil {
-					log.WithError(err).Errorf("roomserver output log: purge room state failure")
-					return false
-				}
-			}
-
-			if err := s.processMessage(*output.NewRoomEvent); err != nil {
-				switch err.(type) {
-				case *queue.ErrorFederationDisabled:
-					log.WithField("error", output.Type).Info(
-						err.Error(),
-					)
-				default:
-					// panic rather than continue with an inconsistent database
-					log.WithFields(log.Fields{
-						"event_id":   ev.EventID(),
-						"event":      string(ev.JSON()),
-						"add":        output.NewRoomEvent.AddsStateEventIDs,
-						"del":        output.NewRoomEvent.RemovesStateEventIDs,
-						log.ErrorKey: err,
-					}).Panicf("roomserver output log: write room event failure")
-				}
-			}
-
-		case api.OutputTypeNewInboundPeek:
-			if err := s.processInboundPeek(*output.NewInboundPeek); err != nil {
-				log.WithFields(log.Fields{
-					"event":      output.NewInboundPeek,
-					log.ErrorKey: err,
-				}).Panicf("roomserver output log: remote peek event failure")
+		if output.NewRoomEvent.RewritesState {
+			if err := s.db.PurgeRoomState(s.ctx, ev.RoomID()); err != nil {
+				log.WithError(err).Errorf("roomserver output log: purge room state failure")
 				return false
 			}
-
-		default:
-			log.WithField("type", output.Type).Debug(
-				"roomserver output log: ignoring unknown output type",
-			)
 		}
 
-		return true
-	})
+		if err := s.processMessage(*output.NewRoomEvent); err != nil {
+			// panic rather than continue with an inconsistent database
+			log.WithFields(log.Fields{
+				"event_id":   ev.EventID(),
+				"event":      string(ev.JSON()),
+				"add":        output.NewRoomEvent.AddsStateEventIDs,
+				"del":        output.NewRoomEvent.RemovesStateEventIDs,
+				log.ErrorKey: err,
+			}).Panicf("roomserver output log: write room event failure")
+		}
+
+	case api.OutputTypeNewInboundPeek:
+		if err := s.processInboundPeek(*output.NewInboundPeek); err != nil {
+			log.WithFields(log.Fields{
+				"event":      output.NewInboundPeek,
+				log.ErrorKey: err,
+			}).Panicf("roomserver output log: remote peek event failure")
+			return false
+		}
+
+	default:
+		log.WithField("type", output.Type).Debug(
+			"roomserver output log: ignoring unknown output type",
+		)
+	}
+
+	return true
 }
 
 // processInboundPeek starts tracking a new federated inbound peek (replacing the existing one if any)

federationapi/federationapi.go

@@ -92,7 +92,7 @@ func NewInternalAPI(
 		FailuresUntilBlacklist: cfg.FederationMaxRetries,
 	}
 
-	js, consumer, _ := jetstream.Prepare(&cfg.Matrix.JetStream)
+	js := jetstream.Prepare(&cfg.Matrix.JetStream)
 
 	queues := queue.NewOutgoingQueues(
 		federationDB, base.ProcessContext,
@@ -120,7 +120,7 @@ func NewInternalAPI(
 		logrus.WithError(err).Panic("failed to start typing server consumer")
 	}
 	keyConsumer := consumers.NewKeyChangeConsumer(
-		base.ProcessContext, &base.Cfg.KeyServer, consumer, queues, federationDB, rsAPI,
+		base.ProcessContext, &base.Cfg.KeyServer, js, queues, federationDB, rsAPI,
 	)
 	if err := keyConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start key server consumer")

federationapi/internal/perform.go

@@ -196,29 +196,23 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err)
 	}
 
-	// No longer reuse the request context from this point forward.
-	// We don't want the client timing out to interrupt the join.
-	var cancel context.CancelFunc
-	ctx, cancel = context.WithCancel(context.Background())
-
 	// Try to perform a send_join using the newly built event.
 	respSendJoin, err := r.federation.SendJoin(
-		ctx,
+		context.Background(),
 		serverName,
 		event,
-		respMakeJoin.RoomVersion,
 	)
 	if err != nil {
 		r.statistics.ForServer(serverName).Failure()
-		cancel()
 		return fmt.Errorf("r.federation.SendJoin: %w", err)
 	}
 	r.statistics.ForServer(serverName).Success()
 
+	authEvents := respSendJoin.AuthEvents.UntrustedEvents(respMakeJoin.RoomVersion)
+
 	// Sanity-check the join response to ensure that it has a create
 	// event, that the room version is known, etc.
-	if err := sanityCheckAuthChain(respSendJoin.AuthEvents); err != nil {
-		cancel()
+	if err = sanityCheckAuthChain(authEvents); err != nil {
 		return fmt.Errorf("sanityCheckAuthChain: %w", err)
 	}
 
@@ -227,41 +221,36 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	// to complete, but if the client does give up waiting, we'll
 	// still continue to process the join anyway so that we don't
 	// waste the effort.
-	go func() {
-		defer cancel()
+	// TODO: Can we expand Check here to return a list of missing auth
+	// events rather than failing one at a time?
+	var respState *gomatrixserverlib.RespState
+	respState, err = respSendJoin.Check(
+		context.Background(),
+		respMakeJoin.RoomVersion,
+		r.keyRing,
+		event,
+		federatedAuthProvider(ctx, r.federation, r.keyRing, serverName),
+	)
+	if err != nil {
+		return fmt.Errorf("respSendJoin.Check: %w", err)
+	}
 
-		// TODO: Can we expand Check here to return a list of missing auth
-		// events rather than failing one at a time?
-		respState, err := respSendJoin.Check(ctx, r.keyRing, event, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName))
-		if err != nil {
-			logrus.WithFields(logrus.Fields{
-				"room_id": roomID,
-				"user_id": userID,
-			}).WithError(err).Error("Failed to process room join response")
-			return
-		}
+	// If we successfully performed a send_join above then the other
+	// server now thinks we're a part of the room. Send the newly
+	// returned state to the roomserver to update our local view.
+	if err = roomserverAPI.SendEventWithState(
+		context.Background(),
+		r.rsAPI,
+		roomserverAPI.KindNew,
+		respState,
+		event.Headered(respMakeJoin.RoomVersion),
+		serverName,
+		nil,
+		false,
+	); err != nil {
+		return fmt.Errorf("roomserverAPI.SendEventWithState: %w", err)
+	}
 
-		// If we successfully performed a send_join above then the other
-		// server now thinks we're a part of the room. Send the newly
-		// returned state to the roomserver to update our local view.
-		if err = roomserverAPI.SendEventWithState(
-			ctx, r.rsAPI,
-			roomserverAPI.KindNew,
-			respState,
-			event.Headered(respMakeJoin.RoomVersion),
-			serverName,
-			nil,
-			false,
-		); err != nil {
-			logrus.WithFields(logrus.Fields{
-				"room_id": roomID,
-				"user_id": userID,
-			}).WithError(err).Error("Failed to send room join response to roomserver")
-			return
-		}
-	}()
-
-	<-ctx.Done()
 	return nil
 }
 
@@ -405,12 +394,13 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 	ctx = context.Background()
 
 	respState := respPeek.ToRespState()
+	authEvents := respState.AuthEvents.UntrustedEvents(respPeek.RoomVersion)
 	// authenticate the state returned (check its auth events etc)
 	// the equivalent of CheckSendJoinResponse()
-	if err = sanityCheckAuthChain(respState.AuthEvents); err != nil {
+	if err = sanityCheckAuthChain(authEvents); err != nil {
 		return fmt.Errorf("sanityCheckAuthChain: %w", err)
 	}
-	if err = respState.Check(ctx, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)); err != nil {
+	if err = respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)); err != nil {
 		return fmt.Errorf("error checking state returned from peeking: %w", err)
 	}
 
@@ -562,10 +552,15 @@ func (r *FederationInternalAPI) PerformInvite(
 
 	inviteRes, err := r.federation.SendInviteV2(ctx, destination, inviteReq)
 	if err != nil {
-		return fmt.Errorf("r.federation.SendInviteV2: %w", err)
+		return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err)
 	}
+	logrus.Infof("GOT INVITE RESPONSE %s", string(inviteRes.Event))
 
-	response.Event = inviteRes.Event.Headered(request.RoomVersion)
+	inviteEvent, err := inviteRes.Event.UntrustedEvent(request.RoomVersion)
+	if err != nil {
+		return fmt.Errorf("r.federation.SendInviteV2 failed to decode event response: %w", err)
+	}
+	response.Event = inviteEvent.Headered(request.RoomVersion)
 	return nil
 }
 

federationapi/inthttp/client.go

@@ -387,14 +387,7 @@ func (h *httpFederationInternalAPI) LookupMissingEvents(
 	if request.Err != nil {
 		return res, request.Err
 	}
-	res.Events = make([]*gomatrixserverlib.Event, 0, len(request.Res.Events))
-	for _, js := range request.Res.Events {
-		ev, err := gomatrixserverlib.NewEventFromUntrustedJSON(js, roomVersion)
-		if err != nil {
-			return res, err
-		}
-		res.Events = append(res.Events, ev)
-	}
+	res.Events = request.Res.Events
 	return res, nil
 }
 

federationapi/queue/destinationqueue.go

@@ -297,7 +297,7 @@ func (oq *destinationQueue) backgroundSend() {
 			// We haven't backed off yet, so wait for the suggested amount of
 			// time.
 			duration := time.Until(*until)
-			logrus.Warnf("Backing off %q for %s", oq.destination, duration)
+			logrus.Debugf("Backing off %q for %s", oq.destination, duration)
 			oq.backingOff.Store(true)
 			destinationQueueBackingOff.Inc()
 			select {

federationapi/queue/queue.go

@@ -22,15 +22,16 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
+
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
-	"github.com/tidwall/gjson"
 )
 
 // OutgoingQueues is a collection of queues for sending transactions to other
@@ -182,23 +183,14 @@ func (oqs *OutgoingQueues) clearQueue(oq *destinationQueue) {
 	destinationQueueTotal.Dec()
 }
 
-type ErrorFederationDisabled struct {
-	Message string
-}
-
-func (e *ErrorFederationDisabled) Error() string {
-	return e.Message
-}
-
 // SendEvent sends an event to the destinations
 func (oqs *OutgoingQueues) SendEvent(
 	ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName,
 	destinations []gomatrixserverlib.ServerName,
 ) error {
 	if oqs.disabled {
-		return &ErrorFederationDisabled{
-			Message: "Federation disabled",
-		}
+		log.Trace("Federation is disabled, not sending event")
+		return nil
 	}
 	if origin != oqs.origin {
 		// TODO: Support virtual hosting; gh issue #577.
@@ -262,9 +254,8 @@ func (oqs *OutgoingQueues) SendEDU(
 	destinations []gomatrixserverlib.ServerName,
 ) error {
 	if oqs.disabled {
-		return &ErrorFederationDisabled{
-			Message: "Federation disabled",
-		}
+		log.Trace("Federation is disabled, not sending EDU")
+		return nil
 	}
 	if origin != oqs.origin {
 		// TODO: Support virtual hosting; gh issue #577.

federationapi/routing/eventauth.go

@@ -65,7 +65,7 @@ func GetEventAuth(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: gomatrixserverlib.RespEventAuth{
-			AuthEvents: gomatrixserverlib.UnwrapEventHeaders(response.AuthChainEvents),
+			AuthEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.AuthChainEvents),
 		},
 	}
 }

federationapi/routing/invite.go

@@ -178,12 +178,12 @@ func processInvite(
 		if isInviteV2 {
 			return util.JSONResponse{
 				Code: http.StatusOK,
-				JSON: gomatrixserverlib.RespInviteV2{Event: &signedEvent},
+				JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent.JSON()},
 			}
 		} else {
 			return util.JSONResponse{
 				Code: http.StatusOK,
-				JSON: gomatrixserverlib.RespInvite{Event: &signedEvent},
+				JSON: gomatrixserverlib.RespInvite{Event: signedEvent.JSON()},
 			}
 		}
 	default:

federationapi/routing/join.go

@@ -351,8 +351,8 @@ func SendJoin(
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: gomatrixserverlib.RespSendJoin{
-			StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents),
-			AuthEvents:  gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents),
+			StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents),
+			AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents),
 			Origin:      cfg.Matrix.ServerName,
 		},
 	}

federationapi/routing/missingevents.go

@@ -62,7 +62,7 @@ func GetMissingEvents(
 	eventsResponse.Events = filterEvents(eventsResponse.Events, roomID)
 
 	resp := gomatrixserverlib.RespMissingEvents{
-		Events: gomatrixserverlib.UnwrapEventHeaders(eventsResponse.Events),
+		Events: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(eventsResponse.Events),
 	}
 
 	return util.JSONResponse{

federationapi/routing/peek.go

@@ -88,8 +88,8 @@ func Peek(
 	}
 
 	respPeek := gomatrixserverlib.RespPeek{
-		StateEvents:     gomatrixserverlib.UnwrapEventHeaders(response.StateEvents),
-		AuthEvents:      gomatrixserverlib.UnwrapEventHeaders(response.AuthChainEvents),
+		StateEvents:     gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.StateEvents),
+		AuthEvents:      gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.AuthChainEvents),
 		RoomVersion:     response.RoomVersion,
 		LatestEvent:     response.LatestEvent.Unwrap(),
 		RenewalInterval: renewalInterval,

federationapi/routing/send.go

@@ -162,7 +162,7 @@ func Send(
 	t.TransactionID = txnID
 	t.Destination = cfg.Matrix.ServerName
 
-	util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs))
+	util.GetLogger(httpReq.Context()).Debugf("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs))
 
 	resp, jsonErr := t.processTransaction(httpReq.Context())
 	if jsonErr != nil {
@@ -221,7 +221,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
 		if err := t.rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
-			util.GetLogger(ctx).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID)
+			util.GetLogger(ctx).WithError(err).Debug("Transaction: Failed to query room version for room", verReq.RoomID)
 			return ""
 		}
 		roomVersions[roomID] = verRes.RoomVersion
@@ -234,7 +234,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res
 			RoomID string `json:"room_id"`
 		}
 		if err := json.Unmarshal(pdu, &header); err != nil {
-			util.GetLogger(ctx).WithError(err).Warn("Transaction: Failed to extract room ID from event")
+			util.GetLogger(ctx).WithError(err).Debug("Transaction: Failed to extract room ID from event")
 			// We don't know the event ID at this point so we can't return the
 			// failure in the PDU results
 			continue
@@ -255,7 +255,10 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res
 					JSON: jsonerror.BadJSON("PDU contains bad JSON"),
 				}
 			}
-			util.GetLogger(ctx).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %s", string(pdu))
+			util.GetLogger(ctx).WithError(err).Debugf("Transaction: Failed to parse event JSON of event %s", string(pdu))
+			continue
+		}
+		if event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("") {
 			continue
 		}
 		if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID(), t.Origin) {
@@ -265,7 +268,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res
 			continue
 		}
 		if err = event.VerifyEventSignatures(ctx, t.keys); err != nil {
-			util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID())
+			util.GetLogger(ctx).WithError(err).Debugf("Transaction: Couldn't validate signature of event %q", event.EventID())
 			results[event.EventID()] = gomatrixserverlib.PDUResult{
 				Error: err.Error(),
 			}
@@ -287,7 +290,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res
 			nil,
 			true,
 		); err != nil {
-			util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't submit event %q to input queue: %s", event.EventID(), err)
+			util.GetLogger(ctx).WithError(err).Errorf("Transaction: Couldn't submit event %q to input queue: %s", event.EventID(), err)
 			results[event.EventID()] = gomatrixserverlib.PDUResult{
 				Error: err.Error(),
 			}
@@ -314,16 +317,16 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				Typing bool   `json:"typing"`
 			}
 			if err := json.Unmarshal(e.Content, &typingPayload); err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal typing event")
+				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal typing event")
 				continue
 			}
 			_, domain, err := gomatrixserverlib.SplitID('@', typingPayload.UserID)
 			if err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to split domain from typing event sender")
+				util.GetLogger(ctx).WithError(err).Debug("Failed to split domain from typing event sender")
 				continue
 			}
 			if domain != t.Origin {
-				util.GetLogger(ctx).Warnf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
+				util.GetLogger(ctx).Debugf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
 				continue
 			}
 			if err := eduserverAPI.SendTyping(ctx, t.eduAPI, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil {
@@ -333,7 +336,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			// https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema
 			var directPayload gomatrixserverlib.ToDeviceMessage
 			if err := json.Unmarshal(e.Content, &directPayload); err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal send-to-device events")
+				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal send-to-device events")
 				continue
 			}
 			for userID, byUser := range directPayload.Messages {
@@ -355,7 +358,7 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 			payload := map[string]eduserverAPI.FederationReceiptMRead{}
 
 			if err := json.Unmarshal(e.Content, &payload); err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal receipt event")
+				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal receipt event")
 				continue
 			}
 
@@ -363,11 +366,11 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				for userID, mread := range receipt.User {
 					_, domain, err := gomatrixserverlib.SplitID('@', userID)
 					if err != nil {
-						util.GetLogger(ctx).WithError(err).Error("Failed to split domain from receipt event sender")
+						util.GetLogger(ctx).WithError(err).Debug("Failed to split domain from receipt event sender")
 						continue
 					}
 					if t.Origin != domain {
-						util.GetLogger(ctx).Warnf("Dropping receipt event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
+						util.GetLogger(ctx).Debugf("Dropping receipt event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
 						continue
 					}
 					if err := t.processReceiptEvent(ctx, userID, roomID, "m.read", mread.Data.TS, mread.EventIDs); err != nil {
@@ -382,20 +385,8 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 				}
 			}
 		case eduserverAPI.MSigningKeyUpdate:
-			var updatePayload eduserverAPI.CrossSigningKeyUpdate
-			if err := json.Unmarshal(e.Content, &updatePayload); err != nil {
-				util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
-					"user_id": updatePayload.UserID,
-				}).Error("Failed to send signing key update to edu server")
-				continue
-			}
-			inputReq := &eduserverAPI.InputCrossSigningKeyUpdateRequest{
-				CrossSigningKeyUpdate: updatePayload,
-			}
-			inputRes := &eduserverAPI.InputCrossSigningKeyUpdateResponse{}
-			if err := t.eduAPI.InputCrossSigningKeyUpdate(ctx, inputReq, inputRes); err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal cross-signing update")
-				continue
+			if err := t.processSigningKeyUpdate(ctx, e); err != nil {
+				logrus.WithError(err).Errorf("Failed to process signing key update")
 			}
 		default:
 			util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU")
@@ -403,6 +394,34 @@ func (t *txnReq) processEDUs(ctx context.Context) {
 	}
 }
 
+func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverlib.EDU) error {
+	var updatePayload eduserverAPI.CrossSigningKeyUpdate
+	if err := json.Unmarshal(e.Content, &updatePayload); err != nil {
+		util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
+			"user_id": updatePayload.UserID,
+		}).Debug("Failed to unmarshal signing key update")
+		return err
+	}
+
+	keys := gomatrixserverlib.CrossSigningKeys{}
+	if updatePayload.MasterKey != nil {
+		keys.MasterKey = *updatePayload.MasterKey
+	}
+	if updatePayload.SelfSigningKey != nil {
+		keys.SelfSigningKey = *updatePayload.SelfSigningKey
+	}
+	uploadReq := &keyapi.PerformUploadDeviceKeysRequest{
+		CrossSigningKeys: keys,
+		UserID:           updatePayload.UserID,
+	}
+	uploadRes := &keyapi.PerformUploadDeviceKeysResponse{}
+	t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes)
+	if uploadRes.Error != nil {
+		return uploadRes.Error
+	}
+	return nil
+}
+
 // processReceiptEvent sends receipt events to the edu server
 func (t *txnReq) processReceiptEvent(ctx context.Context,
 	userID, roomID, receiptType string,

federationapi/routing/send_test.go

@@ -93,11 +93,10 @@ func (o *testEDUProducer) InputCrossSigningKeyUpdate(
 
 type testRoomserverAPI struct {
 	api.RoomserverInternalAPITrace
-	inputRoomEvents            []api.InputRoomEvent
-	queryMissingAuthPrevEvents func(*api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse
-	queryStateAfterEvents      func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse
-	queryEventsByID            func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse
-	queryLatestEventsAndState  func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse
+	inputRoomEvents           []api.InputRoomEvent
+	queryStateAfterEvents     func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse
+	queryEventsByID           func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse
+	queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse
 }
 
 func (t *testRoomserverAPI) InputRoomEvents(
@@ -140,20 +139,6 @@ func (t *testRoomserverAPI) QueryStateAfterEvents(
 	return nil
 }
 
-// Query the state after a list of events in a room from the room server.
-func (t *testRoomserverAPI) QueryMissingAuthPrevEvents(
-	ctx context.Context,
-	request *api.QueryMissingAuthPrevEventsRequest,
-	response *api.QueryMissingAuthPrevEventsResponse,
-) error {
-	response.RoomVersion = testRoomVersion
-	res := t.queryMissingAuthPrevEvents(request)
-	response.RoomExists = res.RoomExists
-	response.MissingAuthEventIDs = res.MissingAuthEventIDs
-	response.MissingPrevEventIDs = res.MissingPrevEventIDs
-	return nil
-}
-
 // Query a list of events by event ID.
 func (t *testRoomserverAPI) QueryEventsByID(
 	ctx context.Context,
@@ -312,15 +297,7 @@ func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []*gomat
 // The purpose of this test is to check that receiving an event over federation for which we have the prev_events works correctly, and passes it on
 // to the roomserver. It's the most basic test possible.
 func TestBasicTransaction(t *testing.T) {
-	rsAPI := &testRoomserverAPI{
-		queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse {
-			return api.QueryMissingAuthPrevEventsResponse{
-				RoomExists:          true,
-				MissingAuthEventIDs: []string{},
-				MissingPrevEventIDs: []string{},
-			}
-		},
-	}
+	rsAPI := &testRoomserverAPI{}
 	pdus := []json.RawMessage{
 		testData[len(testData)-1], // a message event
 	}
@@ -332,15 +309,7 @@ func TestBasicTransaction(t *testing.T) {
 // The purpose of this test is to check that if the event received fails auth checks the event is still sent to the roomserver
 // as it does the auth check.
 func TestTransactionFailAuthChecks(t *testing.T) {
-	rsAPI := &testRoomserverAPI{
-		queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse {
-			return api.QueryMissingAuthPrevEventsResponse{
-				RoomExists:          true,
-				MissingAuthEventIDs: []string{},
-				MissingPrevEventIDs: []string{},
-			}
-		},
-	}
+	rsAPI := &testRoomserverAPI{}
 	pdus := []json.RawMessage{
 		testData[len(testData)-1], // a message event
 	}

federationapi/routing/state.go

@@ -35,12 +35,15 @@ func GetState(
 		return *err
 	}
 
-	state, err := getState(ctx, request, rsAPI, roomID, eventID)
+	stateEvents, authChain, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}
 
-	return util.JSONResponse{Code: http.StatusOK, JSON: state}
+	return util.JSONResponse{Code: http.StatusOK, JSON: &gomatrixserverlib.RespState{
+		AuthEvents:  gomatrixserverlib.NewEventJSONsFromHeaderedEvents(authChain),
+		StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateEvents),
+	}}
 }
 
 // GetStateIDs returns state event IDs & auth event IDs for the roomID, eventID
@@ -55,13 +58,13 @@ func GetStateIDs(
 		return *err
 	}
 
-	state, err := getState(ctx, request, rsAPI, roomID, eventID)
+	stateEvents, authEvents, err := getState(ctx, request, rsAPI, roomID, eventID)
 	if err != nil {
 		return *err
 	}
 
-	stateEventIDs := getIDsFromEvent(state.StateEvents)
-	authEventIDs := getIDsFromEvent(state.AuthEvents)
+	stateEventIDs := getIDsFromEvent(stateEvents)
+	authEventIDs := getIDsFromEvent(authEvents)
 
 	return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.RespStateIDs{
 		StateEventIDs: stateEventIDs,
@@ -97,18 +100,18 @@ func getState(
 	rsAPI api.RoomserverInternalAPI,
 	roomID string,
 	eventID string,
-) (*gomatrixserverlib.RespState, *util.JSONResponse) {
+) (stateEvents, authEvents []*gomatrixserverlib.HeaderedEvent, errRes *util.JSONResponse) {
 	event, resErr := fetchEvent(ctx, rsAPI, eventID)
 	if resErr != nil {
-		return nil, resErr
+		return nil, nil, resErr
 	}
 
 	if event.RoomID() != roomID {
-		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: jsonerror.NotFound("event does not belong to this room")}
+		return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: jsonerror.NotFound("event does not belong to this room")}
 	}
 	resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID)
 	if resErr != nil {
-		return nil, resErr
+		return nil, nil, resErr
 	}
 
 	var response api.QueryStateAndAuthChainResponse
@@ -123,20 +126,17 @@ func getState(
 	)
 	if err != nil {
 		resErr := util.ErrorResponse(err)
-		return nil, &resErr
+		return nil, nil, &resErr
 	}
 
 	if !response.RoomExists {
-		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
+		return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
 	}
 
-	return &gomatrixserverlib.RespState{
-		StateEvents: gomatrixserverlib.UnwrapEventHeaders(response.StateEvents),
-		AuthEvents:  gomatrixserverlib.UnwrapEventHeaders(response.AuthChainEvents),
-	}, nil
+	return response.StateEvents, response.AuthChainEvents, nil
 }
 
-func getIDsFromEvent(events []*gomatrixserverlib.Event) []string {
+func getIDsFromEvent(events []*gomatrixserverlib.HeaderedEvent) []string {
 	IDs := make([]string, len(events))
 	for i := range events {
 		IDs[i] = events[i].EventID()

federationapi/routing/threepid.go

@@ -170,13 +170,18 @@ func ExchangeThirdPartyInvite(
 		util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed")
 		return jsonerror.InternalServerError()
 	}
+	inviteEvent, err := signedEvent.Event.UntrustedEvent(verRes.RoomVersion)
+	if err != nil {
+		util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed")
+		return jsonerror.InternalServerError()
+	}
 
 	// Send the event to the roomserver
 	if err = api.SendEvents(
 		httpReq.Context(), rsAPI,
 		api.KindNew,
 		[]*gomatrixserverlib.HeaderedEvent{
-			signedEvent.Event.Headered(verRes.RoomVersion),
+			inviteEvent.Headered(verRes.RoomVersion),
 		},
 		request.Origin(),
 		cfg.Matrix.ServerName,

federationapi/storage/interface.go

@@ -19,12 +19,10 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	"github.com/matrix-org/dendrite/federationapi/types"
-	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type Database interface {
-	internal.PartitionStorer
 	gomatrixserverlib.KeyDatabase
 
 	UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error)

go.mod

@@ -1,6 +1,6 @@
 module github.com/matrix-org/dendrite
 
-replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.3.3-0.20220104162330-c76d5fd70423
+replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.7.2-0.20220217100407-087330ed46ad
 
 replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c
 
@@ -11,12 +11,11 @@ require (
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/S7evinK/saramajetstream v0.0.0-20210709110708-de6efc8c4a32
-	github.com/Shopify/sarama v1.29.0
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/docker/docker v20.10.12+incompatible
 	github.com/docker/go-connections v0.4.0
+	github.com/frankban/quicktest v1.14.0 // indirect
 	github.com/getsentry/sentry-go v0.12.0
 	github.com/gologme/log v1.3.0
 	github.com/gorilla/mux v1.8.0
@@ -40,13 +39,13 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220128100033-8d79e0c35e32
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220214133635-20632dd262ed
 	github.com/matrix-org/pinecone v0.0.0-20220121094951-351265543ddf
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.10
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nats-io/nats-server/v2 v2.3.2
-	github.com/nats-io/nats.go v1.13.1-0.20211122170419-d7c1d78a50fc
+	github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20211220175533-9d16fdc47b31
@@ -54,22 +53,24 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose v2.7.0+incompatible
-	github.com/prometheus/client_golang v1.12.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/sirupsen/logrus v1.8.1
-	github.com/tidwall/gjson v1.13.0
+	github.com/tidwall/gjson v1.14.0
 	github.com/tidwall/sjson v1.2.4
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.2
 	go.uber.org/atomic v1.9.0
-	golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8
+	golang.org/x/crypto v0.0.0-20220209195652-db638375bc3a
 	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
 	golang.org/x/mobile v0.0.0-20220112015953-858099ff7816
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
+	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	gopkg.in/h2non/bimg.v1 v1.1.5
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	nhooyr.io/websocket v1.8.7
 )
 
-go 1.15
+go 1.16

go.sum

@@ -100,14 +100,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
 github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA=
 github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
-github.com/S7evinK/saramajetstream v0.0.0-20210709110708-de6efc8c4a32 h1:i3fOph9Hjleo6LbuqN9ODFxnwt7mOtYMpCGeC8qJN50=
-github.com/S7evinK/saramajetstream v0.0.0-20210709110708-de6efc8c4a32/go.mod h1:ne+jkLlzafIzaE4Q0Ze81T27dNgXe1wxovVEoAtSHTc=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
-github.com/Shopify/sarama v1.29.0 h1:ARid8o8oieau9XrHI55f/L3EoRAhm9px6sonbD7yuUE=
-github.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3TWSVqrU=
-github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
@@ -350,12 +344,6 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
-github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
-github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -376,13 +364,12 @@ github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNp
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
-github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
-github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
+github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
+github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -425,7 +412,6 @@ github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL9
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -494,7 +480,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo=
@@ -512,8 +497,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -547,10 +533,6 @@ github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu
 github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -577,8 +559,6 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -664,18 +644,6 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj
 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
-github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
-github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
-github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
-github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
-github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
-github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
-github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
-github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
-github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA=
-github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
-github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
-github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -744,8 +712,6 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
-github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
 github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -758,8 +724,9 @@ github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfo
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@@ -1016,8 +983,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220128100033-8d79e0c35e32 h1:DiWPsGAYMlBQq/urm7TJkIeSf9FnfzegcaQUpgwIbUs=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220128100033-8d79e0c35e32/go.mod h1:qFvhfbQ5orQxlH9vCiFnP4dW27xxnWHdNUBKyj/fbiY=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220214133635-20632dd262ed h1:R8EiLWArq7KT96DrUq1xq9scPh8vLwKKeCTnORPyjhU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220214133635-20632dd262ed/go.mod h1:qFvhfbQ5orQxlH9vCiFnP4dW27xxnWHdNUBKyj/fbiY=
 github.com/matrix-org/pinecone v0.0.0-20220121094951-351265543ddf h1:/nqfHUdQHr3WVdbZieaYFvHF1rin5pvDTa/NOZ/qCyE=
 github.com/matrix-org/pinecone v0.0.0-20220121094951-351265543ddf/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@@ -1155,19 +1122,18 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/nats-io/jwt/v2 v2.2.0 h1:Yg/4WFK6vsqMudRg91eBb7Dh6XeVcDMPHycDE8CfltE=
-github.com/nats-io/jwt/v2 v2.2.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
+github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 h1:vU9tpM3apjYlLLeY23zRWJ9Zktr5jp+mloR942LEOpY=
+github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
 github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
 github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/neilalexander/nats-server/v2 v2.3.3-0.20220104162330-c76d5fd70423 h1:BLQVdjMH5XD4BYb0fa+c2Oh2Nr1vrO7GKvRnIJDxChc=
-github.com/neilalexander/nats-server/v2 v2.3.3-0.20220104162330-c76d5fd70423/go.mod h1:9sdEkBhyZMQG1M9TevnlYUwMusRACn2vlgOeqoHKwVo=
+github.com/neilalexander/nats-server/v2 v2.7.2-0.20220217100407-087330ed46ad h1:Z2nWMQsXWWqzj89nW6OaLJSdkFknqhaR5whEOz4++Y8=
+github.com/neilalexander/nats-server/v2 v2.7.2-0.20220217100407-087330ed46ad/go.mod h1:tckmrt0M6bVaDT3kmh9UrIq/CBOBBse+TpXQi5ldaa8=
 github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c h1:G2qsv7D0rY94HAu8pXmElMluuMHQ85waxIDQBhIzV2Q=
 github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
 github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
@@ -1246,8 +1212,6 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
-github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1268,8 +1232,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg=
-github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -1301,12 +1265,12 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
-github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -1400,8 +1364,8 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
-github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
+github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
@@ -1456,8 +1420,6 @@ github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPyS
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
-github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
-github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
@@ -1540,17 +1502,16 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8 h1:kACShD3qhmr/3rLmg1yXyt+N4HcwutKyPRB93s54TIU=
-golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220209195652-db638375bc3a h1:atOEWVSedO4ksXBe/UrlbSLVxQQ9RxM/tT2Jy10IaHo=
+golang.org/x/crypto v0.0.0-20220209195652-db638375bc3a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1564,7 +1525,6 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
@@ -1646,7 +1606,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210427231257-85d9c07bbe3a/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -1776,8 +1735,10 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
+golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
+golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
@@ -1796,10 +1757,10 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M=
+golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1873,9 +1834,7 @@ golang.zx2c4.com/wireguard v0.0.0-20210604143328-f9b48a961cd2/go.mod h1:laHzsbfM
 golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326/go.mod h1:SDoazCvdy7RDjBPNEMBwrXhomlmtG7svs8mgwWEqtVI=
 golang.zx2c4.com/wireguard/windows v0.3.14/go.mod h1:3P4IEAsb+BjlKZmpUXgy74c0iX9AVwwr3WcVJ8nPgME=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
-gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM=
 gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
-gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=

internal/caching/cache_roomservernids.go

@@ -7,14 +7,6 @@ import (
 )
 
 const (
-	RoomServerStateKeyNIDsCacheName       = "roomserver_statekey_nids"
-	RoomServerStateKeyNIDsCacheMaxEntries = 1024
-	RoomServerStateKeyNIDsCacheMutable    = false
-
-	RoomServerEventTypeNIDsCacheName       = "roomserver_eventtype_nids"
-	RoomServerEventTypeNIDsCacheMaxEntries = 64
-	RoomServerEventTypeNIDsCacheMutable    = false
-
 	RoomServerRoomIDsCacheName       = "roomserver_room_ids"
 	RoomServerRoomIDsCacheMaxEntries = 1024
 	RoomServerRoomIDsCacheMutable    = false
@@ -29,44 +21,10 @@ type RoomServerCaches interface {
 // RoomServerNIDsCache contains the subset of functions needed for
 // a roomserver NID cache.
 type RoomServerNIDsCache interface {
-	GetRoomServerStateKeyNID(stateKey string) (types.EventStateKeyNID, bool)
-	StoreRoomServerStateKeyNID(stateKey string, nid types.EventStateKeyNID)
-
-	GetRoomServerEventTypeNID(eventType string) (types.EventTypeNID, bool)
-	StoreRoomServerEventTypeNID(eventType string, nid types.EventTypeNID)
-
 	GetRoomServerRoomID(roomNID types.RoomNID) (string, bool)
 	StoreRoomServerRoomID(roomNID types.RoomNID, roomID string)
 }
 
-func (c Caches) GetRoomServerStateKeyNID(stateKey string) (types.EventStateKeyNID, bool) {
-	val, found := c.RoomServerStateKeyNIDs.Get(stateKey)
-	if found && val != nil {
-		if stateKeyNID, ok := val.(types.EventStateKeyNID); ok {
-			return stateKeyNID, true
-		}
-	}
-	return 0, false
-}
-
-func (c Caches) StoreRoomServerStateKeyNID(stateKey string, nid types.EventStateKeyNID) {
-	c.RoomServerStateKeyNIDs.Set(stateKey, nid)
-}
-
-func (c Caches) GetRoomServerEventTypeNID(eventType string) (types.EventTypeNID, bool) {
-	val, found := c.RoomServerEventTypeNIDs.Get(eventType)
-	if found && val != nil {
-		if eventTypeNID, ok := val.(types.EventTypeNID); ok {
-			return eventTypeNID, true
-		}
-	}
-	return 0, false
-}
-
-func (c Caches) StoreRoomServerEventTypeNID(eventType string, nid types.EventTypeNID) {
-	c.RoomServerEventTypeNIDs.Set(eventType, nid)
-}
-
 func (c Caches) GetRoomServerRoomID(roomNID types.RoomNID) (string, bool) {
 	val, found := c.RoomServerRoomIDs.Get(strconv.Itoa(int(roomNID)))
 	if found && val != nil {

internal/caching/caches.go

@@ -4,14 +4,12 @@ package caching
 // different implementations as long as they satisfy the Cache
 // interface.
 type Caches struct {
-	RoomVersions            Cache // RoomVersionCache
-	ServerKeys              Cache // ServerKeyCache
-	RoomServerStateKeyNIDs  Cache // RoomServerNIDsCache
-	RoomServerEventTypeNIDs Cache // RoomServerNIDsCache
-	RoomServerRoomNIDs      Cache // RoomServerNIDsCache
-	RoomServerRoomIDs       Cache // RoomServerNIDsCache
-	RoomInfos               Cache // RoomInfoCache
-	FederationEvents        Cache // FederationEventsCache
+	RoomVersions       Cache // RoomVersionCache
+	ServerKeys         Cache // ServerKeyCache
+	RoomServerRoomNIDs Cache // RoomServerNIDsCache
+	RoomServerRoomIDs  Cache // RoomServerNIDsCache
+	RoomInfos          Cache // RoomInfoCache
+	FederationEvents   Cache // FederationEventsCache
 }
 
 // Cache is the interface that an implementation must satisfy.

internal/caching/impl_inmemorylru.go

@@ -28,24 +28,6 @@ func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
 	if err != nil {
 		return nil, err
 	}
-	roomServerStateKeyNIDs, err := NewInMemoryLRUCachePartition(
-		RoomServerStateKeyNIDsCacheName,
-		RoomServerStateKeyNIDsCacheMutable,
-		RoomServerStateKeyNIDsCacheMaxEntries,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
-	roomServerEventTypeNIDs, err := NewInMemoryLRUCachePartition(
-		RoomServerEventTypeNIDsCacheName,
-		RoomServerEventTypeNIDsCacheMutable,
-		RoomServerEventTypeNIDsCacheMaxEntries,
-		enablePrometheus,
-	)
-	if err != nil {
-		return nil, err
-	}
 	roomServerRoomIDs, err := NewInMemoryLRUCachePartition(
 		RoomServerRoomIDsCacheName,
 		RoomServerRoomIDsCacheMutable,
@@ -74,18 +56,15 @@ func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) {
 		return nil, err
 	}
 	go cacheCleaner(
-		roomVersions, serverKeys, roomServerStateKeyNIDs,
-		roomServerEventTypeNIDs, roomServerRoomIDs,
+		roomVersions, serverKeys, roomServerRoomIDs,
 		roomInfos, federationEvents,
 	)
 	return &Caches{
-		RoomVersions:            roomVersions,
-		ServerKeys:              serverKeys,
-		RoomServerStateKeyNIDs:  roomServerStateKeyNIDs,
-		RoomServerEventTypeNIDs: roomServerEventTypeNIDs,
-		RoomServerRoomIDs:       roomServerRoomIDs,
-		RoomInfos:               roomInfos,
-		FederationEvents:        federationEvents,
+		RoomVersions:      roomVersions,
+		ServerKeys:        serverKeys,
+		RoomServerRoomIDs: roomServerRoomIDs,
+		RoomInfos:         roomInfos,
+		FederationEvents:  federationEvents,
 	}, nil
 }
 

internal/consumers.go

@@ -1,139 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package internal
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/Shopify/sarama"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/sirupsen/logrus"
-)
-
-// A PartitionStorer has the storage APIs needed by the consumer.
-type PartitionStorer interface {
-	// PartitionOffsets returns the offsets the consumer has reached for each partition.
-	PartitionOffsets(ctx context.Context, topic string) ([]sqlutil.PartitionOffset, error)
-	// SetPartitionOffset records where the consumer has reached for a partition.
-	SetPartitionOffset(ctx context.Context, topic string, partition int32, offset int64) error
-}
-
-// A ContinualConsumer continually consumes logs even across restarts. It requires a PartitionStorer to
-// remember the offset it reached.
-type ContinualConsumer struct {
-	// The parent context for the listener, stop consuming when this context is done
-	Process *process.ProcessContext
-	// The component name
-	ComponentName string
-	// The kafkaesque topic to consume events from.
-	// This is the name used in kafka to identify the stream to consume events from.
-	Topic string
-	// A kafkaesque stream consumer providing the APIs for talking to the event source.
-	// The interface is taken from a client library for Apache Kafka.
-	// But any equivalent event streaming protocol could be made to implement the same interface.
-	Consumer sarama.Consumer
-	// A thing which can load and save partition offsets for a topic.
-	PartitionStore PartitionStorer
-	// ProcessMessage is a function which will be called for each message in the log. Return an error to
-	// stop processing messages. See ErrShutdown for specific control signals.
-	ProcessMessage func(msg *sarama.ConsumerMessage) error
-	// ShutdownCallback is called when ProcessMessage returns ErrShutdown, after the partition has been saved.
-	// It is optional.
-	ShutdownCallback func()
-}
-
-// ErrShutdown can be returned from ContinualConsumer.ProcessMessage to stop the ContinualConsumer.
-var ErrShutdown = fmt.Errorf("shutdown")
-
-// Start starts the consumer consuming.
-// Starts up a goroutine for each partition in the kafka stream.
-// Returns nil once all the goroutines are started.
-// Returns an error if it can't start consuming for any of the partitions.
-func (c *ContinualConsumer) Start() error {
-	_, err := c.StartOffsets()
-	return err
-}
-
-// StartOffsets is the same as Start but returns the loaded offsets as well.
-func (c *ContinualConsumer) StartOffsets() ([]sqlutil.PartitionOffset, error) {
-	offsets := map[int32]int64{}
-
-	partitions, err := c.Consumer.Partitions(c.Topic)
-	if err != nil {
-		return nil, err
-	}
-	for _, partition := range partitions {
-		// Default all the offsets to the beginning of the stream.
-		offsets[partition] = sarama.OffsetOldest
-	}
-
-	storedOffsets, err := c.PartitionStore.PartitionOffsets(context.TODO(), c.Topic)
-	if err != nil {
-		return nil, err
-	}
-	for _, offset := range storedOffsets {
-		// We've already processed events from this partition so advance the offset to where we got to.
-		// ConsumePartition will start streaming from the message with the given offset (inclusive),
-		// so increment 1 to avoid getting the same message a second time.
-		offsets[offset.Partition] = 1 + offset.Offset
-	}
-
-	var partitionConsumers []sarama.PartitionConsumer
-	for partition, offset := range offsets {
-		pc, err := c.Consumer.ConsumePartition(c.Topic, partition, offset)
-		if err != nil {
-			for _, p := range partitionConsumers {
-				p.Close() // nolint: errcheck
-			}
-			return nil, err
-		}
-		partitionConsumers = append(partitionConsumers, pc)
-	}
-	for _, pc := range partitionConsumers {
-		go c.consumePartition(pc)
-		if c.Process != nil {
-			c.Process.ComponentStarted()
-			go func(pc sarama.PartitionConsumer) {
-				<-c.Process.WaitForShutdown()
-				_ = pc.Close()
-				c.Process.ComponentFinished()
-				logrus.Infof("Stopped consumer for %q topic %q", c.ComponentName, c.Topic)
-			}(pc)
-		}
-	}
-
-	return storedOffsets, nil
-}
-
-// consumePartition consumes the room events for a single partition of the kafkaesque stream.
-func (c *ContinualConsumer) consumePartition(pc sarama.PartitionConsumer) {
-	defer pc.Close() // nolint: errcheck
-	for message := range pc.Messages() {
-		msgErr := c.ProcessMessage(message)
-		// Advance our position in the stream so that we will start at the right position after a restart.
-		if err := c.PartitionStore.SetPartitionOffset(context.TODO(), c.Topic, message.Partition, message.Offset); err != nil {
-			panic(fmt.Errorf("the ContinualConsumer in %q failed to SetPartitionOffset: %w", c.ComponentName, err))
-		}
-		// Shutdown if we were told to do so.
-		if msgErr == ErrShutdown {
-			if c.ShutdownCallback != nil {
-				c.ShutdownCallback()
-			}
-			return
-		}
-	}
-}

internal/httputil/httpapi.go

@@ -53,12 +53,13 @@ func MakeAuthAPI(
 	f func(*http.Request, *userapi.Device) util.JSONResponse,
 ) http.Handler {
 	h := func(req *http.Request) util.JSONResponse {
+		logger := util.GetLogger(req.Context())
 		device, err := auth.VerifyUserFromRequest(req, userAPI)
 		if err != nil {
+			logger.Debugf("VerifyUserFromRequest %s -> HTTP %d", req.RemoteAddr, err.Code)
 			return *err
 		}
 		// add the user ID to the logger
-		logger := util.GetLogger((req.Context()))
 		logger = logger.WithField("user_id", device.UserID)
 		req = req.WithContext(util.ContextWithLogger(req.Context(), logger))
 		// add the user to Sentry, if enabled

internal/test/config.go

@@ -20,6 +20,7 @@ import (
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/big"
@@ -94,7 +95,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(database)
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(database)
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database)
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(database)
 
 	cfg.AppServiceAPI.InternalAPI.Listen = assignAddress()
 	cfg.EDUServer.InternalAPI.Listen = assignAddress()
@@ -158,11 +158,10 @@ func NewMatrixKey(matrixKeyPath string) (err error) {
 
 const certificateDuration = time.Hour * 24 * 365 * 10
 
-// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
-func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
+func generateTLSTemplate(dnsNames []string) (*rsa.PrivateKey, *x509.Certificate, error) {
 	priv, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	notBefore := time.Now()
@@ -170,7 +169,7 @@ func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
 	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
 	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	template := x509.Certificate{
@@ -180,20 +179,21 @@ func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		BasicConstraintsValid: true,
+		DNSNames:              dnsNames,
 	}
-	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	if err != nil {
-		return err
-	}
+	return priv, &template, nil
+}
+
+func writeCertificate(tlsCertPath string, derBytes []byte) error {
 	certOut, err := os.Create(tlsCertPath)
 	if err != nil {
 		return err
 	}
 	defer certOut.Close() // nolint: errcheck
-	if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
-		return err
-	}
+	return pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+}
 
+func writePrivateKey(tlsKeyPath string, priv *rsa.PrivateKey) error {
 	keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return err
@@ -205,3 +205,73 @@ func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
 	})
 	return err
 }
+
+// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
+func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
+	priv, template, err := generateTLSTemplate(nil)
+	if err != nil {
+		return err
+	}
+
+	// Self-signed certificate: template == parent
+	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
+	if err != nil {
+		return err
+	}
+
+	if err = writeCertificate(tlsCertPath, derBytes); err != nil {
+		return err
+	}
+	return writePrivateKey(tlsKeyPath, priv)
+}
+
+func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string) error {
+	priv, template, err := generateTLSTemplate([]string{serverName})
+	if err != nil {
+		return err
+	}
+
+	// load the authority key
+	dat, err := ioutil.ReadFile(authorityKeyPath)
+	if err != nil {
+		return err
+	}
+	block, _ := pem.Decode([]byte(dat))
+	if block == nil || block.Type != "RSA PRIVATE KEY" {
+		return errors.New("authority .key is not a valid pem encoded rsa private key")
+	}
+	authorityPriv, err := x509.ParsePKCS1PrivateKey(block.Bytes)
+	if err != nil {
+		return err
+	}
+
+	// load the authority certificate
+	dat, err = ioutil.ReadFile(authorityCertPath)
+	if err != nil {
+		return err
+	}
+	block, _ = pem.Decode([]byte(dat))
+	if block == nil || block.Type != "CERTIFICATE" {
+		return errors.New("authority .crt is not a valid pem encoded x509 cert")
+	}
+	var caCerts []*x509.Certificate
+	caCerts, err = x509.ParseCertificates(block.Bytes)
+	if err != nil {
+		return err
+	}
+	if len(caCerts) != 1 {
+		return errors.New("authority .crt contains none or more than one cert")
+	}
+	authorityCert := caCerts[0]
+
+	// Sign the new certificate using the authority's key/cert
+	derBytes, err := x509.CreateCertificate(rand.Reader, template, authorityCert, &priv.PublicKey, authorityPriv)
+	if err != nil {
+		return err
+	}
+
+	if err = writeCertificate(tlsCertPath, derBytes); err != nil {
+		return err
+	}
+	return writePrivateKey(tlsKeyPath, priv)
+}

internal/version.go

@@ -16,8 +16,8 @@ var build string
 
 const (
 	VersionMajor = 0
-	VersionMinor = 5
-	VersionPatch = 1
+	VersionMinor = 6
+	VersionPatch = 3
 	VersionTag   = "" // example: "rc1"
 )
 

keyserver/api/api.go

@@ -228,7 +228,7 @@ type QueryKeyChangesRequest struct {
 	// The offset of the last received key event, or sarama.OffsetOldest if this is from the beginning
 	Offset int64
 	// The inclusive offset where to track key changes up to. Messages with this offset are included in the response.
-	// Use sarama.OffsetNewest if the offset is unknown (then check the response Offset to avoid racing).
+	// Use types.OffsetNewest if the offset is unknown (then check the response Offset to avoid racing).
 	ToOffset int64
 }
 

keyserver/consumers/cross_signing.go

@@ -1,118 +0,0 @@
-// Copyright 2021 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package consumers
-
-import (
-	"context"
-	"encoding/json"
-
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/keyserver/api"
-	"github.com/matrix-org/dendrite/keyserver/storage"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
-	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
-
-	"github.com/Shopify/sarama"
-)
-
-type OutputCrossSigningKeyUpdateConsumer struct {
-	eduServerConsumer *internal.ContinualConsumer
-	keyDB             storage.Database
-	keyAPI            api.KeyInternalAPI
-	serverName        string
-}
-
-func NewOutputCrossSigningKeyUpdateConsumer(
-	process *process.ProcessContext,
-	cfg *config.Dendrite,
-	kafkaConsumer sarama.Consumer,
-	keyDB storage.Database,
-	keyAPI api.KeyInternalAPI,
-) *OutputCrossSigningKeyUpdateConsumer {
-	// The keyserver both produces and consumes on the TopicOutputKeyChangeEvent
-	// topic. We will only produce events where the UserID matches our server name,
-	// and we will only consume events where the UserID does NOT match our server
-	// name (because the update came from a remote server).
-	consumer := internal.ContinualConsumer{
-		Process:        process,
-		ComponentName:  "keyserver/keyserver",
-		Topic:          cfg.Global.JetStream.TopicFor(jetstream.OutputKeyChangeEvent),
-		Consumer:       kafkaConsumer,
-		PartitionStore: keyDB,
-	}
-	s := &OutputCrossSigningKeyUpdateConsumer{
-		eduServerConsumer: &consumer,
-		keyDB:             keyDB,
-		keyAPI:            keyAPI,
-		serverName:        string(cfg.Global.ServerName),
-	}
-	consumer.ProcessMessage = s.onMessage
-
-	return s
-}
-
-func (s *OutputCrossSigningKeyUpdateConsumer) Start() error {
-	return s.eduServerConsumer.Start()
-}
-
-// onMessage is called in response to a message received on the
-// key change events topic from the key server.
-func (t *OutputCrossSigningKeyUpdateConsumer) onMessage(msg *sarama.ConsumerMessage) error {
-	var m api.DeviceMessage
-	if err := json.Unmarshal(msg.Value, &m); err != nil {
-		logrus.WithError(err).Errorf("failed to read device message from key change topic")
-		return nil
-	}
-	if m.OutputCrossSigningKeyUpdate == nil {
-		// This probably shouldn't happen but stops us from panicking if we come
-		// across an update that doesn't satisfy either types.
-		return nil
-	}
-	switch m.Type {
-	case api.TypeCrossSigningUpdate:
-		return t.onCrossSigningMessage(m)
-	default:
-		return nil
-	}
-}
-
-func (s *OutputCrossSigningKeyUpdateConsumer) onCrossSigningMessage(m api.DeviceMessage) error {
-	output := m.CrossSigningKeyUpdate
-	_, host, err := gomatrixserverlib.SplitID('@', output.UserID)
-	if err != nil {
-		logrus.WithError(err).Errorf("eduserver output log: user ID parse failure")
-		return nil
-	}
-	if host == gomatrixserverlib.ServerName(s.serverName) {
-		// Ignore any messages that contain information about our own users, as
-		// they already originated from this server.
-		return nil
-	}
-	uploadReq := &api.PerformUploadDeviceKeysRequest{
-		UserID: output.UserID,
-	}
-	if output.MasterKey != nil {
-		uploadReq.MasterKey = *output.MasterKey
-	}
-	if output.SelfSigningKey != nil {
-		uploadReq.SelfSigningKey = *output.SelfSigningKey
-	}
-	uploadRes := &api.PerformUploadDeviceKeysResponse{}
-	s.keyAPI.PerformUploadDeviceKeys(context.TODO(), uploadReq, uploadRes)
-	return uploadRes.Error
-}

keyserver/internal/cross_signing.go

@@ -219,25 +219,23 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P
 	}
 
 	// Finally, generate a notification that we updated the keys.
-	if _, host, err := gomatrixserverlib.SplitID('@', req.UserID); err == nil && host == a.ThisServer {
-		update := eduserverAPI.CrossSigningKeyUpdate{
-			UserID: req.UserID,
-		}
-		if mk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster]; ok {
-			update.MasterKey = &mk
-		}
-		if ssk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning]; ok {
-			update.SelfSigningKey = &ssk
-		}
-		if update.MasterKey == nil && update.SelfSigningKey == nil {
-			return
-		}
-		if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil {
-			res.Error = &api.KeyError{
-				Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err),
-			}
-			return
+	update := eduserverAPI.CrossSigningKeyUpdate{
+		UserID: req.UserID,
+	}
+	if mk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster]; ok {
+		update.MasterKey = &mk
+	}
+	if ssk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning]; ok {
+		update.SelfSigningKey = &ssk
+	}
+	if update.MasterKey == nil && update.SelfSigningKey == nil {
+		return
+	}
+	if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil {
+		res.Error = &api.KeyError{
+			Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err),
 		}
+		return
 	}
 }
 
@@ -310,16 +308,18 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req
 
 	// Finally, generate a notification that we updated the signatures.
 	for userID := range req.Signatures {
-		if _, host, err := gomatrixserverlib.SplitID('@', userID); err == nil && host == a.ThisServer {
-			update := eduserverAPI.CrossSigningKeyUpdate{
-				UserID: userID,
-			}
-			if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil {
-				res.Error = &api.KeyError{
-					Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err),
-				}
-				return
+		masterKey := queryRes.MasterKeys[userID]
+		selfSigningKey := queryRes.SelfSigningKeys[userID]
+		update := eduserverAPI.CrossSigningKeyUpdate{
+			UserID:         userID,
+			MasterKey:      &masterKey,
+			SelfSigningKey: &selfSigningKey,
+		}
+		if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil {
+			res.Error = &api.KeyError{
+				Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err),
 			}
+			return
 		}
 	}
 }

keyserver/internal/device_list_update.go

@@ -367,10 +367,13 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam
 					waitTime = fcerr.RetryAfter
 				} else if fcerr.Blacklisted {
 					waitTime = time.Hour * 8
+				} else {
+					// For all other errors (DNS resolution, network etc.) wait 1 hour.
+					waitTime = time.Hour
 				}
 			} else {
 				waitTime = time.Hour
-				logger.WithError(err).Warn("GetUserDevices returned unknown error type")
+				logger.WithError(err).WithField("user_id", userID).Warn("GetUserDevices returned unknown error type")
 			}
 			continue
 		}