Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/fts2

.github/workflows/dendrite.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -19,12 +20,12 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ false }} # disable for now
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.16
+          go-version: 1.18
 
       - uses: actions/cache@v2
         with:
@@ -66,8 +67,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
 
   # run go test with different go versions
   test:
@@ -97,11 +102,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.16", "1.17", "1.18"]
+        go: ["1.18", "1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - uses: actions/cache@v3
@@ -127,13 +132,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.16", "1.17", "1.18"]
+        go: ["1.18", "1.19"]
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies x86
@@ -151,6 +156,7 @@ jobs:
           GOOS: ${{ matrix.goos }}
           GOARCH: ${{ matrix.goarch }}
           CGO_ENABLED: 1
+          CGO_CFLAGS: -fno-stack-protector
         run: go build -trimpath -v -o "bin/" ./cmd/...
 
   # build for Windows 64-bit
@@ -160,13 +166,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.16", "1.17", "1.18"]
+        go: ["1.18", "1.19"]
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup Go ${{ matrix.go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
       - name: Install dependencies
@@ -207,9 +213,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: "1.16"
+          go-version: "1.18"
       - uses: actions/cache@v3
         with:
           path: |
@@ -223,6 +229,31 @@ jobs:
       - name: Test upgrade
         run: ./dendrite-upgrade-tests --head .
 
+  # run database upgrade tests, skipping over one version
+  upgrade_test_direct:
+    name: Upgrade tests from HEAD-2
+    timeout-minutes: 20
+    needs: initial-tests-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.18"
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade
+      - name: Build upgrade-tests
+        run: go build ./cmd/dendrite-upgrade-tests
+      - name: Test upgrade
+        run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
+
   # run Sytest in different variations
   sytest:
     timeout-minutes: 20
@@ -345,6 +376,8 @@ jobs:
       # Build initial Dendrite image
       - run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
         working-directory: dendrite
+        env:
+          DOCKER_BUILDKIT: 1
 
       # Run Complement
       - run: |
@@ -359,7 +392,14 @@ jobs:
 
   integration-tests-done:
     name: Integration tests passed
-    needs: [initial-tests-done, upgrade_test, sytest, complement]
+    needs:
+      [
+        initial-tests-done,
+        upgrade_test,
+        upgrade_test_direct,
+        sytest,
+        complement,
+      ]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
     steps:

CHANGES.md

@@ -1,5 +1,109 @@
 # Changelog
 
+## Dendrite 0.9.5 (2022-08-25)
+
+### Fixes
+
+* The roomserver will now correctly unreject previously rejected events if necessary when reprocessing
+* The handling of event soft-failure has been improved on the roomserver input by no longer applying rejection rules and still calculating state before the event if possible
+* The federation `/state` and `/state_ids` endpoints should now return the correct error code when the state isn't known instead of returning a HTTP 500
+* The federation `/event` should now return outlier events correctly instead of returning a HTTP 500
+* A bug in the federation backoff allowing zero intervals has been corrected
+* The `create-account` utility will no longer error if the homeserver URL ends in a trailing slash
+* A regression in `/sync` introduced in 0.9.4 should be fixed
+
+## Dendrite 0.9.4 (2022-08-19)
+
+### Fixes
+
+* A bug in the roomserver around handling rejected outliers has been fixed
+* Backfilled events will now use the correct history visibility where possible
+* The device list updater backoff has been fixed, which should reduce the number of outbound HTTP requests and `Failed to query device keys for some users` log entries for dead servers
+* The `/sync` endpoint will no longer incorrectly return room entries for retired invites which could cause some rooms to show up in the client "Historical" section
+* The `/createRoom` endpoint will now correctly populate `is_direct` in invite membership events, which may help clients to classify direct messages correctly
+* The `create-account` tool will now log an error if the shared secret is not set in the Dendrite config
+* A couple of minor bugs have been fixed in the membership lazy-loading
+* Queued EDUs in the federation API are now cached properly
+
+## Dendrite 0.9.3 (2022-08-15)
+
+### Important
+
+* This is a **security release** to fix a vulnerability within event auth, affecting all versions of Dendrite before 0.9.3. Upgrading to this version is highly recommended. For more information, [see here](https://github.com/matrix-org/gomatrixserverlib/security/advisories/GHSA-grvv-h2f9-7v9c).
+
+### Fixes
+
+* Dendrite will now correctly parse the `"events_default"` power level value for event auth.
+
+## Dendrite 0.9.2 (2022-08-12)
+
+### Features
+
+* Dendrite now supports history visibility on the `/sync`, `/messages` and `/context` endpoints
+  * It should now be possible to view the history of a room in more cases (as opposed to limiting scrollback to the join event or defaulting to the restrictive `"join"` visibility rule as before)
+* The default room version for newly created rooms is now room version 9
+* New admin endpoint `/_dendrite/admin/resetPassword/{userID}` has been added, which replaces the `-reset-password` flag in `create-account`
+* The `create-account` binary now uses shared secret registration over HTTP to create new accounts, which fixes a number of problems with account data and push rules not being configured correctly for new accounts
+* The internal HTTP APIs for polylith deployments have been refactored for correctness and consistency
+* The federation API will now automatically clean up some EDUs that have failed to send within a certain period of time
+* The `/hierarchy` endpoint will now return potentially joinable rooms (contributed by [texuf](https://github.com/texuf))
+* The user directory will now show or hide users correctly
+
+### Fixes
+
+* Send-to-device messages should no longer be incorrectly duplicated in `/sync`
+* The federation sender will no longer create unnecessary destination queues as a result of a logic error
+* A bug where database migrations may not execute properly when upgrading from older versions has been fixed
+* A crash when failing to update user account data has been fixed
+* A race condition when generating notification counts has been fixed
+* A race condition when setting up NATS has been fixed (contributed by [brianathere](https://github.com/brianathere))
+* Stale cache data for membership lazy-loading is now correctly invalidated when doing a complete sync
+* Data races within user-interactive authentication have been fixed (contributed by [tak-hntlabs](https://github.com/tak-hntlabs))
+
+## Dendrite 0.9.1 (2022-08-03)
+
+### Fixes
+
+* Upgrades a dependency which caused issues building Dendrite with Go 1.19
+* The roomserver will no longer give up prematurely after failing to call `/state_ids`
+* Removes the faulty room info cache, which caused of a number of race conditions and occasional bugs (including when creating and joining rooms)
+* The media endpoint now sets the `Cache-Control` header correctly to prevent web-based clients from hitting media endpoints excessively
+* The sync API will now advance the PDU stream position correctly in all cases (contributed by [sergekh2](https://github.com/sergekh2))
+* The sync API will now delete the correct range of send-to-device messages when advancing the stream position
+* The device list `changed` key in the `/sync` response should now return the correct users
+* A data race when looking up missing state has been fixed
+* The `/send_join` API is now applying stronger validation to the received membership event
+
+## Dendrite 0.9.0 (2022-08-01)
+
+### Features
+
+* Dendrite now uses Ristretto for managing in-memory caches
+  * Should improve cache utilisation considerably over time by more intelligently selecting and managing cache entries compared to the previous LRU-based cache
+  * Defaults to a 1GB cache size if not configured otherwise
+  * The estimated cache size in memory and maximum age can now be configured with new [configuration options](https://github.com/matrix-org/dendrite/blob/e94ef84aaba30e12baf7f524c4e7a36d2fdeb189/dendrite-sample.monolith.yaml#L44-L61) to prevent unbounded cache growth
+* Added support for serving the `/.well-known/matrix/client` hint directly from Dendrite
+  * Configurable with the new [configuration option](https://github.com/matrix-org/dendrite/blob/e94ef84aaba30e12baf7f524c4e7a36d2fdeb189/dendrite-sample.monolith.yaml#L67-L69)
+* Refactored membership updater, which should eliminate some bugs caused by the membership table getting out of sync with the room state
+* The User API is now responsible for sending account data updates to other components, which may fix some races and duplicate account data events
+* Optimised database query for checking whether a remote server is allowed to request an event over federation without using anywhere near as much CPU time (PostgreSQL only)
+* Database migrations have been refactored to eliminate some problems that were present with `goose` and upgrading from older Dendrite versions
+* Media fetching will now use the `/v3` endpoints for downloading media from remote homeservers
+* HTTP 404 and HTTP 405 errors from the client-facing APIs should now be returned with CORS headers so that web-based clients do not produce incorrect access control warnings for unknown endpoints
+* Some preparation work for full history visibility support
+
+### Fixes
+
+* Fixes a crash that could occur during event redaction
+* The `/members` endpoint will no longer incorrectly return HTTP 500 as a result of some invite events
+* Send-to-device messages should now be ordered more reliably and the last position in the stream updated correctly
+* Parsing of appservice configuration files is now less strict (contributed by [Kab1r](https://github.com/Kab1r))
+* The sync API should now identify shared users correctly when waking up for E2EE key changes
+* The federation `/state` endpoint will now return a HTTP 403 when the state before an event isn't known instead of a HTTP 500
+* Presence timestamps should now be calculated with the correct precision
+* A race condition in the roomserver's room info has been fixed
+* A race condition in the sync API has been fixed
+
 ## Dendrite 0.8.9 (2022-07-01)
 
 ### Features

README.md

@@ -21,8 +21,7 @@ As of October 2020 (current [progress below](#progress)), Dendrite has now enter
 This does not mean:
 
 - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
-- All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular,
+- Dendrite is feature-complete. There may be client or federation APIs that are not implemented.
-  presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
 - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
@@ -36,7 +35,10 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
 
 ## Requirements
 
-To build Dendrite, you will need Go 1.16 or later.
+See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
+more information on requirements.
+
+To build Dendrite, you will need Go 1.18 or later.
 
 For a usable federating Dendrite deployment, you will also need:
 
@@ -78,21 +80,21 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
 
 # Create an user account (add -admin for an admin user).
 # Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
-$ ./bin/create-account --config dendrite.yaml -username alice
+$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice
 ```
 
 Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
 
-## <a id="progress"></a> Progress
+## Progress
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of April 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well, although there are still some missing features (like Search).
 
 We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
-than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API).
+than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API).
 This means Dendrite supports amongst others:
 
 - Core room functionality (creating rooms, invites, auth rules)
@@ -119,53 +121,8 @@ We would be grateful for any help on issues marked as
 all have related Sytests which need to pass in order for the issue to be closed. Once you've written your
 code, you can quickly run Sytest to ensure that the test names are now passing.
 
-For example, if the test `Local device key changes get to remote servers` was marked as failing, find the
+If you're new to the project, see our
-test file (e.g via `grep` or via the
+[Contributing page](https://matrix-org.github.io/dendrite/development/contributing) to get up to speed, then
-[CI log output](https://buildkite.com/matrix-dot-org/dendrite/builds/2826#39cff5de-e032-4ad0-ad26-f819e6919c42)
-it's `tests/50federation/40devicelists.pl` ) then to run Sytest:
-
-```
-docker run --rm --name sytest
--v "/Users/kegan/github/sytest:/sytest"
--v "/Users/kegan/github/dendrite:/src"
--v "/Users/kegan/logs:/logs"
--v "/Users/kegan/go/:/gopath"
--e "POSTGRES=1" -e "DENDRITE_TRACE_HTTP=1"
-matrixdotorg/sytest-dendrite:latest tests/50federation/40devicelists.pl
-```
-
-See [sytest.md](docs/sytest.md) for the full description of these flags.
-
-You can try running sytest outside of docker for faster runs, but the dependencies can be temperamental
-and we recommend using docker where possible.
-
-```
-cd sytest
-export PERL5LIB=$HOME/lib/perl5
-export PERL_MB_OPT=--install_base=$HOME
-export PERL_MM_OPT=INSTALL_BASE=$HOME
-./install-deps.pl
-
-./run-tests.pl -I Dendrite::Monolith -d $PATH_TO_DENDRITE_BINARIES
-```
-
-Sometimes Sytest is testing the wrong thing or is flakey, so it will need to be patched.
-Ask on `#dendrite-dev:matrix.org` if you think this is the case for you and we'll be happy to help.
-
-If you're new to the project, see [CONTRIBUTING.md](docs/CONTRIBUTING.md) to get up to speed then
 look for [Good First Issues](https://github.com/matrix-org/dendrite/labels/good%20first%20issue). If you're
 familiar with the project, look for [Help Wanted](https://github.com/matrix-org/dendrite/labels/help-wanted)
 issues.
-
-## Hardware requirements
-
-Dendrite in Monolith + SQLite works in a range of environments including iOS and in-browser via WASM.
-
-For small homeserver installations joined on ~10s rooms on matrix.org with ~100s of users in those rooms, including some
-encrypted rooms:
-
-- Memory: uses around 100MB of RAM, with peaks at around 200MB.
-- Disk space: After a few months of usage, the database grew to around 2GB (in Monolith mode).
-- CPU: Brief spikes when processing events, typically idles at 1% CPU.
-
-This means Dendrite should comfortably work on things like Raspberry Pis.

appservice/inthttp/client.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/opentracing/opentracing-go"
 )
 
 // HTTP paths for the internal HTTP APIs
@@ -42,11 +41,10 @@ func (h *httpAppServiceQueryAPI) RoomAliasExists(
 	request *api.RoomAliasExistsRequest,
 	response *api.RoomAliasExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"RoomAliasExists", h.appserviceURL+AppServiceRoomAliasExistsPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // UserIDExists implements AppServiceQueryAPI
@@ -55,9 +53,8 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
 	request *api.UserIDExistsRequest,
 	response *api.UserIDExistsResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"UserIDExists", h.appserviceURL+AppServiceUserIDExistsPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.appserviceURL + AppServiceUserIDExistsPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }

appservice/inthttp/server.go

@@ -1,43 +1,20 @@
 package inthttp
 
 import (
-	"encoding/json"
-	"net/http"
-
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/util"
 )
 
 // AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
 func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		AppServiceRoomAliasExistsPath,
-		httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
-			var request api.RoomAliasExistsRequest
-			var response api.RoomAliasExistsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		AppServiceUserIDExistsPath,
-		httputil.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
-			var request api.UserIDExistsRequest
-			var response api.UserIDExistsResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := a.UserIDExists(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
 	)
 }

build/docker/Dockerfile.monolith

@@ -8,7 +8,6 @@ COPY . /build
 
 RUN mkdir -p bin
 RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
-RUN go build -trimpath -o bin/ ./cmd/goose
 RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 

build/docker/Dockerfile.polylith

@@ -8,7 +8,6 @@ COPY . /build
 
 RUN mkdir -p bin
 RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
-RUN go build -trimpath -o bin/ ./cmd/goose
 RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 

build/gobind-pinecone/monolith.go

@@ -239,7 +239,7 @@ func (m *DendriteMonolith) Start() {
 	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
 	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
 	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
-	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter)
+	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
 
 	prefix := hex.EncodeToString(pk)
 	cfg := &config.Dendrite{}

build/scripts/Complement.Dockerfile

@@ -1,4 +1,6 @@
-FROM golang:1.16-stretch as build
+#syntax=docker/dockerfile:1.2
+
+FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y sqlite3
 WORKDIR /build
 
@@ -8,14 +10,12 @@ RUN mkdir /dendrite
 
 # Utilise Docker caching when downloading dependencies, this stops us needlessly
 # downloading dependencies every time.
-COPY go.mod .
+RUN --mount=target=. \
-COPY go.sum .
+    --mount=type=cache,target=/go/pkg/mod \
-RUN go mod download
+    --mount=type=cache,target=/root/.cache/go-build \
-
+    go build -o /dendrite ./cmd/generate-config && \
-COPY . .
+    go build -o /dendrite ./cmd/generate-keys && \
-RUN go build -o /dendrite ./cmd/dendrite-monolith-server
+    go build -o /dendrite ./cmd/dendrite-monolith-server
-RUN go build -o /dendrite ./cmd/generate-keys
-RUN go build -o /dendrite ./cmd/generate-config
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
@@ -26,7 +26,7 @@ EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
-CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
+CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
- ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
- cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+    cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

build/scripts/ComplementLocal.Dockerfile

@@ -1,3 +1,5 @@
+#syntax=docker/dockerfile:1.2
+
 # A local development Complement dockerfile, to be used with host mounts
 # /cache -> Contains the entire dendrite code at Dockerfile build time. Builds binaries but only keeps the generate-* ones. Pre-compilation saves time.
 # /dendrite -> Host-mounted sources
@@ -6,48 +8,45 @@
 #
 # Use these mounts to make use of this dockerfile:
 # COMPLEMENT_HOST_MOUNTS='/your/local/dendrite:/dendrite:ro;/your/go/path:/go:ro'
-FROM golang:1.16-stretch
+FROM golang:1.18-stretch
 RUN apt-get update && apt-get install -y sqlite3
 
-WORKDIR /runtime
-
 ENV SERVER_NAME=localhost
 EXPOSE 8008 8448
 
+WORKDIR /runtime
 # This script compiles Dendrite for us.
 RUN echo '\
-#!/bin/bash -eux \n\
+    #!/bin/bash -eux \n\
-if test -f "/runtime/dendrite-monolith-server"; then \n\
+    if test -f "/runtime/dendrite-monolith-server"; then \n\
     echo "Skipping compilation; binaries exist" \n\
     exit 0 \n\
-fi \n\
+    fi \n\
-cd /dendrite \n\
+    cd /dendrite \n\
-go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
+    go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
-' > compile.sh && chmod +x compile.sh
+    ' > compile.sh && chmod +x compile.sh
 
 # This script runs Dendrite for us. Must be run in the /runtime directory.
 RUN echo '\
-#!/bin/bash -eu \n\
+    #!/bin/bash -eu \n\
-./generate-keys --private-key matrix_key.pem \n\
+    ./generate-keys --private-key matrix_key.pem \n\
-./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
+    ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
-./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
+    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
-cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
+    cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
-./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
-' > run.sh && chmod +x run.sh
+    ' > run.sh && chmod +x run.sh
 
 
 WORKDIR /cache
-# Pre-download deps; we don't need to do this if the GOPATH is mounted.
-COPY go.mod .
-COPY go.sum .
-RUN go mod download
-
 # Build the monolith in /cache - we won't actually use this but will rely on build artifacts to speed
 # up the real compilation. Build the generate-* binaries in the true /runtime locations.
 # If the generate-* source is changed, this dockerfile needs re-running.
-COPY . .
+RUN --mount=target=. \
-RUN go build ./cmd/dendrite-monolith-server && go build -o /runtime ./cmd/generate-keys && go build -o /runtime ./cmd/generate-config
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o /runtime ./cmd/generate-config && \
+    go build -o /runtime ./cmd/generate-keys
 
 
 WORKDIR /runtime
-CMD /runtime/compile.sh && /runtime/run.sh
+CMD /runtime/compile.sh && exec /runtime/run.sh

build/scripts/ComplementPostgres.Dockerfile

@@ -1,4 +1,6 @@
-FROM golang:1.16-stretch as build
+#syntax=docker/dockerfile:1.2
+
+FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
@@ -9,16 +11,16 @@ RUN sed -i "s%127.0.0.1/32            md5%127.0.0.1/32            trust%g" /etc/
 
 # This entry script starts postgres, waits for it to be up then starts dendrite
 RUN echo '\
-#!/bin/bash -eu \n\
+    #!/bin/bash -eu \n\
-pg_lsclusters \n\
+    pg_lsclusters \n\
-pg_ctlcluster 9.6 main start \n\
+    pg_ctlcluster 9.6 main start \n\
- \n\
+    \n\
-until pg_isready \n\
+    until pg_isready \n\
-do \n\
+    do \n\
-  echo "Waiting for postgres"; \n\
+    echo "Waiting for postgres"; \n\
-  sleep 1; \n\
+    sleep 1; \n\
-done \n\
+    done \n\
-' > run_postgres.sh && chmod +x run_postgres.sh
+    ' > run_postgres.sh && chmod +x run_postgres.sh
 
 # we will dump the binaries and config file to this location to ensure any local untracked files
 # that come from the COPY . . file don't contaminate the build
@@ -26,14 +28,12 @@ RUN mkdir /dendrite
 
 # Utilise Docker caching when downloading dependencies, this stops us needlessly
 # downloading dependencies every time.
-COPY go.mod .
+RUN --mount=target=. \
-COPY go.sum .
+    --mount=type=cache,target=/go/pkg/mod \
-RUN go mod download
+    --mount=type=cache,target=/root/.cache/go-build \
-
+    go build -o /dendrite ./cmd/generate-config && \
-COPY . .
+    go build -o /dendrite ./cmd/generate-keys && \
-RUN go build -o /dendrite ./cmd/dendrite-monolith-server
+    go build -o /dendrite ./cmd/dendrite-monolith-server
-RUN go build -o /dendrite ./cmd/generate-keys
-RUN go build -o /dendrite ./cmd/generate-config
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
@@ -45,10 +45,10 @@ EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
-CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
+CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
- ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
+    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
- # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
+    # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
- sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
+    sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
- sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
+    sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
- cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
+    cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
- ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

build/scripts/build-test-lint.sh

@@ -13,4 +13,4 @@ go build ./cmd/...
 ./build/scripts/find-lint.sh
 
 echo "Testing..."
-go test -v ./...
+go test --race -v ./...

clientapi/auth/login.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -34,7 +33,7 @@ import (
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
 func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
-	reqBytes, err := ioutil.ReadAll(r)
+	reqBytes, err := io.ReadAll(r)
 	if err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,

clientapi/auth/user_interactive.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/http"
+	"sync"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -102,6 +103,7 @@ type userInteractiveFlow struct {
 // the user already has a valid access token, but we want to double-check
 // that it isn't stolen by re-authenticating them.
 type UserInteractive struct {
+	sync.RWMutex
 	Flows []userInteractiveFlow
 	// Map of login type to implementation
 	Types map[string]Type
@@ -128,6 +130,8 @@ func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI)
 }
 
 func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
+	u.RLock()
+	defer u.RUnlock()
 	for _, f := range u.Flows {
 		if len(f.Stages) == 1 && f.Stages[0] == authType {
 			return true
@@ -137,8 +141,10 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
 }
 
 func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
+	u.Lock()
 	// TODO: Handle multi-stage flows
 	delete(u.Sessions, sessionID)
+	u.Unlock()
 }
 
 type Challenge struct {
@@ -150,12 +156,17 @@ type Challenge struct {
 }
 
 // Challenge returns an HTTP 401 with the supported flows for authenticating
-func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse {
+func (u *UserInteractive) challenge(sessionID string) *util.JSONResponse {
+	u.RLock()
+	completed := u.Sessions[sessionID]
+	flows := u.Flows
+	u.RUnlock()
+
 	return &util.JSONResponse{
 		Code: 401,
 		JSON: Challenge{
-			Completed: u.Sessions[sessionID],
+			Completed: completed,
-			Flows:     u.Flows,
+			Flows:     flows,
 			Session:   sessionID,
 			Params:    make(map[string]interface{}),
 		},
@@ -170,8 +181,10 @@ func (u *UserInteractive) NewSession() *util.JSONResponse {
 		res := jsonerror.InternalServerError()
 		return &res
 	}
+	u.Lock()
 	u.Sessions[sessionID] = []string{}
-	return u.Challenge(sessionID)
+	u.Unlock()
+	return u.challenge(sessionID)
 }
 
 // ResponseWithChallenge mixes together a JSON body (e.g an error with errcode/message) with the
@@ -184,7 +197,7 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter
 		return &ise
 	}
 	_ = json.Unmarshal(b, &mixedObjects)
-	challenge := u.Challenge(sessionID)
+	challenge := u.challenge(sessionID)
 	b, err = json.Marshal(challenge.JSON)
 	if err != nil {
 		ise := jsonerror.InternalServerError()
@@ -213,7 +226,11 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 
 	// extract the type so we know which login type to use
 	authType := gjson.GetBytes(bodyBytes, "auth.type").Str
+
+	u.RLock()
 	loginType, ok := u.Types[authType]
+	u.RUnlock()
+
 	if !ok {
 		return nil, &util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -223,7 +240,12 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 
 	// retrieve the session
 	sessionID := gjson.GetBytes(bodyBytes, "auth.session").Str
-	if _, ok = u.Sessions[sessionID]; !ok {
+
+	u.RLock()
+	_, ok = u.Sessions[sessionID]
+	u.RUnlock()
+
+	if !ok {
 		// if the login type is part of a single stage flow then allow them to omit the session ID
 		if !u.IsSingleStageFlow(authType) {
 			return nil, &util.JSONResponse{

clientapi/clientapi.go

@@ -48,7 +48,6 @@ func AddPublicRoutes(
 
 	syncProducer := &producers.SyncAPIProducer{
 		JetStream:              js,
-		TopicClientData:        cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
 		TopicReceiptEvent:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
 		TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
 		TopicTypingEvent:       cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
@@ -59,6 +58,7 @@ func AddPublicRoutes(
 
 	routing.Setup(
 		base.PublicClientAPIMux,
+		base.PublicWellKnownAPIMux,
 		base.SynapseAdminMux,
 		base.DendriteAdminMux,
 		cfg, rsAPI, asAPI,

clientapi/httputil/httputil.go

@@ -16,7 +16,7 @@ package httputil
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"unicode/utf8"
 
@@ -29,9 +29,9 @@ import (
 func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse {
 	// encoding/json allows invalid utf-8, matrix does not
 	// https://matrix.org/docs/spec/client_server/r0.6.1#api-standards
-	body, err := ioutil.ReadAll(req.Body)
+	body, err := io.ReadAll(req.Body)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
+		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
 		resp := jsonerror.InternalServerError()
 		return &resp
 	}

clientapi/jsonerror/jsonerror.go

@@ -15,11 +15,13 @@
 package jsonerror
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // MatrixError represents the "standard error response" in Matrix.
@@ -213,3 +215,15 @@ func NotTrusted(serverName string) *MatrixError {
 		Err:     fmt.Sprintf("Untrusted server '%s'", serverName),
 	}
 }
+
+// InternalAPIError is returned when Dendrite failed to reach an internal API.
+func InternalAPIError(ctx context.Context, err error) util.JSONResponse {
+	logrus.WithContext(ctx).WithError(err).Error("Error reaching an internal API")
+	return util.JSONResponse{
+		Code: http.StatusInternalServerError,
+		JSON: &MatrixError{
+			ErrCode: "M_INTERNAL_SERVER_ERROR",
+			Err:     "Dendrite encountered an error reaching an internal API.",
+		},
+	}
+}

clientapi/producers/syncapi.go

@@ -21,7 +21,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -32,7 +31,6 @@ import (
 
 // SyncAPIProducer produces events for the sync API server to consume
 type SyncAPIProducer struct {
-	TopicClientData        string
 	TopicReceiptEvent      string
 	TopicSendToDeviceEvent string
 	TopicTypingEvent       string
@@ -42,36 +40,6 @@ type SyncAPIProducer struct {
 	UserAPI                userapi.ClientUserAPI
 }
 
-// SendData sends account data to the sync API server
-func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON, ignoredUsers *types.IgnoredUsers) error {
-	m := &nats.Msg{
-		Subject: p.TopicClientData,
-		Header:  nats.Header{},
-	}
-	m.Header.Set(jetstream.UserID, userID)
-
-	data := eventutil.AccountData{
-		RoomID:       roomID,
-		Type:         dataType,
-		ReadMarker:   readMarker,
-		IgnoredUsers: ignoredUsers,
-	}
-	var err error
-	m.Data, err = json.Marshal(data)
-	if err != nil {
-		return err
-	}
-
-	log.WithFields(log.Fields{
-		"user_id":   userID,
-		"room_id":   roomID,
-		"data_type": dataType,
-	}).Tracef("Producing to topic '%s'", p.TopicClientData)
-
-	_, err = p.JetStream.PublishMsg(m)
-	return err
-}
-
 func (p *SyncAPIProducer) SendReceipt(
 	ctx context.Context,
 	userID, roomID, eventID, receiptType string, timestamp gomatrixserverlib.Timestamp,

clientapi/routing/account_data.go

@@ -17,7 +17,7 @@ package routing
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -25,7 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/userapi/api"
 
 	"github.com/matrix-org/util"
@@ -102,9 +101,9 @@ func SaveAccountData(
 		}
 	}
 
-	body, err := ioutil.ReadAll(req.Body)
+	body, err := io.ReadAll(req.Body)
 	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
+		util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
 		return jsonerror.InternalServerError()
 	}
 
@@ -127,18 +126,6 @@ func SaveAccountData(
 		return util.ErrorResponse(err)
 	}
 
-	var ignoredUsers *types.IgnoredUsers
-	if dataType == "m.ignored_user_list" {
-		ignoredUsers = &types.IgnoredUsers{}
-		_ = json.Unmarshal(body, ignoredUsers)
-	}
-
-	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
-		return jsonerror.InternalServerError()
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
@@ -191,11 +178,6 @@ func SaveReadMarker(
 		return util.ErrorResponse(err)
 	}
 
-	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
-		return jsonerror.InternalServerError()
-	}
-
 	// Handle the read receipt that may be included in the read marker
 	if r.Read != "" {
 		return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)

clientapi/routing/admin.go

@@ -1,23 +1,20 @@
 package routing
 
 import (
+	"encoding/json"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	if device.AccountType != userapi.AccountTypeAdmin {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-		}
-	}
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -30,13 +27,15 @@ func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserv
 		}
 	}
 	res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
-	rsAPI.PerformAdminEvacuateRoom(
+	if err := rsAPI.PerformAdminEvacuateRoom(
 		req.Context(),
 		&roomserverAPI.PerformAdminEvacuateRoomRequest{
 			RoomID: roomID,
 		},
 		res,
-	)
+	); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if err := res.Error; err != nil {
 		return err.JSONResponse()
 	}
@@ -48,13 +47,7 @@ func AdminEvacuateRoom(req *http.Request, device *userapi.Device, rsAPI roomserv
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
-	if device.AccountType != userapi.AccountTypeAdmin {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("This API can only be used by admin users."),
-		}
-	}
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -66,14 +59,26 @@ func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserv
 			JSON: jsonerror.MissingArgument("Expecting user ID."),
 		}
 	}
+	_, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return util.MessageResponse(http.StatusBadRequest, err.Error())
+	}
+	if domain != cfg.Matrix.ServerName {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("User ID must belong to this server."),
+		}
+	}
 	res := &roomserverAPI.PerformAdminEvacuateUserResponse{}
-	rsAPI.PerformAdminEvacuateUser(
+	if err := rsAPI.PerformAdminEvacuateUser(
 		req.Context(),
 		&roomserverAPI.PerformAdminEvacuateUserRequest{
 			UserID: userID,
 		},
 		res,
-	)
+	); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if err := res.Error; err != nil {
 		return err.JSONResponse()
 	}
@@ -84,3 +89,52 @@ func AdminEvacuateUser(req *http.Request, device *userapi.Device, rsAPI roomserv
 		},
 	}
 }
+
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	localpart, ok := vars["localpart"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting user localpart."),
+		}
+	}
+	request := struct {
+		Password string `json:"password"`
+	}{}
+	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
+		}
+	}
+	if request.Password == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting non-empty password."),
+		}
+	}
+	updateReq := &userapi.PerformPasswordUpdateRequest{
+		Localpart:     localpart,
+		Password:      request.Password,
+		LogoutDevices: true,
+	}
+	updateRes := &userapi.PerformPasswordUpdateResponse{}
+	if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Failed to perform password update: " + err.Error()),
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct {
+			Updated bool `json:"password_updated"`
+		}{
+			Updated: updateRes.PasswordUpdated,
+		},
+	}
+}

clientapi/routing/createroom.go

@@ -49,6 +49,7 @@ type createRoomRequest struct {
 	GuestCanJoin              bool                          `json:"guest_can_join"`
 	RoomVersion               gomatrixserverlib.RoomVersion `json:"room_version"`
 	PowerLevelContentOverride json.RawMessage               `json:"power_level_content_override"`
+	IsDirect                  bool                          `json:"is_direct"`
 }
 
 const (
@@ -499,9 +500,17 @@ func createRoom(
 		// Build some stripped state for the invite.
 		var globalStrippedState []gomatrixserverlib.InviteV2StrippedState
 		for _, event := range builtEvents {
+			// Chosen events from the spec:
+			// https://spec.matrix.org/v1.3/client-server-api/#stripped-state
 			switch event.Type() {
+			case gomatrixserverlib.MRoomCreate:
+				fallthrough
 			case gomatrixserverlib.MRoomName:
 				fallthrough
+			case gomatrixserverlib.MRoomAvatar:
+				fallthrough
+			case gomatrixserverlib.MRoomTopic:
+				fallthrough
 			case gomatrixserverlib.MRoomCanonicalAlias:
 				fallthrough
 			case gomatrixserverlib.MRoomEncryption:
@@ -522,7 +531,7 @@ func createRoom(
 			// Build the invite event.
 			inviteEvent, err := buildMembershipEvent(
 				ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite,
-				roomID, true, cfg, evTime, rsAPI, asAPI,
+				roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI,
 			)
 			if err != nil {
 				util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
@@ -556,10 +565,12 @@ func createRoom(
 	if r.Visibility == "public" {
 		// expose this room in the published room list
 		var pubRes roomserverAPI.PerformPublishResponse
-		rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
+		if err := rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
 			RoomID:     roomID,
 			Visibility: "public",
-		}, &pubRes)
+		}, &pubRes); err != nil {
+			return jsonerror.InternalAPIError(ctx, err)
+		}
 		if pubRes.Error != nil {
 			// treat as non-fatal since the room is already made by this point
 			util.GetLogger(ctx).WithError(pubRes.Error).Error("failed to visibility:public")

clientapi/routing/deactivate.go

@@ -1,7 +1,7 @@
 package routing
 
 import (
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
@@ -20,7 +20,7 @@ func Deactivate(
 ) util.JSONResponse {
 	ctx := req.Context()
 	defer req.Body.Close() // nolint:errcheck
-	bodyBytes, err := ioutil.ReadAll(req.Body)
+	bodyBytes, err := io.ReadAll(req.Body)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,

clientapi/routing/device.go

@@ -15,7 +15,7 @@
 package routing
 
 import (
-	"io/ioutil"
+	"io"
 	"net"
 	"net/http"
 
@@ -175,7 +175,7 @@ func DeleteDeviceById(
 	}()
 	ctx := req.Context()
 	defer req.Body.Close() // nolint:errcheck
-	bodyBytes, err := ioutil.ReadAll(req.Body)
+	bodyBytes, err := io.ReadAll(req.Body)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,

clientapi/routing/directory.go

@@ -302,10 +302,12 @@ func SetVisibility(
 	}
 
 	var publishRes roomserverAPI.PerformPublishResponse
-	rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
+	if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
 		RoomID:     roomID,
 		Visibility: v.Visibility,
-	}, &publishRes)
+	}, &publishRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if publishRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
 		return publishRes.Error.JSONResponse()

clientapi/routing/directory_public.go

@@ -23,13 +23,14 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
 )
 
 var (
@@ -196,14 +197,14 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
 
 // sliceInto returns a subslice of `slice` which honours the since/limit values given.
 //
-//    0  1  2  3  4  5  6   index
+//	  0  1  2  3  4  5  6   index
-//   [A, B, C, D, E, F, G]  slice
+//	 [A, B, C, D, E, F, G]  slice
 //
-//   limit=3          => A,B,C (prev='', next='3')
+//	 limit=3          => A,B,C (prev='', next='3')
-//   limit=3&since=3  => D,E,F (prev='0', next='6')
+//	 limit=3&since=3  => D,E,F (prev='0', next='6')
-//   limit=3&since=6  => G     (prev='3', next='')
+//	 limit=3&since=6  => G     (prev='3', next='')
 //
-//  A value of '-1' for prev/next indicates no position.
+//	A value of '-1' for prev/next indicates no position.
 func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
 	prev = -1
 	next = -1

clientapi/routing/joinroom.go

@@ -81,8 +81,9 @@ func JoinRoomByIDOrAlias(
 	done := make(chan util.JSONResponse, 1)
 	go func() {
 		defer close(done)
-		rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes)
+		if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
-		if joinRes.Error != nil {
+			done <- jsonerror.InternalAPIError(req.Context(), err)
+		} else if joinRes.Error != nil {
 			done <- joinRes.Error.JSONResponse()
 		} else {
 			done <- util.JSONResponse{

clientapi/routing/key_backup.go

@@ -91,10 +91,12 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, de
 // Implements GET  /_matrix/client/r0/room_keys/version and GET /_matrix/client/r0/room_keys/version/{version}
 func KeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
 	var queryResp userapi.QueryKeyBackupResponse
-	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
+	if err := userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
 		UserID:  device.UserID,
 		Version: version,
-	}, &queryResp)
+	}, &queryResp); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if queryResp.Error != "" {
 		return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
 	}
@@ -233,13 +235,15 @@ func GetBackupKeys(
 	req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version, roomID, sessionID string,
 ) util.JSONResponse {
 	var queryResp userapi.QueryKeyBackupResponse
-	userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
+	if err := userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
 		UserID:           device.UserID,
 		Version:          version,
 		ReturnKeys:       true,
 		KeysForRoomID:    roomID,
 		KeysForSessionID: sessionID,
-	}, &queryResp)
+	}, &queryResp); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if queryResp.Error != "" {
 		return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
 	}

clientapi/routing/key_crosssigning.go

@@ -72,7 +72,9 @@ func UploadCrossSigningDeviceKeys(
 	sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
 
 	uploadReq.UserID = device.UserID
-	keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes)
+	if err := keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	if err := uploadRes.Error; err != nil {
 		switch {
@@ -114,7 +116,9 @@ func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.Clie
 	}
 
 	uploadReq.UserID = device.UserID
-	keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes)
+	if err := keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	if err := uploadRes.Error; err != nil {
 		switch {

clientapi/routing/keys.go

@@ -62,7 +62,9 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi
 	}
 
 	var uploadRes api.PerformUploadKeysResponse
-	keyAPI.PerformUploadKeys(req.Context(), uploadReq, &uploadRes)
+	if err := keyAPI.PerformUploadKeys(req.Context(), uploadReq, &uploadRes); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if uploadRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(uploadRes.Error).Error("Failed to PerformUploadKeys")
 		return jsonerror.InternalServerError()
@@ -107,12 +109,14 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devic
 		return *resErr
 	}
 	queryRes := api.QueryKeysResponse{}
-	keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
+	if err := keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
 		UserID:        device.UserID,
 		UserToDevices: r.DeviceKeys,
 		Timeout:       r.GetTimeout(),
 		// TODO: Token?
-	}, &queryRes)
+	}, &queryRes); err != nil {
+		return util.ErrorResponse(err)
+	}
 	return util.JSONResponse{
 		Code: 200,
 		JSON: map[string]interface{}{
@@ -145,10 +149,12 @@ func ClaimKeys(req *http.Request, keyAPI api.ClientKeyAPI) util.JSONResponse {
 		return *resErr
 	}
 	claimRes := api.PerformClaimKeysResponse{}
-	keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
+	if err := keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
 		OneTimeKeys: r.OneTimeKeys,
 		Timeout:     r.GetTimeout(),
-	}, &claimRes)
+	}, &claimRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if claimRes.Error != nil {
 		util.GetLogger(req.Context()).WithError(claimRes.Error).Error("failed to PerformClaimKeys")
 		return jsonerror.InternalServerError()

clientapi/routing/peekroom.go

@@ -17,6 +17,7 @@ package routing
 import (
 	"net/http"
 
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -54,7 +55,9 @@ func PeekRoomByIDOrAlias(
 	}
 
 	// Ask the roomserver to perform the peek.
-	rsAPI.PerformPeek(req.Context(), &peekReq, &peekRes)
+	if err := rsAPI.PerformPeek(req.Context(), &peekReq, &peekRes); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if peekRes.Error != nil {
 		return peekRes.Error.JSONResponse()
 	}
@@ -89,7 +92,9 @@ func UnpeekRoomByID(
 	}
 	unpeekRes := roomserverAPI.PerformUnpeekResponse{}
 
-	rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes)
+	if err := rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 	if unpeekRes.Error != nil {
 		return unpeekRes.Error.JSONResponse()
 	}

clientapi/routing/register.go

@@ -19,7 +19,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -371,7 +371,7 @@ func validateRecaptcha(
 
 	// Grab the body of the response from the captcha server
 	var r recaptchaResponse
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return &util.JSONResponse{
 			Code: http.StatusGatewayTimeout,
@@ -539,7 +539,7 @@ func Register(
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	defer req.Body.Close() // nolint: errcheck
-	reqBody, err := ioutil.ReadAll(req.Body)
+	reqBody, err := io.ReadAll(req.Body)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,

clientapi/routing/register_secret_test.go

@@ -2,7 +2,7 @@ package routing
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"testing"
 
 	"github.com/patrickmn/go-cache"
@@ -13,7 +13,7 @@ func TestSharedSecretRegister(t *testing.T) {
 	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
 	sharedSecret := "dendritetest"
 
-	req, err := NewSharedSecretRegistrationRequest(ioutil.NopCloser(bytes.NewBuffer(jsonStr)))
+	req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))
 	if err != nil {
 		t.Fatalf("failed to read request: %s", err)
 	}

clientapi/routing/room_tagging.go

@@ -18,8 +18,6 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/sirupsen/logrus"
-
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
@@ -98,10 +96,6 @@ func PutTag(
 		return jsonerror.InternalServerError()
 	}
 
-	if err = syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
-		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
@@ -150,11 +144,6 @@ func DeleteTag(
 		return jsonerror.InternalServerError()
 	}
 
-	// TODO: user API should do this since it's account data
-	if err := syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
-		logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
-	}
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},

clientapi/routing/routing.go

@@ -48,7 +48,7 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
+	publicAPIMux, wkMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
@@ -74,6 +74,26 @@ func Setup(
 		unstableFeatures["org.matrix."+msc] = true
 	}
 
+	if cfg.Matrix.WellKnownClientName != "" {
+		logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName)
+		wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: struct {
+					HomeserverName struct {
+						BaseUrl string `json:"base_url"`
+					} `json:"m.homeserver"`
+				}{
+					HomeserverName: struct {
+						BaseUrl string `json:"base_url"`
+					}{
+						BaseUrl: cfg.Matrix.WellKnownClientName,
+					},
+				},
+			}
+		})).Methods(http.MethodGet, http.MethodOptions)
+	}
+
 	publicAPIMux.Handle("/versions",
 		httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
 			return util.JSONResponse{
@@ -124,17 +144,23 @@ func Setup(
 	}
 
 	dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
-		httputil.MakeAuthAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateRoom(req, device, rsAPI)
+			return AdminEvacuateRoom(req, cfg, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
-		httputil.MakeAuthAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminEvacuateUser(req, device, rsAPI)
+			return AdminEvacuateUser(req, cfg, device, rsAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/resetPassword/{localpart}",
+		httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminResetPassword(req, cfg, device, userAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
@@ -909,12 +935,12 @@ func Setup(
 			return SearchUserDirectory(
 				req.Context(),
 				device,
-				userAPI,
 				rsAPI,
 				userDirectoryProvider,
-				cfg.Matrix.ServerName,
 				postContent.SearchString,
 				postContent.Limit,
+				federation,
+				cfg.Matrix.ServerName,
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)

clientapi/routing/sendevent.go

@@ -63,9 +63,10 @@ var sendEventDuration = prometheus.NewHistogramVec(
 )
 
 // SendEvent implements:
-//   /rooms/{roomID}/send/{eventType}
+//
-//   /rooms/{roomID}/send/{eventType}/{txnID}
+//	/rooms/{roomID}/send/{eventType}
-//   /rooms/{roomID}/state/{eventType}/{stateKey}
+//	/rooms/{roomID}/send/{eventType}/{txnID}
+//	/rooms/{roomID}/state/{eventType}/{stateKey}
 func SendEvent(
 	req *http.Request,
 	device *userapi.Device,

clientapi/routing/threepid.go

@@ -38,8 +38,9 @@ type threePIDsResponse struct {
 }
 
 // RequestEmailToken implements:
-//     POST /account/3pid/email/requestToken
+//
-//     POST /register/email/requestToken
+//	POST /account/3pid/email/requestToken
+//	POST /register/email/requestToken
 func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {

clientapi/routing/upgrade_room.go

@@ -64,7 +64,9 @@ func UpgradeRoom(
 	}
 	upgradeResp := roomserverAPI.PerformRoomUpgradeResponse{}
 
-	rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp)
+	if err := rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	if upgradeResp.Error != nil {
 		if upgradeResp.Error.Code == roomserverAPI.PerformErrorNoRoom {

clientapi/routing/userdirectory.go

@@ -18,10 +18,13 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"net/http"
+	"strings"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -34,12 +37,12 @@ type UserDirectoryResponse struct {
 func SearchUserDirectory(
 	ctx context.Context,
 	device *userapi.Device,
-	userAPI userapi.ClientUserAPI,
 	rsAPI api.ClientRoomserverAPI,
 	provider userapi.QuerySearchProfilesAPI,
-	serverName gomatrixserverlib.ServerName,
 	searchString string,
 	limit int,
+	federation *gomatrixserverlib.FederationClient,
+	localServerName gomatrixserverlib.ServerName,
 ) util.JSONResponse {
 	if limit < 10 {
 		limit = 10
@@ -51,59 +54,74 @@ func SearchUserDirectory(
 		Limited: false,
 	}
 
-	// First start searching local users.
+	// Get users we share a room with
-	userReq := &userapi.QuerySearchProfilesRequest{
+	knownUsersReq := &api.QueryKnownUsersRequest{
-		SearchString: searchString,
+		UserID: device.UserID,
-		Limit:        limit,
+		Limit:  limit,
 	}
-	userRes := &userapi.QuerySearchProfilesResponse{}
+	knownUsersRes := &api.QueryKnownUsersResponse{}
-	if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
+	if err := rsAPI.QueryKnownUsers(ctx, knownUsersReq, knownUsersRes); err != nil && err != sql.ErrNoRows {
-		return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
+		return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
 	}
 
-	for _, user := range userRes.Profiles {
+knownUsersLoop:
+	for _, profile := range knownUsersRes.Users {
 		if len(results) == limit {
 			response.Limited = true
 			break
 		}
-
+		userID := profile.UserID
-		var userID string
+		// get the full profile of the local user
-		if user.ServerName != "" {
+		localpart, serverName, _ := gomatrixserverlib.SplitID('@', userID)
-			userID = fmt.Sprintf("@%s:%s", user.Localpart, user.ServerName)
+		if serverName == localServerName {
+			userReq := &userapi.QuerySearchProfilesRequest{
+				SearchString: localpart,
+				Limit:        limit,
+			}
+			userRes := &userapi.QuerySearchProfilesResponse{}
+			if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
+				return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
+			}
+			for _, p := range userRes.Profiles {
+				if strings.Contains(p.DisplayName, searchString) ||
+					strings.Contains(p.Localpart, searchString) {
+					profile.DisplayName = p.DisplayName
+					profile.AvatarURL = p.AvatarURL
+					results[userID] = profile
+					if len(results) == limit {
+						response.Limited = true
+						break knownUsersLoop
+					}
+				}
+			}
 		} else {
-			userID = fmt.Sprintf("@%s:%s", user.Localpart, serverName)
+			// If the username already contains the search string, don't bother hitting federation.
-		}
+			// This will result in missing avatars and displaynames, but saves the federation roundtrip.
-		if _, ok := results[userID]; !ok {
+			if strings.Contains(localpart, searchString) {
-			results[userID] = authtypes.FullyQualifiedProfile{
+				results[userID] = profile
-				UserID:      userID,
+				if len(results) == limit {
-				DisplayName: user.DisplayName,
+					response.Limited = true
-				AvatarURL:   user.AvatarURL,
+					break knownUsersLoop
+				}
+				continue
 			}
-		}
+			// TODO: We should probably cache/store this
-	}
+			fedProfile, fedErr := federation.LookupProfile(ctx, serverName, userID, "")
-
+			if fedErr != nil {
-	// Then, if we have enough room left in the response,
+				if x, ok := fedErr.(gomatrix.HTTPError); ok {
-	// start searching for known users from joined rooms.
+					if x.Code == http.StatusNotFound {
-
+						continue
-	if len(results) <= limit {
+					}
-		stateReq := &api.QueryKnownUsersRequest{
+				}
-			UserID:       device.UserID,
-			SearchString: searchString,
-			Limit:        limit - len(results),
-		}
-		stateRes := &api.QueryKnownUsersResponse{}
-		if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows {
-			return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
-		}
-
-		for _, user := range stateRes.Users {
-			if len(results) == limit {
-				response.Limited = true
-				break
 			}
-
+			if strings.Contains(fedProfile.DisplayName, searchString) {
-			if _, ok := results[user.UserID]; !ok {
+				profile.DisplayName = fedProfile.DisplayName
-				results[user.UserID] = user
+				profile.AvatarURL = fedProfile.AvatarURL
+				results[userID] = profile
+				if len(results) == limit {
+					response.Limited = true
+					break knownUsersLoop
+				}
 			}
 		}
 	}

clientapi/routing/voip.go

@@ -22,15 +22,17 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/util"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/util"
 )
 
 // RequestTurnServer implements:
-//     GET /voip/turnServer
+//
+//	GET /voip/turnServer
 func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse {
 	turnConfig := cfg.TURN
 

cmd/create-account/main.go

@@ -15,21 +15,26 @@
 package main
 
 import (
-	"context"
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"net/http"
 	"os"
 	"regexp"
 	"strings"
+	"time"
+
+	"github.com/tidwall/gjson"
 
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/term"
+
+	"github.com/matrix-org/dendrite/setup"
 )
 
 const usage = `Usage: %s
@@ -47,8 +52,6 @@ Example:
 	# read password from stdin
 	%s --config dendrite.yaml -username alice -passwordstdin < my.pass
 	cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin
-	# reset password for a user, can be used with a combination above to read the password
-	%s --config dendrite.yaml -reset-password -username alice -password foobarbaz
 
 Arguments:
 
@@ -59,29 +62,39 @@ var (
 	password           = flag.String("password", "", "The password to associate with the account")
 	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
 	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
-	pwdLess            = flag.Bool("passwordless", false, "Create a passwordless account, e.g. if only an accesstoken is required")
 	isAdmin            = flag.Bool("admin", false, "Create an admin account")
-	resetPassword      = flag.Bool("reset-password", false, "Resets the password for the given username")
+	resetPassword      = flag.Bool("reset-password", false, "Deprecated")
+	serverURL          = flag.String("url", "https://localhost:8448", "The URL to connect to.")
 	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
+	timeout            = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
 )
 
+var cl = http.Client{
+	Timeout:   time.Second * 30,
+	Transport: http.DefaultTransport,
+}
+
 func main() {
 	name := os.Args[0]
 	flag.Usage = func() {
-		_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name, name)
+		_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name)
 		flag.PrintDefaults()
 	}
 	cfg := setup.ParseFlags(true)
 
+	if *resetPassword {
+		logrus.Fatalf("The reset-password flag has been replaced by the POST /_dendrite/admin/resetPassword/{localpart} admin API.")
+	}
+
+	if cfg.ClientAPI.RegistrationSharedSecret == "" {
+		logrus.Fatalln("Shared secret registration is not enabled, enable it by setting a shared secret in the config: 'client_api.registration_shared_secret'")
+	}
+
 	if *username == "" {
 		flag.Usage()
 		os.Exit(1)
 	}
 
-	if *pwdLess && *resetPassword {
-		logrus.Fatalf("Can not reset to an empty password, unable to login afterwards.")
-	}
-
 	if !validUsernameRegex.MatchString(*username) {
 		logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
 		os.Exit(1)
@@ -91,73 +104,102 @@ func main() {
 		logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
 	}
 
-	var pass string
+	pass, err := getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
-	var err error
-	if !*pwdLess {
-		pass, err = getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
-		if err != nil {
-			logrus.Fatalln(err)
-		}
-	}
-
-	// avoid warning about open registration
-	cfg.ClientAPI.RegistrationDisabled = true
-
-	b := base.NewBaseDendrite(cfg, "")
-	defer b.Close() // nolint: errcheck
-
-	accountDB, err := storage.NewUserAPIDatabase(
-		b,
-		&cfg.UserAPI.AccountDatabase,
-		cfg.Global.ServerName,
-		cfg.UserAPI.BCryptCost,
-		cfg.UserAPI.OpenIDTokenLifetimeMS,
-		0, // TODO
-		cfg.Global.ServerNotices.LocalPart,
-	)
 	if err != nil {
-		logrus.WithError(err).Fatalln("Failed to connect to the database")
+		logrus.Fatalln(err)
 	}
 
-	accType := api.AccountTypeUser
+	cl.Timeout = *timeout
-	if *isAdmin {
-		accType = api.AccountTypeAdmin
-	}
 
-	available, err := accountDB.CheckAccountAvailability(context.Background(), *username)
+	accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
-	if err != nil {
-		logrus.Fatalln("Unable check username existence.")
-	}
-	if *resetPassword {
-		if available {
-			logrus.Fatalln("Username could not be found.")
-		}
-		err = accountDB.SetPassword(context.Background(), *username, pass)
-		if err != nil {
-			logrus.Fatalf("Failed to update password for user %s: %s", *username, err.Error())
-		}
-		if _, err = accountDB.RemoveAllDevices(context.Background(), *username, ""); err != nil {
-			logrus.Fatalf("Failed to remove all devices: %s", err.Error())
-		}
-		logrus.Infof("Updated password for user %s and invalidated all logins\n", *username)
-		return
-	}
-	if !available {
-		logrus.Fatalln("Username is already in use.")
-	}
-
-	_, err = accountDB.CreateAccount(context.Background(), *username, pass, "", accType)
 	if err != nil {
 		logrus.Fatalln("Failed to create the account:", err.Error())
 	}
 
-	logrus.Infoln("Created account", *username)
+	logrus.Infof("Created account: %s (AccessToken: %s)", *username, accessToken)
+}
+
+type sharedSecretRegistrationRequest struct {
+	User     string `json:"username"`
+	Password string `json:"password"`
+	Nonce    string `json:"nonce"`
+	MacStr   string `json:"mac"`
+	Admin    bool   `json:"admin"`
+}
+
+func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accessToken string, err error) {
+	registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", strings.Trim(serverURL, "/"))
+	nonceReq, err := http.NewRequest(http.MethodGet, registerURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("unable to create http request: %w", err)
+	}
+	nonceResp, err := cl.Do(nonceReq)
+	if err != nil {
+		return "", fmt.Errorf("unable to get nonce: %w", err)
+	}
+	body, err := io.ReadAll(nonceResp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %w", err)
+	}
+	defer nonceResp.Body.Close() // nolint: errcheck
+
+	nonce := gjson.GetBytes(body, "nonce").Str
+
+	adminStr := "notadmin"
+	if admin {
+		adminStr = "admin"
+	}
+	reg := sharedSecretRegistrationRequest{
+		User:     localpart,
+		Password: password,
+		Nonce:    nonce,
+		Admin:    admin,
+	}
+	macStr, err := getRegisterMac(sharedSecret, nonce, localpart, password, adminStr)
+	if err != nil {
+		return "", err
+	}
+	reg.MacStr = macStr
+
+	js, err := json.Marshal(reg)
+	if err != nil {
+		return "", fmt.Errorf("unable to marshal json: %w", err)
+	}
+	registerReq, err := http.NewRequest(http.MethodPost, registerURL, bytes.NewBuffer(js))
+	if err != nil {
+		return "", fmt.Errorf("unable to create http request: %w", err)
+
+	}
+	regResp, err := cl.Do(registerReq)
+	if err != nil {
+		return "", fmt.Errorf("unable to create account: %w", err)
+	}
+	defer regResp.Body.Close() // nolint: errcheck
+	if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
+		body, _ = io.ReadAll(regResp.Body)
+		return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
+	}
+	r, _ := io.ReadAll(regResp.Body)
+
+	return gjson.GetBytes(r, "access_token").Str, nil
+}
+
+func getRegisterMac(sharedSecret, nonce, localpart, password, adminStr string) (string, error) {
+	joined := strings.Join([]string{nonce, localpart, password, adminStr}, "\x00")
+	mac := hmac.New(sha1.New, []byte(sharedSecret))
+	_, err := mac.Write([]byte(joined))
+	if err != nil {
+		return "", fmt.Errorf("unable to construct mac: %w", err)
+	}
+	regMac := mac.Sum(nil)
+
+	return hex.EncodeToString(regMac), nil
 }
 
 func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string, error) {
 	// read password from file
 	if pwdFile != "" {
-		pw, err := ioutil.ReadFile(pwdFile)
+		pw, err := os.ReadFile(pwdFile)
 		if err != nil {
 			return "", fmt.Errorf("Unable to read password from file: %v", err)
 		}
@@ -166,7 +208,7 @@ func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string,
 
 	// read password from stdin
 	if pwdStdin {
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		if err != nil {
 			return "", fmt.Errorf("Unable to read password from stdin: %v", err)
 		}

cmd/dendrite-demo-pinecone/main.go

@@ -21,7 +21,6 @@ import (
 	"encoding/hex"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -76,11 +75,11 @@ func main() {
 		if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
 			panic(err)
 		}
-		if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil {
+		if err = os.WriteFile(keyfile, sk, 0644); err != nil {
 			panic(err)
 		}
 	} else if err == nil {
-		if sk, err = ioutil.ReadFile(keyfile); err != nil {
+		if sk, err = os.ReadFile(keyfile); err != nil {
 			panic(err)
 		}
 		if len(sk) != ed25519.PrivateKeySize {
@@ -92,7 +91,7 @@ func main() {
 	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
 	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
 	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
-	pManager := pineconeConnections.NewConnectionManager(pRouter)
+	pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
 	pMulticast.Start()
 	if instancePeer != nil && *instancePeer != "" {
 		pManager.AddPeer(*instancePeer)

cmd/dendrite-demo-yggdrasil/README.md

@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.16 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run:
 
@@ -13,10 +13,10 @@ The following command line arguments are accepted:
 * `-peer tcp://a.b.c.d:e` to specify a static Yggdrasil peer to connect to - you will need to supply this if you do not have another Yggdrasil node on your network
 * `-port 12345` to specify a port to listen on for client connections
 
-If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/). 
+If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/).
 
 Then point your favourite Matrix client to  the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in.
 
 If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
 
-Once logged in, you should be able to open the room directory or join a room by its ID.
+Once logged in, you should be able to open the room directory or join a room by its ID.

cmd/dendrite-demo-yggdrasil/yggconn/node.go

@@ -20,7 +20,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -69,7 +68,7 @@ func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
 
 	yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
 	if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
-		yggconf, e := ioutil.ReadFile(yggfile)
+		yggconf, e := os.ReadFile(yggfile)
 		if e != nil {
 			panic(err)
 		}
@@ -88,7 +87,7 @@ func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) {
 	if err != nil {
 		panic(err)
 	}
-	if e := ioutil.WriteFile(yggfile, j, 0600); e != nil {
+	if e := os.WriteFile(yggfile, j, 0600); e != nil {
 		n.log.Printf("Couldn't write private key to file '%s': %s\n", yggfile, e)
 	}
 

cmd/dendrite-upgrade-tests/main.go

@@ -6,7 +6,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -37,6 +37,7 @@ var (
 	flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images")
 	flagHead             = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
 	flagDockerHost       = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
+	flagDirect           = flag.Bool("direct", false, "If a direct upgrade from the defined FROM version to TO should be done")
 	alphaNumerics        = regexp.MustCompile("[^a-zA-Z0-9]+")
 )
 
@@ -46,9 +47,9 @@ const HEAD = "HEAD"
 // We cannot use the dockerfile associated with the repo with each version sadly due to changes in
 // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
 // due to the error:
-//   When using COPY with more than one source file, the destination must be a directory and end with a /
+// When using COPY with more than one source file, the destination must be a directory and end with a /
 // We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
-const Dockerfile = `FROM golang:1.16-stretch as build
+const Dockerfile = `FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
 
@@ -94,7 +95,9 @@ CMD /build/run_dendrite.sh `
 const dendriteUpgradeTestLabel = "dendrite_upgrade_test"
 
 // downloadArchive downloads an arbitrary github archive of the form:
-//   https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
+//
+//	https://github.com/matrix-org/dendrite/archive/v0.3.11.tar.gz
+//
 // and re-tarballs it without the top-level directory which contains branch information. It inserts
 // the contents of `dockerfile` as a root file `Dockerfile` in the re-tarballed directory such that
 // you can directly feed the retarballed archive to `ImageBuild` to have it run said dockerfile.
@@ -125,7 +128,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 		return nil, err
 	}
 	// add top level Dockerfile
-	err = ioutil.WriteFile(path.Join(tmpDir, "Dockerfile"), dockerfile, os.ModePerm)
+	err = os.WriteFile(path.Join(tmpDir, "Dockerfile"), dockerfile, os.ModePerm)
 	if err != nil {
 		return nil, fmt.Errorf("failed to inject /Dockerfile: %w", err)
 	}
@@ -147,7 +150,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
 	if branchOrTagName == HEAD && *flagHead != "" {
 		log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
 		// add top level Dockerfile
-		err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
+		err = os.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
 		if err != nil {
 			return "", fmt.Errorf("custom HEAD: failed to inject /Dockerfile: %w", err)
 		}
@@ -229,7 +232,7 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve
 	return semVers, nil
 }
 
-func calculateVersions(cli *http.Client, from, to string) []string {
+func calculateVersions(cli *http.Client, from, to string, direct bool) []string {
 	semvers, err := getAndSortVersionsFromGithub(cli)
 	if err != nil {
 		log.Fatalf("failed to collect semvers from github: %s", err)
@@ -284,6 +287,9 @@ func calculateVersions(cli *http.Client, from, to string) []string {
 	if to == HEAD {
 		versions = append(versions, HEAD)
 	}
+	if direct {
+		versions = []string{versions[0], versions[len(versions)-1]}
+	}
 	return versions
 }
 
@@ -382,7 +388,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
 		})
 		// ignore errors when cannot get logs, it's just for debugging anyways
 		if err == nil {
-			logbody, err := ioutil.ReadAll(logs)
+			logbody, err := io.ReadAll(logs)
 			if err == nil {
 				log.Printf("Container logs:\n\n%s\n\n", string(logbody))
 			}
@@ -461,7 +467,7 @@ func main() {
 		os.Exit(1)
 	}
 	cleanup(dockerClient)
-	versions := calculateVersions(httpClient, *flagFrom, *flagTo)
+	versions := calculateVersions(httpClient, *flagFrom, *flagTo, *flagDirect)
 	log.Printf("Testing dendrite versions: %v\n", versions)
 
 	branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions)

cmd/dendrite-upgrade-tests/tests.go

@@ -18,9 +18,9 @@ type user struct {
 }
 
 // runTests performs the following operations:
-//  - register alice and bob with branch name muxed into the localpart
+// - register alice and bob with branch name muxed into the localpart
-//  - create a DM room for the 2 users and exchange messages
+// - create a DM room for the 2 users and exchange messages
-//  - create/join a public #global room and exchange messages
+// - create/join a public #global room and exchange messages
 func runTests(baseURL, branchName string) error {
 	// register 2 users
 	users := []user{

cmd/furl/main.go

@@ -9,7 +9,6 @@ import (
 	"encoding/pem"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 
@@ -30,7 +29,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	data, err := ioutil.ReadFile(*requestKey)
+	data, err := os.ReadFile(*requestKey)
 	if err != nil {
 		panic(err)
 	}

cmd/generate-keys/main.go

@@ -38,6 +38,7 @@ var (
 	authorityCertFile = flag.String("tls-authority-cert", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
 	authorityKeyFile  = flag.String("tls-authority-key", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
 	serverName        = flag.String("server", "", "Optional: Create TLS certificate/keys with this domain name set. Useful for integration testing.")
+	keySize           = flag.Int("keysize", 4096, "Optional: Create TLS RSA private key with the given key size")
 )
 
 func main() {
@@ -58,12 +59,12 @@ func main() {
 			log.Fatal("Zero or both of --tls-key and --tls-cert must be supplied")
 		}
 		if *authorityCertFile == "" && *authorityKeyFile == "" {
-			if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile); err != nil {
+			if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile, *keySize); err != nil {
 				panic(err)
 			}
 		} else {
 			// generate the TLS cert/key based on the authority given.
-			if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile); err != nil {
+			if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile, *keySize); err != nil {
 				panic(err)
 			}
 		}

cmd/goose/README.md

@@ -1,109 +0,0 @@
-## Database migrations
-
-We use [goose](https://github.com/pressly/goose) to handle database migrations. This allows us to execute
-both SQL deltas (e.g `ALTER TABLE ...`) as well as manipulate data in the database in Go using Go functions.
-
-To run a migration, the `goose` binary in this directory needs to be built:
-```
-$ go build ./cmd/goose
-```
-
-This binary allows Dendrite databases to be upgraded and downgraded. Sample usage for upgrading the roomserver database:
-
-```
-# for sqlite
-$ ./goose -dir roomserver/storage/sqlite3/deltas sqlite3 ./roomserver.db up
-
-# for postgres
-$ ./goose -dir roomserver/storage/postgres/deltas postgres "user=dendrite dbname=dendrite sslmode=disable" up
-```
-
-For a full list of options, including rollbacks, see https://github.com/pressly/goose or use `goose` with no args.
-
-
-### Rationale
-
-Dendrite creates tables on startup using `CREATE TABLE IF NOT EXISTS`, so you might think that we should also
-apply version upgrades on startup as well. This is convenient and doesn't involve an additional binary to run
-which complicates upgrades. However, combining the upgrade mechanism and the server binary makes it difficult
-to handle rollbacks. Firstly, how do you specify you wish to rollback? We would have to add additional flags
-to the main server binary to say "rollback to version X". Secondly, if you roll back the server binary from
-version 5 to version 4, the version 4 binary doesn't know how to rollback the database from version 5 to
-version 4! For these reasons, we prefer to have a separate "upgrade" binary which is run for database upgrades.
-Rather than roll-our-own migration tool, we decided to use [goose](https://github.com/pressly/goose) as it supports
-complex migrations in Go code in addition to just executing SQL deltas. Other alternatives like
-`github.com/golang-migrate/migrate` [do not support](https://github.com/golang-migrate/migrate/issues/15) these
-kinds of complex migrations.
-
-### Adding new deltas
-
-You can add `.sql` or `.go` files manually or you can use goose to create them for you.
-
-If you only want to add a SQL delta then run:
-
-```
-$ ./goose -dir serverkeyapi/storage/sqlite3/deltas sqlite3 ./foo.db create new_col sql
-2020/09/09 14:37:43 Created new file: serverkeyapi/storage/sqlite3/deltas/20200909143743_new_col.sql
-```
-
-In this case, the version number is `20200909143743`. The important thing is that it is always increasing.
-
-Then add up/downgrade SQL commands to the created file which looks like:
-```sql
--- +goose Up
--- +goose StatementBegin
-SELECT 'up SQL query';
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-SELECT 'down SQL query';
--- +goose StatementEnd
-
-```
-You __must__ keep the `+goose` annotations. You'll need to repeat this process for Postgres.
-
-For complex Go migrations:
-
-```
-$ ./goose -dir serverkeyapi/storage/sqlite3/deltas sqlite3 ./foo.db create complex_update go
-2020/09/09 14:40:38 Created new file: serverkeyapi/storage/sqlite3/deltas/20200909144038_complex_update.go
-```
-
-Then modify the created `.go` file which looks like:
-
-```go
-package migrations
-
-import (
-	"database/sql"
-	"fmt"
-
-	"github.com/pressly/goose"
-)
-
-func init() {
-	goose.AddMigration(upComplexUpdate, downComplexUpdate)
-}
-
-func upComplexUpdate(tx *sql.Tx) error {
-	// This code is executed when the migration is applied.
-	return nil
-}
-
-func downComplexUpdate(tx *sql.Tx) error {
-	// This code is executed when the migration is rolled back.
-	return nil
-}
-
-```
-
-You __must__ import the package in `/cmd/goose/main.go` so `func init()` gets called.
-
-
-#### Database limitations
-
-- SQLite3 does NOT support `ALTER TABLE table_name DROP COLUMN` - you would have to rename the column or drop the table
-  entirely and recreate it. ([example](https://github.com/matrix-org/dendrite/blob/master/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql))
-  
-  More information: [sqlite.org](https://www.sqlite.org/lang_altertable.html)

cmd/goose/main.go

@@ -1,154 +0,0 @@
-// This is custom goose binary
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-
-	"github.com/pressly/goose"
-
-	pgusers "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
-	slusers "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
-
-	_ "github.com/lib/pq"
-	_ "github.com/mattn/go-sqlite3"
-)
-
-const (
-	AppService       = "appservice"
-	FederationSender = "federationapi"
-	KeyServer        = "keyserver"
-	MediaAPI         = "mediaapi"
-	RoomServer       = "roomserver"
-	SigningKeyServer = "signingkeyserver"
-	SyncAPI          = "syncapi"
-	UserAPI          = "userapi"
-)
-
-var (
-	dir       = flags.String("dir", "", "directory with migration files")
-	flags     = flag.NewFlagSet("goose", flag.ExitOnError)
-	component = flags.String("component", "", "dendrite component name")
-	knownDBs  = []string{
-		AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPI,
-	}
-)
-
-// nolint: gocyclo
-func main() {
-	err := flags.Parse(os.Args[1:])
-	if err != nil {
-		panic(err.Error())
-	}
-	args := flags.Args()
-
-	if len(args) < 3 {
-		fmt.Println(
-			`Usage: goose [OPTIONS] DRIVER DBSTRING COMMAND
-
-Drivers:
-    postgres
-    sqlite3
-
-Examples:
-    goose -component roomserver sqlite3 ./roomserver.db status
-    goose -component roomserver sqlite3 ./roomserver.db up
-
-    goose -component roomserver postgres "user=dendrite dbname=dendrite sslmode=disable" status
-
-Options:
-  -component string
-    	Dendrite component name e.g roomserver, signingkeyserver, clientapi, syncapi
-  -table string
-    	migrations table name (default "goose_db_version")
-  -h	print help
-  -v	enable verbose mode
-  -dir string
-        directory with migration files, only relevant when creating new migrations.
-  -version
-    	print version
-
-Commands:
-    up                   Migrate the DB to the most recent version available
-    up-by-one            Migrate the DB up by 1
-    up-to VERSION        Migrate the DB to a specific VERSION
-    down                 Roll back the version by 1
-    down-to VERSION      Roll back to a specific VERSION
-    redo                 Re-run the latest migration
-    reset                Roll back all migrations
-    status               Dump the migration status for the current DB
-    version              Print the current version of the database
-    create NAME [sql|go] Creates new migration file with the current timestamp
-    fix                  Apply sequential ordering to migrations`,
-		)
-		return
-	}
-
-	engine := args[0]
-	if engine != "sqlite3" && engine != "postgres" {
-		fmt.Println("engine must be one of 'sqlite3' or 'postgres'")
-		return
-	}
-
-	knownComponent := false
-	for _, c := range knownDBs {
-		if c == *component {
-			knownComponent = true
-			break
-		}
-	}
-	if !knownComponent {
-		fmt.Printf("component must be one of %v\n", knownDBs)
-		return
-	}
-
-	if engine == "sqlite3" {
-		loadSQLiteDeltas(*component)
-	} else {
-		loadPostgresDeltas(*component)
-	}
-
-	dbstring, command := args[1], args[2]
-
-	db, err := goose.OpenDBWithDriver(engine, dbstring)
-	if err != nil {
-		log.Fatalf("goose: failed to open DB: %v\n", err)
-	}
-
-	defer func() {
-		if err := db.Close(); err != nil {
-			log.Fatalf("goose: failed to close DB: %v\n", err)
-		}
-	}()
-
-	arguments := []string{}
-	if len(args) > 3 {
-		arguments = append(arguments, args[3:]...)
-	}
-
-	// goose demands a directory even though we don't use it for upgrades
-	d := *dir
-	if d == "" {
-		d = os.TempDir()
-	}
-	if err := goose.Run(command, db, d, arguments...); err != nil {
-		log.Fatalf("goose %v: %v", command, err)
-	}
-}
-
-func loadSQLiteDeltas(component string) {
-	switch component {
-	case UserAPI:
-		slusers.LoadFromGoose()
-	}
-}
-
-func loadPostgresDeltas(component string) {
-	switch component {
-	case UserAPI:
-		pgusers.LoadFromGoose()
-	}
-}

cmd/resolve-state/main.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/roomserver/state"
@@ -53,12 +54,10 @@ func main() {
 
 	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
 
-	cache, err := caching.NewInMemoryLRUCache(true)
+	roomserverDB, err := storage.Open(
-	if err != nil {
+		base, &cfg.RoomServer.Database,
-		panic(err)
+		caching.NewRistrettoCache(128*1024*1024, time.Hour, true),
-	}
+	)
-
-	roomserverDB, err := storage.Open(base, &cfg.RoomServer.Database, cache)
 	if err != nil {
 		panic(err)
 	}

dendrite-sample.monolith.yaml

@@ -41,10 +41,33 @@ global:
     max_idle_conns: 5
     conn_max_lifetime: -1
 
+  # Configuration for in-memory caches. Caches can often improve performance by
+  # keeping frequently accessed items (like events, identifiers etc.) in memory
+  # rather than having to read them from the database.
+  cache:
+    # The estimated maximum size for the global cache in bytes, or in terabytes,
+    # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
+    # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
+    # memory limit for the entire process. A cache that is too small may ultimately
+    # provide little or no benefit.
+    max_size_estimated: 1gb
+
+    # The maximum amount of time that a cache entry can live for in memory before
+    # it will be evicted and/or refreshed from the database. Lower values result in
+    # easier admission of new cache entries but may also increase database load in
+    # comparison to higher values, so adjust conservatively. Higher values may make
+    # it harder for new items to make it into the cache, e.g. if new rooms suddenly
+    # become popular.
+    max_age: 1h
+
   # The server name to delegate server-server communications to, with optional port
   # e.g. localhost:443
   well_known_server_name: ""
 
+  # The server name to delegate client-server communications to, with optional port
+  # e.g. localhost:443
+  well_known_client_name: ""
+
   # Lists of domains that the server will trust as identity servers to verify third
   # party identifiers such as phone numbers and email addresses.
   trusted_third_party_id_servers:
@@ -90,6 +113,11 @@ global:
     addresses:
     # - localhost:4222
 
+    # Disable the validation of TLS certificates of NATS. This is
+    # not recommended in production since it may allow NATS traffic
+    # to be sent to an insecure endpoint.
+    disable_tls_validation: false
+
     # Persistent directory to store JetStream streams in. This directory should be
     # preserved across Dendrite restarts.
     storage_path: ./
@@ -150,13 +178,16 @@ client_api:
 
   # TURN server information that this homeserver should send to clients.
   turn:
-    turn_user_lifetime: ""
+    turn_user_lifetime: "5m"
     turn_uris:
     #  - turn:turn.server.org?transport=udp
     #  - turn:turn.server.org?transport=tcp
     turn_shared_secret: ""
-    turn_username: ""
+    # If your TURN server requires static credentials, then you will need to enter
-    turn_password: ""
+    # them here instead of supplying a shared secret. Note that these credentials
+    # will be visible to clients!
+    # turn_username: ""
+    # turn_password: ""
 
   # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
   # number of "slots" have been taken by requests from a specific host. Each "slot"
@@ -164,7 +195,7 @@ client_api:
   # and appservice users are exempt from rate limiting by default.
   rate_limiting:
     enabled: true
-    threshold: 5
+    threshold: 20
     cooloff_ms: 500
     exempt_user_ids:
     #  - "@user:domain.com"

dendrite-sample.polylith.yaml

@@ -31,10 +31,33 @@ global:
   # considered valid by other homeservers.
   key_validity_period: 168h0m0s
 
+  # Configuration for in-memory caches. Caches can often improve performance by
+  # keeping frequently accessed items (like events, identifiers etc.) in memory
+  # rather than having to read them from the database.
+  cache:
+    # The estimated maximum size for the global cache in bytes, or in terabytes,
+    # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
+    # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
+    # memory limit for the entire process. A cache that is too small may ultimately
+    # provide little or no benefit.
+    max_size_estimated: 1gb
+
+    # The maximum amount of time that a cache entry can live for in memory before
+    # it will be evicted and/or refreshed from the database. Lower values result in
+    # easier admission of new cache entries but may also increase database load in
+    # comparison to higher values, so adjust conservatively. Higher values may make
+    # it harder for new items to make it into the cache, e.g. if new rooms suddenly
+    # become popular.
+    max_age: 1h
+
   # The server name to delegate server-server communications to, with optional port
   # e.g. localhost:443
   well_known_server_name: ""
 
+  # The server name to delegate client-server communications to, with optional port
+  # e.g. localhost:443
+  well_known_client_name: ""
+
   # Lists of domains that the server will trust as identity servers to verify third
   # party identifiers such as phone numbers and email addresses.
   trusted_third_party_id_servers:
@@ -80,6 +103,11 @@ global:
     addresses:
       - hostname:4222
 
+    # Disable the validation of TLS certificates of NATS. This is
+    # not recommended in production since it may allow NATS traffic
+    # to be sent to an insecure endpoint.
+    disable_tls_validation: false
+
     # The prefix to use for stream names for this homeserver - really only useful
     # if you are running more than one Dendrite server on the same NATS deployment.
     topic_prefix: Dendrite
@@ -106,7 +134,7 @@ app_service_api:
 
   # Database configuration for this component.
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_appservice?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_appservice?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -153,13 +181,16 @@ client_api:
 
   # TURN server information that this homeserver should send to clients.
   turn:
-    turn_user_lifetime: ""
+    turn_user_lifetime: "5m"
     turn_uris:
     #  - turn:turn.server.org?transport=udp
     #  - turn:turn.server.org?transport=tcp
     turn_shared_secret: ""
-    turn_username: ""
+    # If your TURN server requires static credentials, then you will need to enter
-    turn_password: ""
+    # them here instead of supplying a shared secret. Note that these credentials
+    # will be visible to clients!
+    # turn_username: ""
+    # turn_password: ""
 
   # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
   # number of "slots" have been taken by requests from a specific host. Each "slot"
@@ -167,7 +198,7 @@ client_api:
   # and appservice users are exempt from rate limiting by default.
   rate_limiting:
     enabled: true
-    threshold: 5
+    threshold: 20
     cooloff_ms: 500
     exempt_user_ids:
     #  - "@user:domain.com"
@@ -180,7 +211,7 @@ federation_api:
   external_api:
     listen: http://[::]:8072
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_federationapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_federationapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -217,7 +248,7 @@ key_server:
     listen: http://[::]:7779 # The listen address for incoming API requests
     connect: http://key_server:7779 # The connect address for other components to use
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_keyserver?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_keyserver?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -230,7 +261,7 @@ media_api:
   external_api:
     listen: http://[::]:8074
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_mediaapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_mediaapi?sslmode=disable
     max_open_conns: 5
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -267,7 +298,7 @@ mscs:
   #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
   #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_mscs?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_mscs?sslmode=disable
     max_open_conns: 5
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -278,7 +309,7 @@ room_server:
     listen: http://[::]:7770 # The listen address for incoming API requests
     connect: http://room_server:7770 # The connect address for other components to use
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_roomserver?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_roomserver?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -291,7 +322,7 @@ sync_api:
   external_api:
     listen: http://[::]:8073
   database:
-    connection_string: postgresql://username@password:hostname/dendrite_syncapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_syncapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1
@@ -311,7 +342,7 @@ user_api:
     listen: http://[::]:7781 # The listen address for incoming API requests
     connect: http://user_api:7781 # The connect address for other components to use
   account_database:
-    connection_string: postgresql://username@password:hostname/dendrite_userapi?sslmode=disable
+    connection_string: postgresql://username:password@hostname/dendrite_userapi?sslmode=disable
     max_open_conns: 10
     max_idle_conns: 2
     conn_max_lifetime: -1

docs/CONTRIBUTING.md

@@ -24,7 +24,7 @@ Unfortunately we can't accept contributions without it.
 
 ## Getting up and running
 
-See the [Installation](INSTALL.md) section for information on how to build an
+See the [Installation](installation) section for information on how to build an
 instance of Dendrite. You will likely need this in order to test your changes.
 
 ## Code style
@@ -64,7 +64,7 @@ comment. Please avoid doing this if you can.
 We also have unit tests which we run via:
 
 ```bash
-go test ./...
+go test --race ./...
 ```
 
 In general, we like submissions that come with tests. Anything that proves that the

docs/FAQ.md

@@ -12,7 +12,13 @@ Mostly, although there are still bugs and missing features. If you are a confide
 
 ## Is Dendrite feature-complete?
 
-No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information.
+No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information.
+
+## Why doesn't Dendrite have "x" yet?
+
+Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
+specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 
+the development efforts through [contributing](https://matrix-org.github.io/dendrite/development/contributing). 
 
 ## Is there a migration path from Synapse to Dendrite?
 
@@ -43,6 +49,20 @@ It should do, although we are aware of some minor issues:
 * **Element Android**: registration does not work, but logging in with an existing account does
 * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
 
+## Does Dendrite support Space Summaries?
+
+Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
+
+```
+mscs:
+  mscs:
+    - msc2946
+```
+
+Similarly, [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported.
+
+Please note that MSCs should be considered experimental and can result in significant usability issues when enabled. If you'd like more details on how MSCs are ratified or the current status of MSCs, please see the [Matrix specification documentation](https://spec.matrix.org/proposals/) on the subject.
+
 ## Does Dendrite support push notifications?
 
 Yes, we have experimental support for push notifications. Configure them in the usual way in your Matrix client.
@@ -86,9 +106,16 @@ would be a huge help too, as that will help us to understand where the memory us
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!
 
-## What is being reported when enabling anonymous stats?
+## VOIP and Video Calls don't appear to work on Dendrite
 
-If anonymous stats reporting is enabled, the following data is send to the defined endpoint.
+There is likely an issue with your STUN/TURN configuration on the server. If you believe your configuration to be correct, please see the [troubleshooting](administration/5_troubleshooting.md) for troubleshooting recommendations.
+
+## What is being reported when enabling phone-home statistics?
+
+Phone-home statistics contain your server's domain name, some configuration information about
+your deployment and aggregated information about active users on your deployment. They are sent
+to the endpoint URL configured in your Dendrite configuration file only. The following is an
+example of the data that is sent:
 
 ```json
 {
@@ -106,7 +133,7 @@ If anonymous stats reporting is enabled, the following data is send to the defin
     "go_arch": "amd64",
     "go_os": "linux",
     "go_version": "go1.16.13",
-    "homeserver": "localhost:8800",
+    "homeserver": "my.domain.com",
     "log_level": "trace",
     "memory_rss": 93452,
     "monolith": true,

docs/Gemfile.lock

@@ -233,6 +233,8 @@ GEM
     multipart-post (2.1.1)
     nokogiri (1.13.6-arm64-darwin)
       racc (~> 1.4)
+    nokogiri (1.13.6-x86_64-linux)
+      racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
@@ -263,7 +265,7 @@ GEM
     thread_safe (0.3.6)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
-    tzinfo (1.2.9)
+    tzinfo (1.2.10)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
@@ -273,11 +275,11 @@ GEM
 
 PLATFORMS
   arm64-darwin-21
+  x86_64-linux
 
 DEPENDENCIES
   github-pages (~> 226)
   jekyll-feed (~> 0.15.1)
-  minima (~> 2.5.1)
 
 BUNDLED WITH
    2.3.7

docs/administration/1_createusers.md

@@ -14,9 +14,8 @@ User accounts can be created on a Dendrite instance in a number of ways.
 The `create-account` tool is built in the `bin` folder when building Dendrite with
 the `build.sh` script.
 
-It uses the `dendrite.yaml` configuration file to connect to the Dendrite user database
+It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires 
-and create the account entries directly. It can therefore be used even if Dendrite is not
+shared secret registration to be enabled as explained below.
-running yet, as long as the database is up.
 
 An example of using `create-account` to create a **normal account**:
 
@@ -32,6 +31,22 @@ To create a new **admin account**, add the `-admin` flag:
 ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
 ```
 
+By default `create-account` uses `https://localhost:8448` to connect to Dendrite, this can be overwritten using 
+the `-url` flag:
+
+```bash
+./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url http://localhost:8008
+```
+
+An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`:
+
+```bash
+docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME
+```
+```bash
+docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
+```
+
 ## Using shared secret registration
 
 Dendrite supports the Synapse-compatible shared secret registration endpoint.

docs/administration/4_adminapi.md

@@ -13,19 +13,78 @@ without warning.
 
 More endpoints will be added in the future.
 
-## `/_dendrite/admin/evacuateRoom/{roomID}`
+Endpoints may be used directly through curl:
+
+```
+curl --header "Authorization: Bearer <access_token>" -X <POST|GET|PUT> <Endpoint URI> -d '<Request Body Contents>'
+```
+
+An `access_token` can be obtained through most Element-based matrix clients by going to `Settings` -> `Help & About` -> `Advanced` -> `Access Token`.
+Be aware that an `access_token` allows a client to perform actions as an user and should be kept **secret**.
+
+The user must be an administrator in the `account_accounts` table in order to use these endpoints.
+
+Existing user accounts can be set to administrative accounts by changing `account_type` to `3` in `account_accounts`
+
+```
+UPDATE account_accounts SET account_type = 3 WHERE localpart = '$localpart';
+```
+
+Where `$localpart` is the username only (e.g. `alice`).
+
+## GET `/_dendrite/admin/evacuateRoom/{roomID}`
 
 This endpoint will instruct Dendrite to part all local users from the given `roomID`
 in the URL. It may take some time to complete. A JSON body will be returned containing
 the user IDs of all affected users.
 
-## `/_dendrite/admin/evacuateUser/{userID}`
+## GET `/_dendrite/admin/evacuateUser/{userID}`
 
 This endpoint will instruct Dendrite to part the given local `userID` in the URL from
 all rooms which they are currently joined. A JSON body will be returned containing
 the room IDs of all affected rooms.
 
-## `/_synapse/admin/v1/register`
+## POST `/_dendrite/admin/resetPassword/{localpart}`
+
+Request body format:
+
+```
+{
+    "password": "new_password_here"
+}
+```
+
+Reset the password of a local user. The `localpart` is the username only, i.e. if
+the full user ID is `@alice:domain.com` then the local part is `alice`.
+
+## POST `/_synapse/admin/v1/send_server_notice`
+
+Request body format:
+```
+{
+    "user_id": "@target_user:server_name",
+    "content": {
+       "msgtype": "m.text",
+       "body": "This is my message"
+    }
+}
+```
+
+Send a server notice to a specific user. See the [Matrix Spec](https://spec.matrix.org/v1.3/client-server-api/#server-notices) for additional details on server notice behaviour.
+If successfully sent, the API will return the following response:
+
+```
+{
+     "event_id": "<event_id>"
+}
+```
+
+## GET `/_synapse/admin/v1/register`
 
 Shared secret registration — please see the [user creation page](createusers) for
 guidance on configuring and using this endpoint.
+
+## GET `/_matrix/client/v3/admin/whois/{userId}`
+
+From the [Matrix Spec](https://spec.matrix.org/v1.3/client-server-api/#get_matrixclientv3adminwhoisuserid). 
+Gets information about a particular user. `userId` is the full user ID (e.g. `@alice:domain.com`)

docs/administration/5_troubleshooting.md

@@ -0,0 +1,88 @@
+---
+title: Troubleshooting
+parent: Administration
+permalink: /administration/troubleshooting
+---
+
+# Troubleshooting
+
+If your Dendrite installation is acting strangely, there are a few things you should
+check before seeking help.
+
+## 1. Logs
+
+Dendrite, by default, will log all warnings and errors to stdout, in addition to any
+other locations configured in the `dendrite.yaml` configuration file. Often there will
+be clues in the logs.
+
+You can increase this log level to the more verbose `debug` level if necessary by adding
+this to the config and restarting Dendrite:
+
+```
+logging:
+- type: std
+  level: debug
+```
+
+Look specifically for lines that contain `level=error` or `level=warning`.
+
+## 2. Federation tester
+
+If you are experiencing problems federating with other homeservers, you should check
+that the [Federation Tester](https://federationtester.matrix.org) is passing for your
+server.
+
+Common reasons that it may not pass include:
+
+1. Incorrect DNS configuration;
+2. Misconfigured DNS SRV entries or well-known files;
+3. Invalid TLS/SSL certificates;
+4. Reverse proxy configuration issues (if applicable).
+
+Correct any errors if shown and re-run the federation tester to check the results.
+
+## 3. System time
+
+Matrix relies heavily on TLS which requires the system time to be correct. If the clock
+drifts then you may find that federation no works reliably (or at all) and clients may
+struggle to connect to your Dendrite server.
+
+Ensure that your system time is correct and consider syncing to a reliable NTP source.
+
+## 4. Database connections
+
+If you are using the PostgreSQL database, you should ensure that Dendrite's configured
+number of database connections does not exceed the maximum allowed by PostgreSQL.
+
+Open your `postgresql.conf` configuration file and check the value of `max_connections`
+(which is typically `100` by default). Then open your `dendrite.yaml` configuration file
+and ensure that:
+
+1. If you are using the `global.database` section, that `max_open_conns` does not exceed
+   that number;
+2. If you are **not** using the `global.database` section, that the sum total of all
+   `max_open_conns` across all `database` blocks does not exceed that number.
+
+## 5. File descriptors
+
+Dendrite requires a sufficient number of file descriptors for every connection it makes
+to a remote server, every connection to the database engine and every file it is reading
+or writing to at a given time (media, logs etc). We recommend ensuring that the limit is
+no lower than 65535 for Dendrite.
+
+Dendrite will check at startup if there are a sufficient number of available descriptors.
+If there aren't, you will see a log lines like this:
+
+```
+level=warning msg="IMPORTANT: Process file descriptor limit is currently 65535, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
+```
+
+Follow the [Optimisation](../installation/11_optimisation.md) instructions to correct the
+available number of file descriptors.
+
+## 6. STUN/TURN Server tester
+
+If you are experiencing problems with VoIP or video calls, you should check that Dendrite
+is able to successfully connect your TURN server using 
+[Matrix VoIP Tester](https://test.voip.librepush.net/). This can highlight any issues
+that the server may encounter so that you can begin the troubleshooting process.

docs/caddy/monolith/CaddyFile

@@ -1,68 +0,0 @@
-{
-    # debug
-    admin off
-    email example@example.com
-    default_sni example.com
-    # Debug endpoint
-    # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory	
-}
-
-#######################################################################
-#   Snippets
-#______________________________________________________________________
-
-(handle_errors_maintenance) {
-	handle_errors {
-		@maintenance expression {http.error.status_code} == 502
-		rewrite @maintenance maintenance.html
-		root * "/path/to/service/pages"
-		file_server
-	}
-}
-
-(matrix-well-known-header) {
-    # Headers
-    header Access-Control-Allow-Origin "*"
-    header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
-    header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
-    header Content-Type "application/json"
-}
-
-#######################################################################
-
-example.com {
-
-	# ...
-
-	handle /.well-known/matrix/server {
-		import matrix-well-known-header
-		respond `{ "m.server": "matrix.example.com:443" }` 200
-	}
-	
-	handle /.well-known/matrix/client {
-		import matrix-well-known-header
-		respond `{ "m.homeserver": { "base_url": "https://matrix.example.com" } }` 200
-	}
-
-	import handle_errors_maintenance
-}
-
-example.com:8448 {
-	# server<->server HTTPS traffic
-    reverse_proxy http://dendrite-host:8008
-}
-
-matrix.example.com {
-
-	handle /_matrix/* {
-		# client<->server HTTPS traffic
-		reverse_proxy  http://dendrite-host:8008
-	}
-
-	handle_path /* {
-		# Client webapp (Element SPA or ...)
-		file_server {
-			root /path/to/www/example.com/matrix-web-client/
-		}	
-	}
-}

docs/caddy/monolith/Caddyfile

@@ -0,0 +1,57 @@
+# Sample Caddyfile for using Caddy in front of Dendrite.
+#
+# Customize email address and domain names.
+# Optional settings commented out.
+#
+# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST.
+# Documentation: https://caddyserver.com/docs/
+#
+# Bonus tip: If your IP address changes, use Caddy's
+# dynamic DNS plugin to update your DNS records to
+# point to your new IP automatically:
+# https://github.com/mholt/caddy-dynamicdns
+#
+
+
+# Global options block
+{
+	# In case there is a problem with your certificates.
+	# email example@example.com
+
+	# Turn off the admin endpoint if you don't need graceful config
+	# changes and/or are running untrusted code on your machine.
+	# admin off
+
+	# Enable this if your clients don't send ServerName in TLS handshakes.
+	# default_sni example.com
+
+	# Enable debug mode for verbose logging.
+	# debug
+
+	# Use Let's Encrypt's staging endpoint for testing.
+	# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
+
+	# If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
+	# else, enable these and put the alternate port numbers here.
+	# http_port  8080
+	# https_port 8443
+}
+
+# The server name of your matrix homeserver. This example shows
+# "well-known delegation" from the registered domain to a subdomain,
+# which is only needed if your server_name doesn't match your Matrix
+# homeserver URL (i.e. you can show users a vanity domain that looks
+# nice and is easy to remember but still have your Matrix server on
+# its own subdomain or hosted service).
+example.com {
+	header /.well-known/matrix/* Content-Type application/json
+	header /.well-known/matrix/* Access-Control-Allow-Origin *
+	respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
+	respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
+}
+
+# The actual domain name whereby your Matrix server is accessed.
+matrix.example.com {
+	# Set localhost:8008 to the address of your Dendrite server, if different
+	reverse_proxy /_matrix/* localhost:8008
+}

docs/caddy/polylith/Caddyfile

@@ -0,0 +1,66 @@
+# Sample Caddyfile for using Caddy in front of Dendrite.
+#
+# Customize email address and domain names.
+# Optional settings commented out.
+#
+# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST.
+# Documentation: https://caddyserver.com/docs/
+#
+# Bonus tip: If your IP address changes, use Caddy's
+# dynamic DNS plugin to update your DNS records to
+# point to your new IP automatically:
+# https://github.com/mholt/caddy-dynamicdns
+#
+
+
+# Global options block
+{
+	# In case there is a problem with your certificates.
+	# email example@example.com
+
+	# Turn off the admin endpoint if you don't need graceful config
+	# changes and/or are running untrusted code on your machine.
+	# admin off
+
+	# Enable this if your clients don't send ServerName in TLS handshakes.
+	# default_sni example.com
+
+	# Enable debug mode for verbose logging.
+	# debug
+
+	# Use Let's Encrypt's staging endpoint for testing.
+	# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
+
+	# If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
+	# else, enable these and put the alternate port numbers here.
+	# http_port  8080
+	# https_port 8443
+}
+
+# The server name of your matrix homeserver. This example shows
+# "well-known delegation" from the registered domain to a subdomain,
+# which is only needed if your server_name doesn't match your Matrix
+# homeserver URL (i.e. you can show users a vanity domain that looks
+# nice and is easy to remember but still have your Matrix server on
+# its own subdomain or hosted service).
+example.com {
+	header /.well-known/matrix/* Content-Type application/json
+	header /.well-known/matrix/* Access-Control-Allow-Origin *
+	respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
+	respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
+}
+
+# The actual domain name whereby your Matrix server is accessed.
+matrix.example.com {
+	# Change the end of each reverse_proxy line to the correct
+	# address for your various services.
+	@sync_api {
+		path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$
+	}
+	reverse_proxy @sync_api sync_api:8073
+
+	reverse_proxy /_matrix/client* client_api:8071
+	reverse_proxy /_matrix/federation* federation_api:8071
+	reverse_proxy /_matrix/key* federation_api:8071
+	reverse_proxy /_matrix/media* media_api:8071
+}

docs/installation/10_starting_polylith.md

@@ -2,7 +2,7 @@
 title: Starting the polylith
 parent: Installation
 has_toc: true
-nav_order: 9
+nav_order: 10
 permalink: /installation/start/polylith
 ---
 

docs/installation/11_optimisation.md

@@ -2,7 +2,7 @@
 title: Optimise your installation
 parent: Installation
 has_toc: true
-nav_order: 10
+nav_order: 11
 permalink: /installation/start/optimisation
 ---
 

docs/installation/1_planning.md

@@ -75,7 +75,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie
 
 ### Go
 
-At this time, Dendrite supports being built with Go 1.16 or later. We do not support building
+At this time, Dendrite supports being built with Go 1.18 or later. We do not support building
 Dendrite with older versions of Go than this. If you are installing Go using a package manager,
 you should check (by running `go version`) that you are using a suitable version before you start.
 
@@ -95,12 +95,13 @@ enabled.
 To do so, follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). JetStream must be enabled, either by passing the `-js` flag to `nats-server`,
 or by specifying the `store_dir` option in the the `jetstream` configuration.
 
-### Reverse proxy (polylith deployments)
+### Reverse proxy
 
-Polylith deployments require a reverse proxy, such as [NGINX](https://www.nginx.com) or
+A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or
-[HAProxy](http://www.haproxy.org). Configuring those is not covered in this documentation,
+[HAProxy](http://www.haproxy.org) is required for polylith deployments and is useful for monolith
-although a [sample configuration for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
+deployments. Configuring those is not covered in this documentation, although sample configurations
-is provided.
+for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and
+[NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided.
 
 ### Windows
 

docs/installation/2_domainname.md

@@ -14,27 +14,38 @@ that take the format `@user:example.com`.
 For federation to work, the server name must be resolvable by other homeservers on the internet
 — that is, the domain must be registered and properly configured with the relevant DNS records.
 
-Matrix servers discover each other when federating using the following methods:
+Matrix servers usually discover each other when federating using the following methods:
 
-1. If a well-known delegation exists on `example.com`, use the path server from the
+1. If a well-known delegation exists on `example.com`, use the domain and port from the
    well-known file to connect to the remote homeserver;
-2. If a DNS SRV delegation exists on `example.com`, use the hostname and port from the DNS SRV
+2. If a DNS SRV delegation exists on `example.com`, use the IP address and port from the DNS SRV
    record to connect to the remote homeserver;
 3. If neither well-known or DNS SRV delegation are configured, attempt to connect to the remote
    homeserver by connecting to `example.com` port TCP/8448 using HTTPS.
 
+The exact details of how server name resolution works can be found in
+[the spec](https://spec.matrix.org/v1.3/server-server-api/#resolving-server-names).
+
 ## TLS certificates
 
 Matrix federation requires that valid TLS certificates are present on the domain. You must
-obtain certificates from a publicly accepted Certificate Authority (CA). [LetsEncrypt](https://letsencrypt.org)
+obtain certificates from a publicly-trusted certificate authority (CA). [Let's Encrypt](https://letsencrypt.org)
-is an example of such a CA that can be used. Self-signed certificates are not suitable for
+is a popular choice of CA because the certificates are publicly-trusted, free, and automated
-federation and will typically not be accepted by other homeservers.
+via the ACME protocol. (Self-signed certificates are not suitable for federation and will typically
+not be accepted by other homeservers.)
 
-A common practice to help ease the management of certificates is to install a reverse proxy in
+Automating the renewal of TLS certificates is best practice. There are many tools for this,
-front of Dendrite which manages the TLS certificates and HTTPS proxying itself. Software such as
+but the simplest way to achieve TLS automation is to have your reverse proxy do it for you.
-[NGINX](https://www.nginx.com) and [HAProxy](http://www.haproxy.org) can be used for the task.
+[Caddy](https://caddyserver.com) is recommended as a production-grade reverse proxy with
-Although the finer details of configuring these are not described here, you must reverse proxy
+automatic TLS which is commonly used in front of Dendrite. It obtains and renews TLS certificates
-all `/_matrix` paths to your Dendrite server.
+automatically and by default as long as your domain name is pointed at your server first.
+Although the finer details of [configuring Caddy](https://caddyserver.com/docs/) is not described
+here, in general, you must reverse proxy all `/_matrix` paths to your Dendrite server. For example,
+with Caddy:
+
+```
+reverse_proxy /_matrix/* localhost:8008
+```
 
 It is possible for the reverse proxy to listen on the standard HTTPS port TCP/443 so long as your
 domain delegation is configured to point to port TCP/443.
@@ -51,17 +62,12 @@ you will be able to delegate from `example.com` to `matrix.example.com` so that
 
 Delegation can be performed in one of two ways:
 
-* **Well-known delegation**: A well-known text file is served over HTTPS on the domain name
+* **Well-known delegation (preferred)**: A well-known text file is served over HTTPS on the domain
-  that you want to use, pointing to your server on `matrix.example.com` port 8448;
+  name that you want to use, pointing to your server on `matrix.example.com` port 8448;
-* **DNS SRV delegation**: A DNS SRV record is created on the domain name that you want to
+* **DNS SRV delegation (not recommended)**: See the SRV delegation section below for details.
-  use, pointing to your server on `matrix.example.com` port TCP/8448.
 
-If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or DNS SRV
+If you are using a reverse proxy to forward `/_matrix` to Dendrite, your well-known or delegation
-delegation must refer to the hostname and port that the reverse proxy is listening on instead.
+must refer to the hostname and port that the reverse proxy is listening on instead.
-
-Well-known delegation is typically easier to set up and usually preferred. However, you can use
-either or both methods to delegate. If you configure both methods of delegation, it is important
-that they both agree and refer to the same hostname and port.
 
 ## Well-known delegation
 
@@ -74,20 +80,46 @@ and contain the following JSON document:
 
 ```json
 {
-    "m.server": "https://matrix.example.com:8448"
+    "m.server": "matrix.example.com:8448"
 }
 ```
 
+For example, this can be done with the following Caddy config:
+
+```
+handle /.well-known/matrix/client {
+	header Content-Type application/json
+	header Access-Control-Allow-Origin *
+	respond `{"m.homeserver": {"base_url": "https://matrix.example.com:8448"}}`
+}
+```
+
+You can also serve `.well-known` with Dendrite itself by setting the `well_known_server_name` config
+option to the value you want for `m.server`. This is primarily useful if Dendrite is exposed on
+`example.com:443` and you don't want to set up a separate webserver just for serving the `.well-known`
+file.
+
+```yaml
+global:
+...
+   well_known_server_name: "example.com:443"
+```
+
 ## DNS SRV delegation
 
-Using DNS SRV delegation requires creating DNS SRV records on the `example.com` zone which
+This method is not recommended, as the behavior of SRV records in Matrix is rather unintuitive:
-refer to your Dendrite installation.
+SRV records will only change the IP address and port that other servers connect to, they won't
+affect the domain name. In technical terms, the `Host` header and TLS SNI of federation requests
+will still be `example.com` even if the SRV record points at `matrix.example.com`.
 
-Assuming that your Dendrite installation is listening for HTTPS connections at `matrix.example.com`
+In practice, this means that the server must be configured with valid TLS certificates for
-port 8448, the DNS SRV record must have the following fields:
+`example.com`, rather than `matrix.example.com` as one might intuitively expect. If there's a
+reverse proxy in between, the proxy configuration must be written as if it's `example.com`, as the
+proxy will never see the name `matrix.example.com` in incoming requests.
 
-* Name: `@` (or whichever term your DNS provider uses to signal the root)
+This behavior also means that if `example.com` and `matrix.example.com` point at the same IP
-* Service: `_matrix`
+address, there is no reason to have a SRV record pointing at `matrix.example.com`. It can still
-* Protocol: `_tcp`
+be used to change the port number, but it won't do anything else.
-* Port: `8448`
+
-* Target: `matrix.example.com`
+If you understand how SRV records work and still want to use them, the service name is `_matrix` and
+the protocol is `_tcp`.

docs/installation/3_build.md

@@ -0,0 +1,38 @@
+---
+title: Building Dendrite
+parent: Installation
+has_toc: true
+nav_order: 3
+permalink: /installation/build
+---
+
+# Build all Dendrite commands
+
+Dendrite has numerous utility commands in addition to the actual server binaries.
+Build them all from the root of the source repo with `build.sh` (Linux/Mac):
+
+```sh
+./build.sh
+```
+
+or `build.cmd` (Windows):
+
+```powershell
+build.cmd
+```
+
+The resulting binaries will be placed in the `bin` subfolder.
+
+# Installing as a monolith
+
+You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`:
+
+```sh
+go install ./cmd/dendrite-monolith-server
+```
+
+Alternatively, you can specify a custom path for the binary to be written to using `go build`:
+
+```sh
+go build -o /usr/local/bin/ ./cmd/dendrite-monolith-server
+```

docs/installation/4_database.md

@@ -17,7 +17,9 @@ filenames in the Dendrite configuration file and start Dendrite. The databases w
 and populated automatically.
 
 Note that Dendrite **cannot share a single SQLite database across multiple components**. Each
-component must be configured with its own SQLite database filename.
+component must be configured with its own SQLite database filename. You will have to remove
+the `global.database` section from your Dendrite config and add it to each individual section
+instead in order to use SQLite.
 
 ### Connection strings
 

docs/installation/6_install_polylith.md

@@ -29,5 +29,6 @@ Polylith deployments require a reverse proxy in order to ensure that requests ar
 sent to the correct endpoint. You must ensure that a suitable reverse proxy is installed
 and configured.
 
-A [sample configuration file](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
+Sample configurations are provided
-is provided for [NGINX](https://www.nginx.com).
+for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy/polylith/Caddyfile)
+and [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf).

docs/installation/7_configuration.md

@@ -1,13 +1,13 @@
 ---
-title: Populate the configuration
+title: Configuring Dendrite
 parent: Installation
 nav_order: 7
 permalink: /installation/configuration
 ---
 
-# Populate the configuration
+# Configuring Dendrite
 
-The configuration file is used to configure Dendrite. Sample configuration files are
+A YAML configuration file is used to configure Dendrite. Sample configuration files are
 present in the top level of the Dendrite repository:
 
 * [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml)

docs/installation/8_signingkey.md

@@ -1,7 +1,7 @@
 ---
 title: Generating signing keys
 parent: Installation
-nav_order: 4
+nav_order: 8
 permalink: /installation/signingkeys
 ---
 

docs/installation/9_starting_monolith.md

@@ -15,8 +15,9 @@ you can start your Dendrite monolith deployment by starting the `dendrite-monoli
 ./dendrite-monolith-server -config /path/to/dendrite.yaml
 ```
 
-If you want to change the addresses or ports that Dendrite listens on, you
+By default, Dendrite will listen HTTP on port 8008. If you want to change the addresses
-can use the `-http-bind-address` and `-https-bind-address` command line arguments:
+or ports that Dendrite listens on, you can use the `-http-bind-address` and
+`-https-bind-address` command line arguments:
 
 ```bash
 ./dendrite-monolith-server -config /path/to/dendrite.yaml \

federationapi/api/api.go

@@ -110,7 +110,7 @@ type FederationClientError struct {
 	Blacklisted bool
 }
 
-func (e *FederationClientError) Error() string {
+func (e FederationClientError) Error() string {
 	return fmt.Sprintf("%s - (retry_after=%s, blacklisted=%v)", e.Err, e.RetryAfter.String(), e.Blacklisted)
 }
 

federationapi/consumers/roomserver.go

@@ -208,9 +208,11 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 // joinedHostsAtEvent works out a list of matrix servers that were joined to
 // the room at the event (including peeking ones)
 // It is important to use the state at the event for sending messages because:
-//   1) We shouldn't send messages to servers that weren't in the room.
+//
-//   2) If a server is kicked from the rooms it should still be told about the
+//  1. We shouldn't send messages to servers that weren't in the room.
-//      kick event,
+//  2. If a server is kicked from the rooms it should still be told about the
+//     kick event.
+//
 // Usually the list can be calculated locally, but sometimes it will need fetch
 // events from the room server.
 // Returns an error if there was a problem talking to the room server.

federationapi/consumers/sendtodevice.go

@@ -95,6 +95,11 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msg *nats.Ms
 		return true
 	}
 
+	// The SyncAPI is already handling sendToDevice for the local server
+	if destServerName == t.ServerName {
+		return true
+	}
+
 	// Pack the EDU and marshal it
 	edu := &gomatrixserverlib.EDU{
 		Type:   gomatrixserverlib.MDirectToDevice,

federationapi/federationapi.go

@@ -15,6 +15,8 @@
 package federationapi
 
 import (
+	"time"
+
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
@@ -167,5 +169,16 @@ func NewInternalAPI(
 	if err = presenceConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start presence consumer")
 	}
+
+	var cleanExpiredEDUs func()
+	cleanExpiredEDUs = func() {
+		logrus.Infof("Cleaning expired EDUs")
+		if err := federationDB.DeleteExpiredEDUs(base.Context()); err != nil {
+			logrus.WithError(err).Error("Failed to clean expired EDUs")
+		}
+		time.AfterFunc(time.Hour, cleanExpiredEDUs)
+	}
+	time.AfterFunc(time.Minute, cleanExpiredEDUs)
+
 	return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, stats, caches, queues, keyRing)
 }

federationapi/federationapi_keys_test.go

@@ -6,10 +6,9 @@ import (
 	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"os"
-	"reflect"
 	"testing"
 	"time"
 
@@ -64,13 +63,10 @@ func TestMain(m *testing.M) {
 			}
 
 			// Create a new cache but don't enable prometheus!
-			s.cache, err = caching.NewInMemoryLRUCache(false)
+			s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false)
-			if err != nil {
-				panic("can't create cache: " + err.Error())
-			}
 
 			// Create a temporary directory for JetStream.
-			d, err := ioutil.TempDir("./", "jetstream*")
+			d, err := os.MkdirTemp("./", "jetstream*")
 			if err != nil {
 				panic(err)
 			}
@@ -140,7 +136,7 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err
 	// And respond.
 	res = &http.Response{
 		StatusCode: 200,
-		Body:       ioutil.NopCloser(bytes.NewReader(body)),
+		Body:       io.NopCloser(bytes.NewReader(body)),
 	}
 	return
 }
@@ -170,72 +166,6 @@ func TestServersRequestOwnKeys(t *testing.T) {
 	}
 }
 
-func TestCachingBehaviour(t *testing.T) {
-	// Server A will request Server B's key, which has a validity
-	// period of an hour from now. We should retrieve the key and
-	// it should make it into the cache automatically.
-
-	req := gomatrixserverlib.PublicKeyLookupRequest{
-		ServerName: serverB.name,
-		KeyID:      serverKeyID,
-	}
-	ts := gomatrixserverlib.AsTimestamp(time.Now())
-
-	res, err := serverA.api.FetchKeys(
-		context.Background(),
-		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
-			req: ts,
-		},
-	)
-	if err != nil {
-		t.Fatalf("server A failed to retrieve server B key: %s", err)
-	}
-	if len(res) != 1 {
-		t.Fatalf("server B should have returned one key but instead returned %d keys", len(res))
-	}
-	if _, ok := res[req]; !ok {
-		t.Fatalf("server B isn't included in the key fetch response")
-	}
-
-	// At this point, if the previous key request was a success,
-	// then the cache should now contain the key. Check if that's
-	// the case - if it isn't then there's something wrong with
-	// the cache implementation or we failed to get the key.
-
-	cres, ok := serverA.cache.GetServerKey(req, ts)
-	if !ok {
-		t.Fatalf("server B key should be in cache but isn't")
-	}
-	if !reflect.DeepEqual(cres, res[req]) {
-		t.Fatalf("the cached result from server B wasn't what server B gave us")
-	}
-
-	// If we ask the cache for the same key but this time for an event
-	// that happened in +30 minutes. Since the validity period is for
-	// another hour, then we should get a response back from the cache.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*30)),
-	)
-	if !ok {
-		t.Fatalf("server B key isn't in cache when it should be (+30 minutes)")
-	}
-
-	// If we ask the cache for the same key but this time for an event
-	// that happened in +90 minutes then we should expect to get no
-	// cache result. This is because the cache shouldn't return a result
-	// that is obviously past the validity of the event.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*90)),
-	)
-	if ok {
-		t.Fatalf("server B key is in cache when it shouldn't be (+90 minutes)")
-	}
-}
-
 func TestRenewalBehaviour(t *testing.T) {
 	// Server A will request Server C's key but their validity period
 	// is an hour in the past. We'll retrieve the key as, even though it's
@@ -262,32 +192,7 @@ func TestRenewalBehaviour(t *testing.T) {
 		t.Fatalf("server C isn't included in the key fetch response")
 	}
 
-	// If we ask the cache for the server key for an event that happened
+	originalValidity := res[req].ValidUntilTS
-	// 90 minutes ago then we should get a cache result, as the key hadn't
-	// passed its validity by that point. The fact that the key is now in
-	// the cache is, in itself, proof that we successfully retrieved the
-	// key before.
-
-	oldcached, ok := serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*90)),
-	)
-	if !ok {
-		t.Fatalf("server C key isn't in cache when it should be (-90 minutes)")
-	}
-
-	// If we now ask the cache for the same key but this time for an event
-	// that only happened 30 minutes ago then we shouldn't get a cached
-	// result, as the event happened after the key validity expired. This
-	// is really just for sanity checking.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
-	)
-	if ok {
-		t.Fatalf("server B key is in cache when it shouldn't be (-30 minutes)")
-	}
 
 	// We're now going to kick server C into renewing its key. Since we're
 	// happy at this point that the key that we already have is from the past
@@ -308,24 +213,13 @@ func TestRenewalBehaviour(t *testing.T) {
 	if len(res) != 1 {
 		t.Fatalf("server C should have returned one key but instead returned %d keys", len(res))
 	}
-	if _, ok = res[req]; !ok {
+	if _, ok := res[req]; !ok {
 		t.Fatalf("server C isn't included in the key fetch response")
 	}
 
-	// We're now going to ask the cache what the new key validity is. If
+	currentValidity := res[req].ValidUntilTS
-	// it is still the same as the previous validity then we've failed to
-	// retrieve the renewed key. If it's newer then we've successfully got
-	// the renewed key.
 
-	newcached, ok := serverA.cache.GetServerKey(
+	if originalValidity == currentValidity {
-		req,
+		t.Fatalf("server C key should have renewed but didn't")
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
-	)
-	if !ok {
-		t.Fatalf("server B key isn't in cache when it shouldn't be (post-renewal)")
 	}
-	if oldcached.ValidUntilTS >= newcached.ValidUntilTS {
-		t.Fatalf("the server B key should have been renewed but wasn't")
-	}
-	t.Log(res)
 }

federationapi/federationapi_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -31,11 +32,12 @@ type fedRoomserverAPI struct {
 }
 
 // PerformJoin will call this function
-func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) {
+func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) error {
 	if f.inputRoomEvents == nil {
-		return
+		return nil
 	}
 	f.inputRoomEvents(ctx, req, res)
+	return nil
 }
 
 // keychange consumer calls this
@@ -48,6 +50,7 @@ func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.Que
 
 // TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate
 type fedClient struct {
+	fedClientMutex sync.Mutex
 	api.FederationClient
 	allowJoins []*test.Room
 	keys       map[gomatrixserverlib.ServerName]struct {
@@ -59,6 +62,8 @@ type fedClient struct {
 }
 
 func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) {
+	f.fedClientMutex.Lock()
+	defer f.fedClientMutex.Unlock()
 	fmt.Println("GetServerKeys:", matrixServer)
 	var keys gomatrixserverlib.ServerKeys
 	var keyID gomatrixserverlib.KeyID
@@ -122,6 +127,8 @@ func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName
 	return
 }
 func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
+	f.fedClientMutex.Lock()
+	defer f.fedClientMutex.Unlock()
 	for _, r := range f.allowJoins {
 		if r.ID == event.RoomID() {
 			r.InsertEvent(f.t, event.Headered(r.Version))
@@ -134,6 +141,8 @@ func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName
 }
 
 func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
+	f.fedClientMutex.Lock()
+	defer f.fedClientMutex.Unlock()
 	for _, edu := range t.EDUs {
 		if edu.Type == gomatrixserverlib.MDeviceListUpdate {
 			f.sentTxn = true
@@ -242,6 +251,8 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 
 	testrig.MustPublishMsgs(t, jsctx, msg)
 	time.Sleep(500 * time.Millisecond)
+	fc.fedClientMutex.Lock()
+	defer fc.fedClientMutex.Unlock()
 	if !fc.sentTxn {
 		t.Fatalf("did not send device list update")
 	}

federationapi/inthttp/client.go

@@ -10,7 +10,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/opentracing/opentracing-go"
 )
 
 // HTTP paths for the internal HTTP API
@@ -48,7 +47,11 @@ func NewFederationAPIClient(federationSenderURL string, httpClient *http.Client,
 	if httpClient == nil {
 		return nil, errors.New("NewFederationInternalAPIHTTP: httpClient is <nil>")
 	}
-	return &httpFederationInternalAPI{federationSenderURL, httpClient, cache}, nil
+	return &httpFederationInternalAPI{
+		federationAPIURL: federationSenderURL,
+		httpClient:       httpClient,
+		cache:            cache,
+	}, nil
 }
 
 type httpFederationInternalAPI struct {
@@ -63,11 +66,10 @@ func (h *httpFederationInternalAPI) PerformLeave(
 	request *api.PerformLeaveRequest,
 	response *api.PerformLeaveResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"PerformLeave", h.federationAPIURL+FederationAPIPerformLeaveRequestPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIPerformLeaveRequestPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // Handle sending an invite to a remote server.
@@ -76,11 +78,10 @@ func (h *httpFederationInternalAPI) PerformInvite(
 	request *api.PerformInviteRequest,
 	response *api.PerformInviteResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformInviteRequest")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"PerformInvite", h.federationAPIURL+FederationAPIPerformInviteRequestPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIPerformInviteRequestPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // Handle starting a peek on a remote server.
@@ -89,11 +90,10 @@ func (h *httpFederationInternalAPI) PerformOutboundPeek(
 	request *api.PerformOutboundPeekRequest,
 	response *api.PerformOutboundPeekResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformOutboundPeekRequest")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"PerformOutboundPeek", h.federationAPIURL+FederationAPIPerformOutboundPeekRequestPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIPerformOutboundPeekRequestPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // QueryJoinedHostServerNamesInRoom implements FederationInternalAPI
@@ -102,11 +102,10 @@ func (h *httpFederationInternalAPI) QueryJoinedHostServerNamesInRoom(
 	request *api.QueryJoinedHostServerNamesInRoomRequest,
 	response *api.QueryJoinedHostServerNamesInRoomResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"QueryJoinedHostServerNamesInRoom", h.federationAPIURL+FederationAPIQueryJoinedHostServerNamesInRoomPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIQueryJoinedHostServerNamesInRoomPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // Handle an instruction to make_join & send_join with a remote server.
@@ -115,12 +114,10 @@ func (h *httpFederationInternalAPI) PerformJoin(
 	request *api.PerformJoinRequest,
 	response *api.PerformJoinResponse,
 ) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest")
+	if err := httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"PerformJoinRequest", h.federationAPIURL+FederationAPIPerformJoinRequestPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIPerformJoinRequestPath
+	); err != nil {
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
-	if err != nil {
 		response.LastError = &gomatrix.HTTPError{
 			Message:      err.Error(),
 			Code:         0,
@@ -135,11 +132,10 @@ func (h *httpFederationInternalAPI) PerformDirectoryLookup(
 	request *api.PerformDirectoryLookupRequest,
 	response *api.PerformDirectoryLookupResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"PerformDirectoryLookup", h.federationAPIURL+FederationAPIPerformDirectoryLookupRequestPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIPerformDirectoryLookupRequestPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 // Handle an instruction to broadcast an EDU to all servers in rooms we are joined to.
@@ -148,101 +144,61 @@ func (h *httpFederationInternalAPI) PerformBroadcastEDU(
 	request *api.PerformBroadcastEDURequest,
 	response *api.PerformBroadcastEDUResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformBroadcastEDU")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"PerformBroadcastEDU", h.federationAPIURL+FederationAPIPerformBroadcastEDUPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIPerformBroadcastEDUPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 type getUserDevices struct {
 	S      gomatrixserverlib.ServerName
 	UserID string
-	Res    *gomatrixserverlib.RespUserDevices
-	Err    *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) GetUserDevices(
 	ctx context.Context, s gomatrixserverlib.ServerName, userID string,
 ) (gomatrixserverlib.RespUserDevices, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetUserDevices")
+	return httputil.CallInternalProxyAPI[getUserDevices, gomatrixserverlib.RespUserDevices, *api.FederationClientError](
-	defer span.Finish()
+		"GetUserDevices", h.federationAPIURL+FederationAPIGetUserDevicesPath, h.httpClient,
-
+		ctx, &getUserDevices{
-	var result gomatrixserverlib.RespUserDevices
+			S:      s,
-	request := getUserDevices{
+			UserID: userID,
-		S:      s,
+		},
-		UserID: userID,
+	)
-	}
-	var response getUserDevices
-	apiURL := h.federationAPIURL + FederationAPIGetUserDevicesPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return result, err
-	}
-	if response.Err != nil {
-		return result, response.Err
-	}
-	return *response.Res, nil
 }
 
 type claimKeys struct {
 	S           gomatrixserverlib.ServerName
 	OneTimeKeys map[string]map[string]string
-	Res         *gomatrixserverlib.RespClaimKeys
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) ClaimKeys(
 	ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
 ) (gomatrixserverlib.RespClaimKeys, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ClaimKeys")
+	return httputil.CallInternalProxyAPI[claimKeys, gomatrixserverlib.RespClaimKeys, *api.FederationClientError](
-	defer span.Finish()
+		"ClaimKeys", h.federationAPIURL+FederationAPIClaimKeysPath, h.httpClient,
-
+		ctx, &claimKeys{
-	var result gomatrixserverlib.RespClaimKeys
+			S:           s,
-	request := claimKeys{
+			OneTimeKeys: oneTimeKeys,
-		S:           s,
+		},
-		OneTimeKeys: oneTimeKeys,
+	)
-	}
-	var response claimKeys
-	apiURL := h.federationAPIURL + FederationAPIClaimKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return result, err
-	}
-	if response.Err != nil {
-		return result, response.Err
-	}
-	return *response.Res, nil
 }
 
 type queryKeys struct {
 	S    gomatrixserverlib.ServerName
 	Keys map[string][]string
-	Res  *gomatrixserverlib.RespQueryKeys
-	Err  *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) QueryKeys(
 	ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string,
 ) (gomatrixserverlib.RespQueryKeys, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeys")
+	return httputil.CallInternalProxyAPI[queryKeys, gomatrixserverlib.RespQueryKeys, *api.FederationClientError](
-	defer span.Finish()
+		"QueryKeys", h.federationAPIURL+FederationAPIQueryKeysPath, h.httpClient,
-
+		ctx, &queryKeys{
-	var result gomatrixserverlib.RespQueryKeys
+			S:    s,
-	request := queryKeys{
+			Keys: keys,
-		S:    s,
+		},
-		Keys: keys,
+	)
-	}
-	var response queryKeys
-	apiURL := h.federationAPIURL + FederationAPIQueryKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return result, err
-	}
-	if response.Err != nil {
-		return result, response.Err
-	}
-	return *response.Res, nil
 }
 
 type backfill struct {
@@ -250,32 +206,20 @@ type backfill struct {
 	RoomID   string
 	Limit    int
 	EventIDs []string
-	Res      *gomatrixserverlib.Transaction
-	Err      *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) Backfill(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string,
 ) (gomatrixserverlib.Transaction, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "Backfill")
+	return httputil.CallInternalProxyAPI[backfill, gomatrixserverlib.Transaction, *api.FederationClientError](
-	defer span.Finish()
+		"Backfill", h.federationAPIURL+FederationAPIBackfillPath, h.httpClient,
-
+		ctx, &backfill{
-	request := backfill{
+			S:        s,
-		S:        s,
+			RoomID:   roomID,
-		RoomID:   roomID,
+			Limit:    limit,
-		Limit:    limit,
+			EventIDs: eventIDs,
-		EventIDs: eventIDs,
+		},
-	}
+	)
-	var response backfill
-	apiURL := h.federationAPIURL + FederationAPIBackfillPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.Transaction{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.Transaction{}, response.Err
-	}
-	return *response.Res, nil
 }
 
 type lookupState struct {
@@ -283,63 +227,39 @@ type lookupState struct {
 	RoomID      string
 	EventID     string
 	RoomVersion gomatrixserverlib.RoomVersion
-	Res         *gomatrixserverlib.RespState
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupState(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
 ) (gomatrixserverlib.RespState, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupState")
+	return httputil.CallInternalProxyAPI[lookupState, gomatrixserverlib.RespState, *api.FederationClientError](
-	defer span.Finish()
+		"LookupState", h.federationAPIURL+FederationAPILookupStatePath, h.httpClient,
-
+		ctx, &lookupState{
-	request := lookupState{
+			S:           s,
-		S:           s,
+			RoomID:      roomID,
-		RoomID:      roomID,
+			EventID:     eventID,
-		EventID:     eventID,
+			RoomVersion: roomVersion,
-		RoomVersion: roomVersion,
+		},
-	}
+	)
-	var response lookupState
-	apiURL := h.federationAPIURL + FederationAPILookupStatePath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.RespState{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.RespState{}, response.Err
-	}
-	return *response.Res, nil
 }
 
 type lookupStateIDs struct {
 	S       gomatrixserverlib.ServerName
 	RoomID  string
 	EventID string
-	Res     *gomatrixserverlib.RespStateIDs
-	Err     *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupStateIDs(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string,
 ) (gomatrixserverlib.RespStateIDs, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupStateIDs")
+	return httputil.CallInternalProxyAPI[lookupStateIDs, gomatrixserverlib.RespStateIDs, *api.FederationClientError](
-	defer span.Finish()
+		"LookupStateIDs", h.federationAPIURL+FederationAPILookupStateIDsPath, h.httpClient,
-
+		ctx, &lookupStateIDs{
-	request := lookupStateIDs{
+			S:       s,
-		S:       s,
+			RoomID:  roomID,
-		RoomID:  roomID,
+			EventID: eventID,
-		EventID: eventID,
+		},
-	}
+	)
-	var response lookupStateIDs
-	apiURL := h.federationAPIURL + FederationAPILookupStateIDsPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.RespStateIDs{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.RespStateIDs{}, response.Err
-	}
-	return *response.Res, nil
 }
 
 type lookupMissingEvents struct {
@@ -347,64 +267,38 @@ type lookupMissingEvents struct {
 	RoomID      string
 	Missing     gomatrixserverlib.MissingEvents
 	RoomVersion gomatrixserverlib.RoomVersion
-	Res         struct {
-		Events []gomatrixserverlib.RawJSON `json:"events"`
-	}
-	Err *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupMissingEvents(
 	ctx context.Context, s gomatrixserverlib.ServerName, roomID string,
 	missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
 ) (res gomatrixserverlib.RespMissingEvents, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupMissingEvents")
+	return httputil.CallInternalProxyAPI[lookupMissingEvents, gomatrixserverlib.RespMissingEvents, *api.FederationClientError](
-	defer span.Finish()
+		"LookupMissingEvents", h.federationAPIURL+FederationAPILookupMissingEventsPath, h.httpClient,
-
+		ctx, &lookupMissingEvents{
-	request := lookupMissingEvents{
+			S:           s,
-		S:           s,
+			RoomID:      roomID,
-		RoomID:      roomID,
+			Missing:     missing,
-		Missing:     missing,
+			RoomVersion: roomVersion,
-		RoomVersion: roomVersion,
+		},
-	}
+	)
-	apiURL := h.federationAPIURL + FederationAPILookupMissingEventsPath
-	err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &request)
-	if err != nil {
-		return res, err
-	}
-	if request.Err != nil {
-		return res, request.Err
-	}
-	res.Events = request.Res.Events
-	return res, nil
 }
 
 type getEvent struct {
 	S       gomatrixserverlib.ServerName
 	EventID string
-	Res     *gomatrixserverlib.Transaction
-	Err     *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) GetEvent(
 	ctx context.Context, s gomatrixserverlib.ServerName, eventID string,
 ) (gomatrixserverlib.Transaction, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetEvent")
+	return httputil.CallInternalProxyAPI[getEvent, gomatrixserverlib.Transaction, *api.FederationClientError](
-	defer span.Finish()
+		"GetEvent", h.federationAPIURL+FederationAPIGetEventPath, h.httpClient,
-
+		ctx, &getEvent{
-	request := getEvent{
+			S:       s,
-		S:       s,
+			EventID: eventID,
-		EventID: eventID,
+		},
-	}
+	)
-	var response getEvent
-	apiURL := h.federationAPIURL + FederationAPIGetEventPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.Transaction{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.Transaction{}, response.Err
-	}
-	return *response.Res, nil
 }
 
 type getEventAuth struct {
@@ -412,135 +306,86 @@ type getEventAuth struct {
 	RoomVersion gomatrixserverlib.RoomVersion
 	RoomID      string
 	EventID     string
-	Res         *gomatrixserverlib.RespEventAuth
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) GetEventAuth(
 	ctx context.Context, s gomatrixserverlib.ServerName,
 	roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string,
 ) (gomatrixserverlib.RespEventAuth, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "GetEventAuth")
+	return httputil.CallInternalProxyAPI[getEventAuth, gomatrixserverlib.RespEventAuth, *api.FederationClientError](
-	defer span.Finish()
+		"GetEventAuth", h.federationAPIURL+FederationAPIGetEventAuthPath, h.httpClient,
-
+		ctx, &getEventAuth{
-	request := getEventAuth{
+			S:           s,
-		S:           s,
+			RoomVersion: roomVersion,
-		RoomVersion: roomVersion,
+			RoomID:      roomID,
-		RoomID:      roomID,
+			EventID:     eventID,
-		EventID:     eventID,
+		},
-	}
+	)
-	var response getEventAuth
-	apiURL := h.federationAPIURL + FederationAPIGetEventAuthPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return gomatrixserverlib.RespEventAuth{}, err
-	}
-	if response.Err != nil {
-		return gomatrixserverlib.RespEventAuth{}, response.Err
-	}
-	return *response.Res, nil
 }
 
 func (h *httpFederationInternalAPI) QueryServerKeys(
 	ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerKeys")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"QueryServerKeys", h.federationAPIURL+FederationAPIQueryServerKeysPath,
-
+		h.httpClient, ctx, req, res,
-	apiURL := h.federationAPIURL + FederationAPIQueryServerKeysPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
 
 type lookupServerKeys struct {
 	S           gomatrixserverlib.ServerName
 	KeyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp
-	ServerKeys  []gomatrixserverlib.ServerKeys
-	Err         *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) LookupServerKeys(
 	ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
 ) ([]gomatrixserverlib.ServerKeys, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "LookupServerKeys")
+	return httputil.CallInternalProxyAPI[lookupServerKeys, []gomatrixserverlib.ServerKeys, *api.FederationClientError](
-	defer span.Finish()
+		"LookupServerKeys", h.federationAPIURL+FederationAPILookupServerKeysPath, h.httpClient,
-
+		ctx, &lookupServerKeys{
-	request := lookupServerKeys{
+			S:           s,
-		S:           s,
+			KeyRequests: keyRequests,
-		KeyRequests: keyRequests,
+		},
-	}
+	)
-	var response lookupServerKeys
-	apiURL := h.federationAPIURL + FederationAPILookupServerKeysPath
-	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return []gomatrixserverlib.ServerKeys{}, err
-	}
-	if response.Err != nil {
-		return []gomatrixserverlib.ServerKeys{}, response.Err
-	}
-	return response.ServerKeys, nil
 }
 
 type eventRelationships struct {
 	S       gomatrixserverlib.ServerName
 	Req     gomatrixserverlib.MSC2836EventRelationshipsRequest
 	RoomVer gomatrixserverlib.RoomVersion
-	Res     gomatrixserverlib.MSC2836EventRelationshipsResponse
-	Err     *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) MSC2836EventRelationships(
 	ctx context.Context, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest,
 	roomVersion gomatrixserverlib.RoomVersion,
 ) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "MSC2836EventRelationships")
+	return httputil.CallInternalProxyAPI[eventRelationships, gomatrixserverlib.MSC2836EventRelationshipsResponse, *api.FederationClientError](
-	defer span.Finish()
+		"MSC2836EventRelationships", h.federationAPIURL+FederationAPIEventRelationshipsPath, h.httpClient,
-
+		ctx, &eventRelationships{
-	request := eventRelationships{
+			S:       s,
-		S:       s,
+			Req:     r,
-		Req:     r,
+			RoomVer: roomVersion,
-		RoomVer: roomVersion,
+		},
-	}
+	)
-	var response eventRelationships
-	apiURL := h.federationAPIURL + FederationAPIEventRelationshipsPath
-	err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return res, err
-	}
-	if response.Err != nil {
-		return res, response.Err
-	}
-	return response.Res, nil
 }
 
 type spacesReq struct {
 	S             gomatrixserverlib.ServerName
 	SuggestedOnly bool
 	RoomID        string
-	Res           gomatrixserverlib.MSC2946SpacesResponse
-	Err           *api.FederationClientError
 }
 
 func (h *httpFederationInternalAPI) MSC2946Spaces(
 	ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool,
 ) (res gomatrixserverlib.MSC2946SpacesResponse, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "MSC2946Spaces")
+	return httputil.CallInternalProxyAPI[spacesReq, gomatrixserverlib.MSC2946SpacesResponse, *api.FederationClientError](
-	defer span.Finish()
+		"MSC2836EventRelationships", h.federationAPIURL+FederationAPISpacesSummaryPath, h.httpClient,
-
+		ctx, &spacesReq{
-	request := spacesReq{
+			S:             dst,
-		S:             dst,
+			SuggestedOnly: suggestedOnly,
-		SuggestedOnly: suggestedOnly,
+			RoomID:        roomID,
-		RoomID:        roomID,
+		},
-	}
+	)
-	var response spacesReq
-	apiURL := h.federationAPIURL + FederationAPISpacesSummaryPath
-	err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
-	if err != nil {
-		return res, err
-	}
-	if response.Err != nil {
-		return res, response.Err
-	}
-	return response.Res, nil
 }
 
 func (s *httpFederationInternalAPI) KeyRing() *gomatrixserverlib.KeyRing {
@@ -614,11 +459,10 @@ func (h *httpFederationInternalAPI) InputPublicKeys(
 	request *api.InputPublicKeysRequest,
 	response *api.InputPublicKeysResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"InputPublicKey", h.federationAPIURL+FederationAPIInputPublicKeyPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIInputPublicKeyPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
 func (h *httpFederationInternalAPI) QueryPublicKeys(
@@ -626,9 +470,8 @@ func (h *httpFederationInternalAPI) QueryPublicKeys(
 	request *api.QueryPublicKeysRequest,
 	response *api.QueryPublicKeysResponse,
 ) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey")
+	return httputil.CallInternalRPCAPI(
-	defer span.Finish()
+		"QueryPublicKeys", h.federationAPIURL+FederationAPIQueryPublicKeyPath,
-
+		h.httpClient, ctx, request, response,
-	apiURL := h.federationAPIURL + FederationAPIQueryPublicKeyPath
+	)
-	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }

federationapi/inthttp/server.go

@@ -1,12 +1,14 @@
 package inthttp
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
@@ -15,372 +17,180 @@ import (
 func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIQueryJoinedHostServerNamesInRoomPath,
-		httputil.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", intAPI.QueryJoinedHostServerNamesInRoom),
-			var request api.QueryJoinedHostServerNamesInRoomRequest
-			var response api.QueryJoinedHostServerNamesInRoomResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.ErrorResponse(err)
-			}
-			if err := intAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(
-		FederationAPIPerformJoinRequestPath,
-		httputil.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse {
-			var request api.PerformJoinRequest
-			var response api.PerformJoinResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			intAPI.PerformJoin(req.Context(), &request, &response)
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-	internalAPIMux.Handle(
-		FederationAPIPerformLeaveRequestPath,
-		httputil.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse {
-			var request api.PerformLeaveRequest
-			var response api.PerformLeaveResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformLeave(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIPerformInviteRequestPath,
-		httputil.MakeInternalAPI("PerformInviteRequest", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", intAPI.PerformInvite),
-			var request api.PerformInviteRequest
-			var response api.PerformInviteResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformInvite(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
 	)
+
+	internalAPIMux.Handle(
+		FederationAPIPerformLeaveRequestPath,
+		httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", intAPI.PerformLeave),
+	)
+
 	internalAPIMux.Handle(
 		FederationAPIPerformDirectoryLookupRequestPath,
-		httputil.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", intAPI.PerformDirectoryLookup),
-			var request api.PerformDirectoryLookupRequest
-			var response api.PerformDirectoryLookupResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformDirectoryLookup(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIPerformBroadcastEDUPath,
-		httputil.MakeInternalAPI("PerformBroadcastEDU", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", intAPI.PerformBroadcastEDU),
-			var request api.PerformBroadcastEDURequest
-			var response api.PerformBroadcastEDUResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.PerformBroadcastEDU(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
 	)
+
+	internalAPIMux.Handle(
+		FederationAPIPerformJoinRequestPath,
+		httputil.MakeInternalRPCAPI(
+			"FederationAPIPerformJoinRequest",
+			func(ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse) error {
+				intAPI.PerformJoin(ctx, req, res)
+				return nil
+			},
+		),
+	)
+
 	internalAPIMux.Handle(
 		FederationAPIGetUserDevicesPath,
-		httputil.MakeInternalAPI("GetUserDevices", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request getUserDevices
+			"FederationAPIGetUserDevices",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *getUserDevices) (*gomatrixserverlib.RespUserDevices, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.GetUserDevices(ctx, req.S, req.UserID)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.GetUserDevices(req.Context(), request.S, request.UserID)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIClaimKeysPath,
-		httputil.MakeInternalAPI("ClaimKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request claimKeys
+			"FederationAPIClaimKeys",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *claimKeys) (*gomatrixserverlib.RespClaimKeys, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.ClaimKeys(ctx, req.S, req.OneTimeKeys)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.ClaimKeys(req.Context(), request.S, request.OneTimeKeys)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIQueryKeysPath,
-		httputil.MakeInternalAPI("QueryKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request queryKeys
+			"FederationAPIQueryKeys",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *queryKeys) (*gomatrixserverlib.RespQueryKeys, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.QueryKeys(ctx, req.S, req.Keys)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.QueryKeys(req.Context(), request.S, request.Keys)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIBackfillPath,
-		httputil.MakeInternalAPI("Backfill", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request backfill
+			"FederationAPIBackfill",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *backfill) (*gomatrixserverlib.Transaction, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.Backfill(ctx, req.S, req.RoomID, req.Limit, req.EventIDs)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.Backfill(req.Context(), request.S, request.RoomID, request.Limit, request.EventIDs)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupStatePath,
-		httputil.MakeInternalAPI("LookupState", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request lookupState
+			"FederationAPILookupState",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *lookupState) (*gomatrixserverlib.RespState, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.LookupState(ctx, req.S, req.RoomID, req.EventID, req.RoomVersion)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.LookupState(req.Context(), request.S, request.RoomID, request.EventID, request.RoomVersion)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupStateIDsPath,
-		httputil.MakeInternalAPI("LookupStateIDs", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request lookupStateIDs
+			"FederationAPILookupStateIDs",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *lookupStateIDs) (*gomatrixserverlib.RespStateIDs, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.LookupStateIDs(ctx, req.S, req.RoomID, req.EventID)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.LookupStateIDs(req.Context(), request.S, request.RoomID, request.EventID)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupMissingEventsPath,
-		httputil.MakeInternalAPI("LookupMissingEvents", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request lookupMissingEvents
+			"FederationAPILookupMissingEvents",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *lookupMissingEvents) (*gomatrixserverlib.RespMissingEvents, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.LookupMissingEvents(ctx, req.S, req.RoomID, req.Missing, req.RoomVersion)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.LookupMissingEvents(req.Context(), request.S, request.RoomID, request.Missing, request.RoomVersion)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			for _, event := range res.Events {
-				js, err := json.Marshal(event)
-				if err != nil {
-					return util.MessageResponse(http.StatusInternalServerError, err.Error())
-				}
-				request.Res.Events = append(request.Res.Events, js)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIGetEventPath,
-		httputil.MakeInternalAPI("GetEvent", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request getEvent
+			"FederationAPIGetEvent",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *getEvent) (*gomatrixserverlib.Transaction, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.GetEvent(ctx, req.S, req.EventID)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.GetEvent(req.Context(), request.S, request.EventID)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIGetEventAuthPath,
-		httputil.MakeInternalAPI("GetEventAuth", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request getEventAuth
+			"FederationAPIGetEventAuth",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *getEventAuth) (*gomatrixserverlib.RespEventAuth, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.GetEventAuth(ctx, req.S, req.RoomVersion, req.RoomID, req.EventID)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.GetEventAuth(req.Context(), request.S, request.RoomVersion, request.RoomID, request.EventID)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = &res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIQueryServerKeysPath,
-		httputil.MakeInternalAPI("QueryServerKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", intAPI.QueryServerKeys),
-			var request api.QueryServerKeysRequest
-			var response api.QueryServerKeysResponse
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.QueryServerKeys(req.Context(), &request, &response); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPILookupServerKeysPath,
-		httputil.MakeInternalAPI("LookupServerKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request lookupServerKeys
+			"FederationAPILookupServerKeys",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *lookupServerKeys) (*[]gomatrixserverlib.ServerKeys, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.LookupServerKeys(ctx, req.S, req.KeyRequests)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.LookupServerKeys(req.Context(), request.S, request.KeyRequests)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.ServerKeys = res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPIEventRelationshipsPath,
-		httputil.MakeInternalAPI("MSC2836EventRelationships", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request eventRelationships
+			"FederationAPIMSC2836EventRelationships",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *eventRelationships) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.MSC2836EventRelationships(ctx, req.S, req.Req, req.RoomVer)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.MSC2836EventRelationships(req.Context(), request.S, request.Req, request.RoomVer)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
 	internalAPIMux.Handle(
 		FederationAPISpacesSummaryPath,
-		httputil.MakeInternalAPI("MSC2946SpacesSummary", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalProxyAPI(
-			var request spacesReq
+			"FederationAPIMSC2946SpacesSummary",
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+			func(ctx context.Context, req *spacesReq) (*gomatrixserverlib.MSC2946SpacesResponse, error) {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
+				res, err := intAPI.MSC2946Spaces(ctx, req.S, req.RoomID, req.SuggestedOnly)
-			}
+				return &res, federationClientError(err)
-			res, err := intAPI.MSC2946Spaces(req.Context(), request.S, request.RoomID, request.SuggestedOnly)
+			},
-			if err != nil {
+		),
-				ferr, ok := err.(*api.FederationClientError)
-				if ok {
-					request.Err = ferr
-				} else {
-					request.Err = &api.FederationClientError{
-						Err: err.Error(),
-					}
-				}
-			}
-			request.Res = res
-			return util.JSONResponse{Code: http.StatusOK, JSON: request}
-		}),
 	)
+
+	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIQueryPublicKeyPath,
-		httputil.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", func(req *http.Request) util.JSONResponse {
 			request := api.QueryPublicKeysRequest{}
 			response := api.QueryPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -394,8 +204,10 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+
+	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIInputPublicKeyPath,
-		httputil.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIInputPublicKeys", func(req *http.Request) util.JSONResponse {
 			request := api.InputPublicKeysRequest{}
 			response := api.InputPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -408,3 +220,18 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 		}),
 	)
 }
+
+func federationClientError(err error) error {
+	switch ferr := err.(type) {
+	case nil:
+		return nil
+	case api.FederationClientError:
+		return &ferr
+	case *api.FederationClientError:
+		return ferr
+	default:
+		return &api.FederationClientError{
+			Err: err.Error(),
+		}
+	}
+}

federationapi/queue/destinationqueue.go

@@ -127,6 +127,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 		oq.destination, // the destination server name
 		receipt,        // NIDs from federationapi_queue_json table
 		event.Type,
+		nil, // this will use the default expireEDUTypes map
 	); err != nil {
 		logrus.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination)
 		return

federationapi/queue/queue.go

@@ -158,7 +158,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
 	oqs.queuesMutex.Lock()
 	defer oqs.queuesMutex.Unlock()
 	oq, ok := oqs.queues[destination]
-	if !ok || oq != nil {
+	if !ok || oq == nil {
 		destinationQueueTotal.Inc()
 		oq = &destinationQueue{
 			queues:           oqs,

federationapi/routing/devices.go

@@ -30,9 +30,11 @@ func GetUserDevices(
 	userID string,
 ) util.JSONResponse {
 	var res keyapi.QueryDeviceMessagesResponse
-	keyAPI.QueryDeviceMessages(req.Context(), &keyapi.QueryDeviceMessagesRequest{
+	if err := keyAPI.QueryDeviceMessages(req.Context(), &keyapi.QueryDeviceMessagesRequest{
 		UserID: userID,
-	}, &res)
+	}, &res); err != nil {
+		return util.ErrorResponse(err)
+	}
 	if res.Error != nil {
 		util.GetLogger(req.Context()).WithError(res.Error).Error("keyAPI.QueryDeviceMessages failed")
 		return jsonerror.InternalServerError()
@@ -47,7 +49,9 @@ func GetUserDevices(
 	for _, dev := range res.Devices {
 		sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID))
 	}
-	keyAPI.QuerySignatures(req.Context(), sigReq, sigRes)
+	if err := keyAPI.QuerySignatures(req.Context(), sigReq, sigRes); err != nil {
+		return jsonerror.InternalAPIError(req.Context(), err)
+	}
 
 	response := gomatrixserverlib.RespUserDevices{
 		UserID:   userID,

federationapi/routing/invite.go

@@ -141,10 +141,16 @@ func processInvite(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}

federationapi/routing/join.go

@@ -21,13 +21,14 @@ import (
 	"sort"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 // MakeJoin implements the /make_join API
@@ -202,6 +203,14 @@ func SendJoin(
 		}
 	}
 
+	// Check that the event is from the server sending the request.
+	if event.Origin() != request.Origin() {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The join must be sent by the server it originated on"),
+		}
+	}
+
 	// Check that a state key is provided.
 	if event.StateKey() == nil || event.StateKeyEquals("") {
 		return util.JSONResponse{
@@ -216,6 +225,22 @@ func SendJoin(
 		}
 	}
 
+	// Check that the sender belongs to the server that is sending us
+	// the request. By this point we've already asserted that the sender
+	// and the state key are equal so we don't need to check both.
+	var domain gomatrixserverlib.ServerName
+	if _, domain, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender of the join is invalid"),
+		}
+	} else if domain != request.Origin() {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender of the join must belong to the origin server"),
+		}
+	}
+
 	// Check that the room ID is correct.
 	if event.RoomID() != roomID {
 		return util.JSONResponse{
@@ -242,14 +267,6 @@ func SendJoin(
 		}
 	}
 
-	// Check that the event is from the server sending the request.
-	if event.Origin() != request.Origin() {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The join must be sent by the server it originated on"),
-		}
-	}
-
 	// Check that this is in fact a join event
 	membership, err := event.Membership()
 	if err != nil {
@@ -266,10 +283,17 @@ func SendJoin(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: join.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}
@@ -305,6 +329,12 @@ func SendJoin(
 			JSON: jsonerror.NotFound("Room does not exist"),
 		}
 	}
+	if !stateAndAuthChainResponse.StateKnown {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("State not known"),
+		}
+	}
 
 	// Check if the user is already in the room. If they're already in then
 	// there isn't much point in sending another join event into the room.
@@ -368,7 +398,7 @@ func SendJoin(
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
 	if !alreadyJoined {
 		var response api.InputRoomEventsResponse
-		rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
+		if err := rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
 			InputRoomEvents: []api.InputRoomEvent{
 				{
 					Kind:          api.KindNew,
@@ -377,7 +407,9 @@ func SendJoin(
 					TransactionID: nil,
 				},
 			},
-		}, &response)
+		}, &response); err != nil {
+			return jsonerror.InternalAPIError(httpReq.Context(), err)
+		}
 		if response.ErrMsg != "" {
 			util.GetLogger(httpReq.Context()).WithField(logrus.ErrorKey, response.ErrMsg).Error("SendEvents failed")
 			if response.NotAllowed {
@@ -412,13 +444,13 @@ func SendJoin(
 // a restricted room join. If the room version does not support restricted
 // joins then this function returns with no side effects. This returns three
 // values:
-// * an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
+//   - an optional JSON response body (i.e. M_UNABLE_TO_AUTHORISE_JOIN) which
-//   should always be sent back to the client if one is specified
+//     should always be sent back to the client if one is specified
-// * a user ID of an authorising user, typically a user that has power to
+//   - a user ID of an authorising user, typically a user that has power to
-//   issue invites in the room, if one has been found
+//     issue invites in the room, if one has been found
-// * an error if there was a problem finding out if this was allowable,
+//   - an error if there was a problem finding out if this was allowable,
-//   like if the room version isn't known or a problem happened talking to
+//     like if the room version isn't known or a problem happened talking to
-//   the roomserver
+//     the roomserver
 func checkRestrictedJoin(
 	httpReq *http.Request,
 	rsAPI api.FederationRoomserverAPI,

federationapi/routing/keys.go

@@ -19,7 +19,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/dendrite/clientapi/httputil"
+	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/keyserver/api"
@@ -61,9 +61,11 @@ func QueryDeviceKeys(
 	}
 
 	var queryRes api.QueryKeysResponse
-	keyAPI.QueryKeys(httpReq.Context(), &api.QueryKeysRequest{
+	if err := keyAPI.QueryKeys(httpReq.Context(), &api.QueryKeysRequest{
 		UserToDevices: qkr.DeviceKeys,
-	}, &queryRes)
+	}, &queryRes); err != nil {
+		return jsonerror.InternalAPIError(httpReq.Context(), err)
+	}
 	if queryRes.Error != nil {
 		util.GetLogger(httpReq.Context()).WithError(queryRes.Error).Error("Failed to QueryKeys")
 		return jsonerror.InternalServerError()
@@ -113,9 +115,11 @@ func ClaimOneTimeKeys(
 	}
 
 	var claimRes api.PerformClaimKeysResponse
-	keyAPI.PerformClaimKeys(httpReq.Context(), &api.PerformClaimKeysRequest{
+	if err := keyAPI.PerformClaimKeys(httpReq.Context(), &api.PerformClaimKeysRequest{
 		OneTimeKeys: cor.OneTimeKeys,
-	}, &claimRes)
+	}, &claimRes); err != nil {
+		return jsonerror.InternalAPIError(httpReq.Context(), err)
+	}
 	if claimRes.Error != nil {
 		util.GetLogger(httpReq.Context()).WithError(claimRes.Error).Error("Failed to PerformClaimKeys")
 		return jsonerror.InternalServerError()
@@ -184,7 +188,7 @@ func NotaryKeys(
 ) util.JSONResponse {
 	if req == nil {
 		req = &gomatrixserverlib.PublicKeyNotaryLookupRequest{}
-		if reqErr := httputil.UnmarshalJSONRequest(httpReq, &req); reqErr != nil {
+		if reqErr := clienthttputil.UnmarshalJSONRequest(httpReq, &req); reqErr != nil {
 			return *reqErr
 		}
 	}

federationapi/routing/leave.go

@@ -231,10 +231,17 @@ func SendLeave(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: leave.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}
@@ -270,7 +277,7 @@ func SendLeave(
 	// We are responsible for notifying other servers that the user has left
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
 	var response api.InputRoomEventsResponse
-	rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
+	if err := rsAPI.InputRoomEvents(httpReq.Context(), &api.InputRoomEventsRequest{
 		InputRoomEvents: []api.InputRoomEvent{
 			{
 				Kind:          api.KindNew,
@@ -279,7 +286,9 @@ func SendLeave(
 				TransactionID: nil,
 			},
 		},
-	}, &response)
+	}, &response); err != nil {
+		return jsonerror.InternalAPIError(httpReq.Context(), err)
+	}
 
 	if response.ErrMsg != "" {
 		util.GetLogger(httpReq.Context()).WithField(logrus.ErrorKey, response.ErrMsg).WithField("not_allowed", response.NotAllowed).Error("producer.SendEvents failed")

federationapi/routing/send.go

@@ -458,7 +458,9 @@ func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverli
 		UserID:           updatePayload.UserID,
 	}
 	uploadRes := &keyapi.PerformUploadDeviceKeysResponse{}
-	t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes)
+	if err := t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil {
+		return err
+	}
 	if uploadRes.Error != nil {
 		return uploadRes.Error
 	}

federationapi/routing/send_test.go

@@ -64,11 +64,12 @@ func (t *testRoomserverAPI) InputRoomEvents(
 	ctx context.Context,
 	request *api.InputRoomEventsRequest,
 	response *api.InputRoomEventsResponse,
-) {
+) error {
 	t.inputRoomEvents = append(t.inputRoomEvents, request.InputRoomEvents...)
 	for _, ire := range request.InputRoomEvents {
 		fmt.Println("InputRoomEvents: ", ire.Event.EventID())
 	}
+	return nil
 }
 
 // Query the latest events and state for a room from the room server.

federationapi/routing/state.go

@@ -135,6 +135,12 @@ func getState(
 		return nil, nil, &resErr
 	}
 
+	if !response.StateKnown {
+		return nil, nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("State not known"),
+		}
+	}
 	if response.IsRejected {
 		return nil, nil, &util.JSONResponse{
 			Code: http.StatusNotFound,

federationapi/statistics/statistics.go

@@ -5,10 +5,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 	"go.uber.org/atomic"
+
+	"github.com/matrix-org/dendrite/federationapi/storage"
 )
 
 // Statistics contains information about all of the remote federated
@@ -126,13 +127,13 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 
 		go func() {
 			until, ok := s.backoffUntil.Load().(time.Time)
-			if ok {
+			if ok && !until.IsZero() {
 				select {
 				case <-time.After(time.Until(until)):
 				case <-s.interrupt:
 				}
+				s.backoffStarted.Store(false)
 			}
-			s.backoffStarted.Store(false)
 		}()
 	}
 

federationapi/storage/interface.go

@@ -16,6 +16,7 @@ package storage
 
 import (
 	"context"
+	"time"
 
 	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	"github.com/matrix-org/dendrite/federationapi/types"
@@ -38,7 +39,7 @@ type Database interface {
 	GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error)
 
 	AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error
-	AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string) error
+	AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error
 
 	CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
 	CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
@@ -70,4 +71,6 @@ type Database interface {
 	// Query the notary for the server keys for the given server. If `optKeyIDs` is not empty, multiple server keys may be returned (between 1 - len(optKeyIDs))
 	// such that the combination of all server keys will include all the `optKeyIDs`.
 	GetNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, optKeyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error)
+	// DeleteExpiredEDUs cleans up expired EDUs
+	DeleteExpiredEDUs(ctx context.Context) error
 }

federationapi/storage/postgres/deltas/2021020411080000_rooms.go

@@ -15,23 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
+func UpRemoveRoomsTable(ctx context.Context, tx *sql.Tx) error {
-	goose.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
+	_, err := tx.ExecContext(ctx, `
-}
-
-func LoadRemoveRoomsTable(m *sqlutil.Migrations) {
-	m.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
-}
-
-func UpRemoveRoomsTable(tx *sql.Tx) error {
-	_, err := tx.Exec(`
 		DROP TABLE IF EXISTS federationsender_rooms;
 	`)
 	if err != nil {

federationapi/storage/postgres/deltas/2022042812473400_addexpiresat.go

@@ -0,0 +1,44 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func UpAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus ADD COLUMN IF NOT EXISTS expires_at BIGINT NOT NULL DEFAULT 0;")
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, "UPDATE federationsender_queue_edus SET expires_at = $1 WHERE edu_type != 'm.direct_to_device'", gomatrixserverlib.AsTimestamp(time.Now().Add(time.Hour*24)))
+	if err != nil {
+		return fmt.Errorf("failed to update queue_edus: %w", err)
+	}
+	return nil
+}
+
+func DownAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus DROP COLUMN expires_at;")
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

federationapi/storage/postgres/queue_edus_table.go

@@ -19,9 +19,11 @@ import (
 	"database/sql"
 
 	"github.com/lib/pq"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const queueEDUsSchema = `
@@ -31,7 +33,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
     -- The domain part of the user ID the EDU event is for.
 	server_name TEXT NOT NULL,
 	-- The JSON NID from the federationsender_queue_edus_json table.
-	json_nid BIGINT NOT NULL
+	json_nid BIGINT NOT NULL,
+	-- The expiry time of this edu, if any.
+	expires_at BIGINT NOT NULL DEFAULT 0
 );
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
@@ -43,8 +47,8 @@ CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
 `
 
 const insertQueueEDUSQL = "" +
-	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid)" +
+	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid, expires_at)" +
-	" VALUES ($1, $2, $3)"
+	" VALUES ($1, $2, $3, $4)"
 
 const deleteQueueEDUSQL = "" +
 	"DELETE FROM federationsender_queue_edus WHERE server_name = $1 AND json_nid = ANY($2)"
@@ -65,6 +69,12 @@ const selectQueueEDUCountSQL = "" +
 const selectQueueServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_edus"
 
+const selectExpiredEDUsSQL = "" +
+	"SELECT DISTINCT json_nid FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
+const deleteExpiredEDUsSQL = "" +
+	"DELETE FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
 type queueEDUsStatements struct {
 	db                                   *sql.DB
 	insertQueueEDUStmt                   *sql.Stmt
@@ -73,6 +83,8 @@ type queueEDUsStatements struct {
 	selectQueueEDUReferenceJSONCountStmt *sql.Stmt
 	selectQueueEDUCountStmt              *sql.Stmt
 	selectQueueEDUServerNamesStmt        *sql.Stmt
+	selectExpiredEDUsStmt                *sql.Stmt
+	deleteExpiredEDUsStmt                *sql.Stmt
 }
 
 func NewPostgresQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
@@ -81,27 +93,34 @@ func NewPostgresQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
 	}
 	_, err = s.db.Exec(queueEDUsSchema)
 	if err != nil {
-		return
+		return s, err
 	}
-	if s.insertQueueEDUStmt, err = s.db.Prepare(insertQueueEDUSQL); err != nil {
+
-		return
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "federationapi: add expiresat column",
+			Up:      deltas.UpAddexpiresat,
+		},
+	)
+	if err := m.Up(context.Background()); err != nil {
+		return s, err
 	}
-	if s.deleteQueueEDUStmt, err = s.db.Prepare(deleteQueueEDUSQL); err != nil {
+
-		return
+	return s, nil
-	}
+}
-	if s.selectQueueEDUStmt, err = s.db.Prepare(selectQueueEDUSQL); err != nil {
+
-		return
+func (s *queueEDUsStatements) Prepare() error {
-	}
+	return sqlutil.StatementList{
-	if s.selectQueueEDUReferenceJSONCountStmt, err = s.db.Prepare(selectQueueEDUReferenceJSONCountSQL); err != nil {
+		{&s.insertQueueEDUStmt, insertQueueEDUSQL},
-		return
+		{&s.deleteQueueEDUStmt, deleteQueueEDUSQL},
-	}
+		{&s.selectQueueEDUStmt, selectQueueEDUSQL},
-	if s.selectQueueEDUCountStmt, err = s.db.Prepare(selectQueueEDUCountSQL); err != nil {
+		{&s.selectQueueEDUReferenceJSONCountStmt, selectQueueEDUReferenceJSONCountSQL},
-		return
+		{&s.selectQueueEDUCountStmt, selectQueueEDUCountSQL},
-	}
+		{&s.selectQueueEDUServerNamesStmt, selectQueueServerNamesSQL},
-	if s.selectQueueEDUServerNamesStmt, err = s.db.Prepare(selectQueueServerNamesSQL); err != nil {
+		{&s.selectExpiredEDUsStmt, selectExpiredEDUsSQL},
-		return
+		{&s.deleteExpiredEDUsStmt, deleteExpiredEDUsSQL},
-	}
+	}.Prepare(s.db)
-	return
 }
 
 func (s *queueEDUsStatements) InsertQueueEDU(
@@ -110,6 +129,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 	eduType string,
 	serverName gomatrixserverlib.ServerName,
 	nid int64,
+	expiresAt gomatrixserverlib.Timestamp,
 ) error {
 	stmt := sqlutil.TxStmt(txn, s.insertQueueEDUStmt)
 	_, err := stmt.ExecContext(
@@ -117,6 +137,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 		eduType,    // the EDU type
 		serverName, // destination server name
 		nid,        // JSON blob NID
+		expiresAt,  // timestamp of expiry
 	)
 	return err
 }
@@ -150,7 +171,7 @@ func (s *queueEDUsStatements) SelectQueueEDUs(
 		}
 		result = append(result, nid)
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *queueEDUsStatements) SelectQueueEDUReferenceJSONCount(
@@ -200,3 +221,33 @@ func (s *queueEDUsStatements) SelectQueueEDUServerNames(
 
 	return result, rows.Err()
 }
+
+func (s *queueEDUsStatements) SelectExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectExpiredEDUsStmt)
+	rows, err := stmt.QueryContext(ctx, expiredBefore)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectExpiredEDUs: rows.close() failed")
+	var result []int64
+	var nid int64
+	for rows.Next() {
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+	return result, rows.Err()
+}
+
+func (s *queueEDUsStatements) DeleteExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteExpiredEDUsStmt)
+	_, err := stmt.ExecContext(ctx, expiredBefore)
+	return err
+}

federationapi/storage/postgres/storage.go

@@ -82,9 +82,16 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, err
 	}
-	m := sqlutil.NewMigrations()
+	m := sqlutil.NewMigrator(d.db)
-	deltas.LoadRemoveRoomsTable(m)
+	m.AddMigrations(sqlutil.Migration{
-	if err = m.RunDeltas(d.db, dbProperties); err != nil {
+		Version: "federationsender: drop federationsender_rooms",
+		Up:      deltas.UpRemoveRoomsTable,
+	})
+	err = m.Up(base.Context())
+	if err != nil {
+		return nil, err
+	}
+	if err = queueEDUs.Prepare(); err != nil {
 		return nil, err
 	}
 	d.Database = shared.Database{

federationapi/storage/shared/storage_edus.go

@@ -20,10 +20,21 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
+// defaultExpiry for EDUs if not listed below
+var defaultExpiry = time.Hour * 24
+
+// defaultExpireEDUTypes contains EDUs which can/should be expired after a given time
+// if the target server isn't reachable for some reason.
+var defaultExpireEDUTypes = map[string]time.Duration{
+	gomatrixserverlib.MTyping:   time.Minute,
+	gomatrixserverlib.MPresence: time.Minute * 10,
+}
+
 // AssociateEDUWithDestination creates an association that the
 // destination queues will use to determine which JSON blobs to send
 // to which servers.
@@ -32,7 +43,21 @@ func (d *Database) AssociateEDUWithDestination(
 	serverName gomatrixserverlib.ServerName,
 	receipt *Receipt,
 	eduType string,
+	expireEDUTypes map[string]time.Duration,
 ) error {
+	if expireEDUTypes == nil {
+		expireEDUTypes = defaultExpireEDUTypes
+	}
+	expiresAt := gomatrixserverlib.AsTimestamp(time.Now().Add(defaultExpiry))
+	if duration, ok := expireEDUTypes[eduType]; ok {
+		// Keep EDUs for at least x minutes before deleting them
+		expiresAt = gomatrixserverlib.AsTimestamp(time.Now().Add(duration))
+	}
+	// We forcibly set m.direct_to_device events to 0, as we always want them
+	// to be delivered. (required for E2EE)
+	if eduType == gomatrixserverlib.MDirectToDevice {
+		expiresAt = 0
+	}
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		if err := d.FederationQueueEDUs.InsertQueueEDU(
 			ctx,         // context
@@ -40,6 +65,7 @@ func (d *Database) AssociateEDUWithDestination(
 			eduType,     // EDU type for coalescing
 			serverName,  // destination server name
 			receipt.nid, // NID from the federationapi_queue_json table
+			expiresAt,   // The timestamp this EDU will expire
 		); err != nil {
 			return fmt.Errorf("InsertQueueEDU: %w", err)
 		}
@@ -84,6 +110,7 @@ func (d *Database) GetPendingEDUs(
 				return fmt.Errorf("json.Unmarshal: %w", err)
 			}
 			edus[&Receipt{nid}] = &event
+			d.Cache.StoreFederationQueuedEDU(nid, &event)
 		}
 
 		return nil
@@ -150,3 +177,34 @@ func (d *Database) GetPendingEDUServerNames(
 ) ([]gomatrixserverlib.ServerName, error) {
 	return d.FederationQueueEDUs.SelectQueueEDUServerNames(ctx, nil)
 }
+
+// DeleteExpiredEDUs deletes expired EDUs and evicts them from the cache.
+func (d *Database) DeleteExpiredEDUs(ctx context.Context) error {
+	var jsonNIDs []int64
+	err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) (err error) {
+		expiredBefore := gomatrixserverlib.AsTimestamp(time.Now())
+		jsonNIDs, err = d.FederationQueueEDUs.SelectExpiredEDUs(ctx, txn, expiredBefore)
+		if err != nil {
+			return err
+		}
+		if len(jsonNIDs) == 0 {
+			return nil
+		}
+
+		if err = d.FederationQueueJSON.DeleteQueueJSON(ctx, txn, jsonNIDs); err != nil {
+			return err
+		}
+
+		return d.FederationQueueEDUs.DeleteExpiredEDUs(ctx, txn, expiredBefore)
+	})
+
+	if err != nil {
+		return err
+	}
+
+	for i := range jsonNIDs {
+		d.Cache.EvictFederationQueuedEDU(jsonNIDs[i])
+	}
+
+	return nil
+}

federationapi/storage/sqlite3/deltas/2021020411080000_rooms.go

@@ -15,23 +15,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/pressly/goose"
 )
 
-func LoadFromGoose() {
+func UpRemoveRoomsTable(ctx context.Context, tx *sql.Tx) error {
-	goose.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
+	_, err := tx.ExecContext(ctx, `
-}
-
-func LoadRemoveRoomsTable(m *sqlutil.Migrations) {
-	m.AddMigration(UpRemoveRoomsTable, DownRemoveRoomsTable)
-}
-
-func UpRemoveRoomsTable(tx *sql.Tx) error {
-	_, err := tx.Exec(`
 		DROP TABLE IF EXISTS federationsender_rooms;
 	`)
 	if err != nil {

federationapi/storage/sqlite3/deltas/2022042812473400_addexpiresat.go

@@ -0,0 +1,68 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func UpAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus RENAME TO federationsender_queue_edus_old;")
+	if err != nil {
+		return fmt.Errorf("failed to rename table: %w", err)
+	}
+
+	_, err = tx.ExecContext(ctx, `
+CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
+	edu_type TEXT NOT NULL,
+	server_name TEXT NOT NULL,
+	json_nid BIGINT NOT NULL,
+	expires_at BIGINT NOT NULL DEFAULT 0
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
+    ON federationsender_queue_edus (json_nid, server_name);
+`)
+	if err != nil {
+		return fmt.Errorf("failed to create new table: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, `
+INSERT
+    INTO federationsender_queue_edus (
+        edu_type, server_name, json_nid, expires_at
+    )  SELECT edu_type, server_name, json_nid, 0 FROM federationsender_queue_edus_old;
+`)
+	if err != nil {
+		return fmt.Errorf("failed to update queue_edus: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, "UPDATE federationsender_queue_edus SET expires_at = $1 WHERE edu_type != 'm.direct_to_device'", gomatrixserverlib.AsTimestamp(time.Now().Add(time.Hour*24)))
+	if err != nil {
+		return fmt.Errorf("failed to update queue_edus: %w", err)
+	}
+	return nil
+}
+
+func DownAddexpiresat(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE federationsender_queue_edus DROP COLUMN expires_at;")
+	if err != nil {
+		return fmt.Errorf("failed to rename table: %w", err)
+	}
+	return nil
+}

federationapi/storage/sqlite3/queue_edus_table.go

@@ -20,9 +20,11 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const queueEDUsSchema = `
@@ -32,7 +34,9 @@ CREATE TABLE IF NOT EXISTS federationsender_queue_edus (
     -- The domain part of the user ID the EDU event is for.
 	server_name TEXT NOT NULL,
 	-- The JSON NID from the federationsender_queue_edus_json table.
-	json_nid BIGINT NOT NULL
+	json_nid BIGINT NOT NULL,
+	-- The expiry time of this edu, if any.
+	expires_at BIGINT NOT NULL DEFAULT 0
 );
 
 CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_edus_json_nid_idx
@@ -44,8 +48,8 @@ CREATE INDEX IF NOT EXISTS federationsender_queue_edus_server_name_idx
 `
 
 const insertQueueEDUSQL = "" +
-	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid)" +
+	"INSERT INTO federationsender_queue_edus (edu_type, server_name, json_nid, expires_at)" +
-	" VALUES ($1, $2, $3)"
+	" VALUES ($1, $2, $3, $4)"
 
 const deleteQueueEDUsSQL = "" +
 	"DELETE FROM federationsender_queue_edus WHERE server_name = $1 AND json_nid IN ($2)"
@@ -66,13 +70,22 @@ const selectQueueEDUCountSQL = "" +
 const selectQueueServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_edus"
 
+const selectExpiredEDUsSQL = "" +
+	"SELECT DISTINCT json_nid FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
+const deleteExpiredEDUsSQL = "" +
+	"DELETE FROM federationsender_queue_edus WHERE expires_at > 0 AND expires_at <= $1"
+
 type queueEDUsStatements struct {
-	db                                   *sql.DB
+	db                 *sql.DB
-	insertQueueEDUStmt                   *sql.Stmt
+	insertQueueEDUStmt *sql.Stmt
+	// deleteQueueEDUStmt                *sql.Stmt - prepared at runtime due to variadic
 	selectQueueEDUStmt                   *sql.Stmt
 	selectQueueEDUReferenceJSONCountStmt *sql.Stmt
 	selectQueueEDUCountStmt              *sql.Stmt
 	selectQueueEDUServerNamesStmt        *sql.Stmt
+	selectExpiredEDUsStmt                *sql.Stmt
+	deleteExpiredEDUsStmt                *sql.Stmt
 }
 
 func NewSQLiteQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
@@ -81,24 +94,33 @@ func NewSQLiteQueueEDUsTable(db *sql.DB) (s *queueEDUsStatements, err error) {
 	}
 	_, err = db.Exec(queueEDUsSchema)
 	if err != nil {
-		return
+		return s, err
 	}
-	if s.insertQueueEDUStmt, err = db.Prepare(insertQueueEDUSQL); err != nil {
+
-		return
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(
+		sqlutil.Migration{
+			Version: "federationapi: add expiresat column",
+			Up:      deltas.UpAddexpiresat,
+		},
+	)
+	if err := m.Up(context.Background()); err != nil {
+		return s, err
 	}
-	if s.selectQueueEDUStmt, err = db.Prepare(selectQueueEDUSQL); err != nil {
+
-		return
+	return s, nil
-	}
+}
-	if s.selectQueueEDUReferenceJSONCountStmt, err = db.Prepare(selectQueueEDUReferenceJSONCountSQL); err != nil {
+
-		return
+func (s *queueEDUsStatements) Prepare() error {
-	}
+	return sqlutil.StatementList{
-	if s.selectQueueEDUCountStmt, err = db.Prepare(selectQueueEDUCountSQL); err != nil {
+		{&s.insertQueueEDUStmt, insertQueueEDUSQL},
-		return
+		{&s.selectQueueEDUStmt, selectQueueEDUSQL},
-	}
+		{&s.selectQueueEDUReferenceJSONCountStmt, selectQueueEDUReferenceJSONCountSQL},
-	if s.selectQueueEDUServerNamesStmt, err = db.Prepare(selectQueueServerNamesSQL); err != nil {
+		{&s.selectQueueEDUCountStmt, selectQueueEDUCountSQL},
-		return
+		{&s.selectQueueEDUServerNamesStmt, selectQueueServerNamesSQL},
-	}
+		{&s.selectExpiredEDUsStmt, selectExpiredEDUsSQL},
-	return
+		{&s.deleteExpiredEDUsStmt, deleteExpiredEDUsSQL},
+	}.Prepare(s.db)
 }
 
 func (s *queueEDUsStatements) InsertQueueEDU(
@@ -107,6 +129,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 	eduType string,
 	serverName gomatrixserverlib.ServerName,
 	nid int64,
+	expiresAt gomatrixserverlib.Timestamp,
 ) error {
 	stmt := sqlutil.TxStmt(txn, s.insertQueueEDUStmt)
 	_, err := stmt.ExecContext(
@@ -114,6 +137,7 @@ func (s *queueEDUsStatements) InsertQueueEDU(
 		eduType,    // the EDU type
 		serverName, // destination server name
 		nid,        // JSON blob NID
+		expiresAt,  // timestamp of expiry
 	)
 	return err
 }
@@ -159,7 +183,7 @@ func (s *queueEDUsStatements) SelectQueueEDUs(
 		}
 		result = append(result, nid)
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *queueEDUsStatements) SelectQueueEDUReferenceJSONCount(
@@ -209,3 +233,33 @@ func (s *queueEDUsStatements) SelectQueueEDUServerNames(
 
 	return result, rows.Err()
 }
+
+func (s *queueEDUsStatements) SelectExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectExpiredEDUsStmt)
+	rows, err := stmt.QueryContext(ctx, expiredBefore)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectExpiredEDUs: rows.close() failed")
+	var result []int64
+	var nid int64
+	for rows.Next() {
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+	return result, rows.Err()
+}
+
+func (s *queueEDUsStatements) DeleteExpiredEDUs(
+	ctx context.Context, txn *sql.Tx,
+	expiredBefore gomatrixserverlib.Timestamp,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteExpiredEDUsStmt)
+	_, err := stmt.ExecContext(ctx, expiredBefore)
+	return err
+}

federationapi/storage/sqlite3/storage.go

@@ -81,9 +81,16 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, err
 	}
-	m := sqlutil.NewMigrations()
+	m := sqlutil.NewMigrator(d.db)
-	deltas.LoadRemoveRoomsTable(m)
+	m.AddMigrations(sqlutil.Migration{
-	if err = m.RunDeltas(d.db, dbProperties); err != nil {
+		Version: "federationsender: drop federationsender_rooms",
+		Up:      deltas.UpRemoveRoomsTable,
+	})
+	err = m.Up(base.Context())
+	if err != nil {
+		return nil, err
+	}
+	if err = queueEDUs.Prepare(); err != nil {
 		return nil, err
 	}
 	d.Database = shared.Database{