Merge branch 'main' of github.com:matrix-org/dendrite into landing-page

.github/codecov.yaml

@@ -0,0 +1,20 @@
+flag_management:
+  default_rules:
+    carryforward: true
+
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0%
+        base: auto
+        flags:
+          - unittests
+    patch:
+      default:
+        target: 75%
+        threshold: 0%
+        base: auto
+        flags:
+          - unittests

.github/workflows/dendrite.yml

@@ -68,7 +68,7 @@ jobs:
 
   # run go test with different go versions
   test:
-    timeout-minutes: 5
+    timeout-minutes: 10
     name: Unit tests (Go ${{ matrix.go }})
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
@@ -94,14 +94,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18", "1.19"]
+        go: ["1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
         uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
-          cache: true
+      - uses: actions/cache@v3
+        # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-unit-
       - name: Set up gotestfmt
         uses: gotesttools/gotestfmt-action@v2
         with:
@@ -194,6 +202,66 @@ jobs:
         with:
           jobs: ${{ toJSON(needs) }}
 
+  # run go test with different go versions
+  integration:
+    timeout-minutes: 20
+    needs: initial-tests-done
+    name: Integration tests (Go ${{ matrix.go }})
+    runs-on: ubuntu-latest
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:13-alpine
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.19"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+        with:
+          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ matrix.go }}-test-race-
+      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: dendrite
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests
+
   # run database upgrade tests
   upgrade_test:
     name: Upgrade tests
@@ -263,7 +331,7 @@ jobs:
             postgres: postgres
             api: full-http
     container:
-      image: matrixdotorg/sytest-dendrite:latest
+      image: matrixdotorg/sytest-dendrite
       volumes:
         - ${{ github.workspace }}:/src
         - /root/.cache/go-build:/github/home/.cache/go-build
@@ -404,6 +472,7 @@ jobs:
         upgrade_test_direct,
         sytest,
         complement,
+        integration
       ]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped

.github/workflows/docker.yml

@@ -68,18 +68,6 @@ jobs:
             ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release monolith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_monolith_release
@@ -98,6 +86,18 @@ jobs:
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+
   polylith:
     name: Polylith image
     runs-on: ubuntu-latest
@@ -148,18 +148,6 @@ jobs:
             ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
 
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
       - name: Build release polylith image
         if: github.event_name == 'release' # Only for GitHub releases
         id: docker_build_polylith_release
@@ -178,6 +166,18 @@ jobs:
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
             ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
 
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
+          format: "sarif"
+          output: "trivy-results.sarif"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest

.github/workflows/gh-pages.yml

@@ -0,0 +1,52 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Deploy GitHub Pages dependencies preinstalled
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["main"]
+    paths:
+      - 'docs/**' # only execute if we have docs changes
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow one concurrent deployment
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Pages
+        uses: actions/configure-pages@v2
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./docs
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v1
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v1

.github/workflows/helm.yml

@@ -0,0 +1,39 @@
+name: Release Charts
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'helm/**' # only execute if we have helm chart changes
+
+jobs:
+  release:
+    # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions
+    # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.10.0
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.4.1
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        with:
+          config: helm/cr.yaml
+          charts_dir: helm/

.github/workflows/k8s.yml

@@ -0,0 +1,90 @@
+name: k8s
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - 'helm/**' # only execute if we have helm chart changes
+  pull_request:
+    branches: ["main"]
+    paths:
+      - 'helm/**'
+
+jobs:
+  lint:
+    name: Lint Helm chart
+    runs-on: ubuntu-latest
+    outputs:
+      changed: ${{ steps.list-changed.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: azure/setup-helm@v3
+        with:
+          version: v3.10.0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          check-latest: true
+      - uses: helm/chart-testing-action@v2.3.1
+      - name: Get changed status
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config helm/ct.yaml --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+              echo "::set-output name=changed::true"
+          fi
+
+      - name: Run lint
+        run: ct lint --config helm/ct.yaml
+
+  # only bother to run if lint step reports a change to the helm chart
+  install:
+    needs:
+      - lint
+    if: ${{ needs.lint.outputs.changed == 'true' }}
+    name: Install Helm charts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.checkoutCommit }}
+      - name: Install Kubernetes tools
+        uses: yokawasa/action-setup-kube-tools@v0.8.2
+        with:
+          setup-tools: |
+            helmv3
+          helm: "3.10.3"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.3.1
+      - name: Create k3d cluster
+        uses: nolar/setup-k3d-k3s@v1
+        with:
+          version: v1.21
+      - name: Remove node taints
+        run: |
+          kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
+      - name: Run chart-testing (install)
+        run: ct install --config helm/ct.yaml
+
+      # Install the chart using helm directly and test with create-account
+      - name: Install chart
+        run: |
+          helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite
+      - name: Wait for Postgres and Dendrite to be up
+        run: |
+          kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A
+          kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A
+          kubectl get pods -A
+          kubectl get services
+          kubectl get ingress
+      - name: Run create account
+        run: |
+          podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)
+          kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword

.github/workflows/schedules.yaml

@@ -10,79 +10,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # run go test with different go versions
-  test:
-    timeout-minutes: 20
-    name: Unit tests (Go ${{ matrix.go }})
-    runs-on: ubuntu-latest
-    # Service containers to run with `container-job`
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres:13-alpine
-        # Provide the password for postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dendrite
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    strategy:
-      fail-fast: false
-      matrix:
-        go: ["1.18", "1.19"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go }}
-      - name: Set up gotestfmt
-        uses: gotesttools/gotestfmt-action@v2
-        with:
-          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go${{ matrix.go }}-test-race-
-      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
-        env:
-          POSTGRES_HOST: localhost
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: dendrite
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-
-  # Dummy step to gate other tests on without repeating the whole list
-  initial-tests-done:
-    name: Initial tests passed
-    needs: [test]
-    runs-on: ubuntu-latest
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    steps:
-      - name: Check initial tests passed
-        uses: re-actors/alls-green@release/v1
-        with:
-          jobs: ${{ toJSON(needs) }}
-
   # run Sytest in different variations
   sytest:
     timeout-minutes: 60
-    needs: initial-tests-done
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
     strategy:
@@ -104,13 +34,23 @@ jobs:
       image: matrixdotorg/sytest-dendrite:latest
       volumes:
         - ${{ github.workspace }}:/src
+        - /root/.cache/go-build:/github/home/.cache/go-build
+        - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
         API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         RACE_DETECTION: 1
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            /gopath/pkg/mod
+          key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-sytest-
       - name: Run Sytest
         run: /bootstrap.sh dendrite
         working-directory: /src
@@ -133,3 +73,41 @@ jobs:
           path: |
             /logs/results.tap
             /logs/**/*.log*
+          
+  element_web:
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tecolicom/actions-use-apt-tools@v1
+        with:
+          # Our test suite includes some screenshot tests with unusual diacritics, which are
+          # supposed to be covered by STIXGeneral.
+          tools: fonts-stix
+      - uses: actions/checkout@v2
+        with:
+          repository: matrix-org/matrix-react-sdk
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+      - name: Fetch layered build
+        run: scripts/ci/layered.sh
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+      - name: Build
+        env:
+          CI_PACKAGE: true
+        run: yarn build
+        working-directory: ./element-web
+      - name: Edit Test Config
+        run: |
+          sed -i '/HOMESERVER/c\        HOMESERVER: "dendrite",' cypress.config.ts
+      - name: "Run cypress tests"
+        uses: cypress-io/github-action@v4.1.1
+        with:
+          browser: chrome
+          start: npx serve -p 8080 ./element-web/webapp
+          wait-on: 'http://localhost:8080'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          TMPDIR: ${{ runner.temp }}

.gitignore

@@ -56,6 +56,7 @@ dendrite.yaml
 
 # Database files
 *.db
+*.db-journal
 
 # Log files
 *.log*

CHANGES.md

@@ -1,5 +1,62 @@
 # Changelog
 
+## Dendrite 0.11.0 (2023-01-20)
+
+The last three missing federation API Sytests have been fixed - bringing us to 100% server-server Synapse parity, with client-server parity at 93% 🎉
+
+### Features
+
+* Added `/_dendrite/admin/purgeRoom/{roomID}` to clean up the database
+* The default room version was updated to 10 (contributed by [FSG-Cat](https://github.com/FSG-Cat))
+
+### Fixes
+
+* An oversight in the `create-config` binary, which now correctly sets the media path if specified (contributed by [BieHDC](https://github.com/BieHDC))
+* The Helm chart now uses the `$.Chart.AppVersion` as the default image version to pull, with the possibility to override it (contributed by [genofire](https://github.com/genofire))
+
+## Dendrite 0.10.9 (2023-01-17)
+
+### Features
+
+* Stale device lists are now cleaned up on startup, removing entries for users the server doesn't share a room with anymore
+* Dendrite now has its own Helm chart
+* Guest access is now handled correctly (disallow joins, kick guests on revocation of guest access, as well as over federation)
+
+### Fixes
+
+* Push rules have seen several tweaks and fixes, which should, for example, fix notifications for `m.read_receipts`
+* Outgoing presence will now correctly be sent to newly joined hosts
+* Fixes the `/_dendrite/admin/resetPassword/{userID}` admin endpoint to use the correct variable
+* Federated backfilling for medium/large rooms has been fixed
+* `/login` causing wrong device list updates has been resolved
+* `/sync` should now return the correct room summary heroes
+* The default config options for `recaptcha_sitekey_class` and `recaptcha_form_field` are now set correctly 
+* `/messages` now omits empty `state` to be more spec compliant (contributed by [handlerug](https://github.com/handlerug))
+* `/sync` has been optimised to only query state events for history visibility if they are really needed
+
+## Dendrite 0.10.8 (2022-11-29)
+
+### Features
+
+* The built-in NATS Server has been updated to version 2.9.8
+* A number of under-the-hood changes have been merged for future virtual hosting support in Dendrite (running multiple domain names on the same Dendrite deployment)
+
+### Fixes
+
+* Event auth handling of invites has been refactored, which should fix some edge cases being handled incorrectly
+* Fix a bug when returning an empty protocol list, which could cause Element to display "The homeserver may be too old to support third party networks" when opening the public room directory
+* The sync API will no longer filter out the user's own membership when using lazy-loading
+* Dendrite will now correctly detect JetStream consumers being deleted, stopping the consumer goroutine as needed
+* A panic in the federation API where the server list could go out of bounds has been fixed
+* Blacklisted servers will now be excluded when querying joined servers, which improves CPU usage and performs less unnecessary outbound requests
+* A database writer will now be used to assign state key NIDs when requesting NIDs that may not exist yet
+* Dendrite will now correctly move local aliases for an upgraded room when the room is upgraded remotely
+* Dendrite will now correctly move account data for an upgraded room when the room is upgraded remotely
+* Missing state key NIDs will now be allocated on request rather than returning an error
+* Guest access is now correctly denied on a number of endpoints
+* Presence information will now be correctly sent for new private chats
+* A number of unspecced fields have been removed from outbound `/send` transactions
+
 ## Dendrite 0.10.7 (2022-11-04)
 
 ### Features

Dockerfile

@@ -27,6 +27,7 @@ RUN --mount=target=. \
 # The dendrite base image
 #
 FROM alpine:latest AS dendrite-base
+RUN apk --update --no-cache add curl
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
@@ -63,30 +64,3 @@ WORKDIR /etc/dendrite
 ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
 EXPOSE 8008 8448
 
-#
-# Builds the Complement image, used for integration tests
-#
-FROM base AS complement
-LABEL org.opencontainers.image.title="Dendrite (Complement)"
-RUN apk add --no-cache sqlite openssl ca-certificates
-
-COPY --from=build /out/generate-config /usr/bin/generate-config
-COPY --from=build /out/generate-keys /usr/bin/generate-keys
-COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
-
-WORKDIR /dendrite
-RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
-    mkdir /ca && \
-    openssl genrsa -out /ca/ca.key 2048 && \
-    openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
-
-ENV SERVER_NAME=localhost
-ENV API=0
-EXPOSE 8008 8448
-
-# At runtime, generate TLS cert based on the CA now mounted at /ca
-# At runtime, replace the SERVER_NAME with what we are told
-CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
-    /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
-    cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

appservice/appservice.go

@@ -24,6 +24,8 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
 	"github.com/matrix-org/dendrite/appservice/inthttp"
@@ -32,12 +34,11 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddInternalRoutes registers HTTP handlers for internal API calls
-func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
-	inthttp.AddRoutes(queryAPI, router)
+func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI, enableMetrics bool) {
+	inthttp.AddRoutes(queryAPI, router, enableMetrics)
 }
 
 // NewInternalAPI returns a concerete implementation of the internal API. Callers

appservice/appservice_test.go

@@ -0,0 +1,223 @@
+package appservice_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/gorilla/mux"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/appservice/inthttp"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/userapi"
+
+	"github.com/matrix-org/dendrite/test/testrig"
+)
+
+func TestAppserviceInternalAPI(t *testing.T) {
+
+	// Set expected results
+	existingProtocol := "irc"
+	wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
+	wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
+	wantProtocolResult := map[string]api.ASProtocolResponse{
+		existingProtocol: wantProtocolResponse,
+	}
+
+	// create a dummy AS url, handling some cases
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.Contains(r.URL.Path, "location"):
+			// Check if we've got an existing protocol, if so, return a proper response.
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "user"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		case strings.Contains(r.URL.Path, "protocol"):
+			if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
+				if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
+					t.Fatalf("failed to encode response: %s", err)
+				}
+				return
+			}
+			if err := json.NewEncoder(w).Encode(nil); err != nil {
+				t.Fatalf("failed to encode response: %s", err)
+			}
+			return
+		default:
+			t.Logf("hit location: %s", r.URL.Path)
+		}
+	}))
+
+	// The test cases to run
+	runCases := func(t *testing.T, testAPI api.AppServiceInternalAPI) {
+		t.Run("UserIDExists", func(t *testing.T) {
+			testUserIDExists(t, testAPI, "@as-testing:test", true)
+			testUserIDExists(t, testAPI, "@as1-testing:test", false)
+		})
+
+		t.Run("AliasExists", func(t *testing.T) {
+			testAliasExists(t, testAPI, "@asroom-testing:test", true)
+			testAliasExists(t, testAPI, "@asroom1-testing:test", false)
+		})
+
+		t.Run("Locations", func(t *testing.T) {
+			testLocations(t, testAPI, existingProtocol, wantLocationResponse)
+			testLocations(t, testAPI, "abc", nil)
+		})
+
+		t.Run("User", func(t *testing.T) {
+			testUser(t, testAPI, existingProtocol, wantUserResponse)
+			testUser(t, testAPI, "abc", nil)
+		})
+
+		t.Run("Protocols", func(t *testing.T) {
+			testProtocol(t, testAPI, existingProtocol, wantProtocolResult)
+			testProtocol(t, testAPI, existingProtocol, wantProtocolResult) // tests the cache
+			testProtocol(t, testAPI, "", wantProtocolResult)               // tests getting all protocols
+			testProtocol(t, testAPI, "abc", nil)
+		})
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, closeBase := testrig.CreateBaseDendrite(t, dbType)
+		defer closeBase()
+
+		// Create a dummy application service
+		base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
+			{
+				ID:              "someID",
+				URL:             srv.URL,
+				ASToken:         "",
+				HSToken:         "",
+				SenderLocalpart: "senderLocalPart",
+				NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+					"users":   {{RegexpObject: regexp.MustCompile("as-.*")}},
+					"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
+				},
+				Protocols: []string{existingProtocol},
+			},
+		}
+
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(base)
+		usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
+
+		// Finally execute the tests
+		t.Run("HTTP API", func(t *testing.T) {
+			router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
+			appservice.AddInternalRoutes(router, asAPI, base.EnableMetrics)
+			apiURL, cancel := test.ListenAndServe(t, router, false)
+			defer cancel()
+
+			asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{})
+			if err != nil {
+				t.Fatalf("failed to create HTTP client: %s", err)
+			}
+			runCases(t, asHTTPApi)
+		})
+
+		t.Run("Monolith", func(t *testing.T) {
+			runCases(t, asAPI)
+		})
+	})
+}
+
+func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
+	ctx := context.Background()
+	userResp := &api.UserIDExistsResponse{}
+
+	if err := asAPI.UserIDExists(ctx, &api.UserIDExistsRequest{
+		UserID: userID,
+	}, userResp); err != nil {
+		t.Errorf("failed to get userID: %s", err)
+	}
+	if userResp.UserIDExists != wantExists {
+		t.Errorf("unexpected result for UserIDExists(%s): %v, expected %v", userID, userResp.UserIDExists, wantExists)
+	}
+}
+
+func testAliasExists(t *testing.T, asAPI api.AppServiceInternalAPI, alias string, wantExists bool) {
+	ctx := context.Background()
+	aliasResp := &api.RoomAliasExistsResponse{}
+
+	if err := asAPI.RoomAliasExists(ctx, &api.RoomAliasExistsRequest{
+		Alias: alias,
+	}, aliasResp); err != nil {
+		t.Errorf("failed to get alias: %s", err)
+	}
+	if aliasResp.AliasExists != wantExists {
+		t.Errorf("unexpected result for RoomAliasExists(%s): %v, expected %v", alias, aliasResp.AliasExists, wantExists)
+	}
+}
+
+func testLocations(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASLocationResponse) {
+	ctx := context.Background()
+	locationResp := &api.LocationResponse{}
+
+	if err := asAPI.Locations(ctx, &api.LocationRequest{
+		Protocol: proto,
+	}, locationResp); err != nil {
+		t.Errorf("failed to get locations: %s", err)
+	}
+	if !reflect.DeepEqual(locationResp.Locations, wantResult) {
+		t.Errorf("unexpected result for Locations(%s): %+v, expected %+v", proto, locationResp.Locations, wantResult)
+	}
+}
+
+func testUser(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASUserResponse) {
+	ctx := context.Background()
+	userResp := &api.UserResponse{}
+
+	if err := asAPI.User(ctx, &api.UserRequest{
+		Protocol: proto,
+	}, userResp); err != nil {
+		t.Errorf("failed to get user: %s", err)
+	}
+	if !reflect.DeepEqual(userResp.Users, wantResult) {
+		t.Errorf("unexpected result for User(%s): %+v, expected %+v", proto, userResp.Users, wantResult)
+	}
+}
+
+func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult map[string]api.ASProtocolResponse) {
+	ctx := context.Background()
+	protoResp := &api.ProtocolResponse{}
+
+	if err := asAPI.Protocols(ctx, &api.ProtocolRequest{
+		Protocol: proto,
+	}, protoResp); err != nil {
+		t.Errorf("failed to get Protocols: %s", err)
+	}
+	if !reflect.DeepEqual(protoResp.Protocols, wantResult) {
+		t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
+	}
+}

appservice/inthttp/server.go

@@ -8,29 +8,29 @@ import (
 )
 
 // AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
-func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
+func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
 	internalAPIMux.Handle(
 		AppServiceRoomAliasExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
+		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", enableMetrics, a.RoomAliasExists),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceUserIDExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
+		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", enableMetrics, a.UserIDExists),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceProtocolsPath,
-		httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
+		httputil.MakeInternalRPCAPI("AppserviceProtocols", enableMetrics, a.Protocols),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceLocationsPath,
-		httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
+		httputil.MakeInternalRPCAPI("AppserviceLocations", enableMetrics, a.Locations),
 	)
 
 	internalAPIMux.Handle(
 		AppServiceUserPath,
-		httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
+		httputil.MakeInternalRPCAPI("AppserviceUser", enableMetrics, a.User),
 	)
 }

are-we-synapse-yet.list

@@ -936,4 +936,12 @@ fst Room state after a rejected message event is the same as before
 fst Room state after a rejected state event is the same as before
 fpb Federation publicRoom Name/topic keys are correct
 fed New federated private chats get full presence information (SYN-115) (10 subtests)
-dvk Rejects invalid device keys
+dvk Rejects invalid device keys
+rmv User can create and send/receive messages in a room with version 10
+rmv local user can join room with version 10
+rmv User can invite local user to room with version 10
+rmv remote user can join room with version 10
+rmv User can invite remote user to room with version 10
+rmv Remote user can backfill in a room with version 10
+rmv Can reject invites over federation for rooms with version 10
+rmv Can receive redactions from regular users over federation in room version 10

build/dendritejs-pinecone/main.go

@@ -180,14 +180,14 @@ func startup() {
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck
 
+	rsAPI := roomserver.NewInternalAPI(base)
+
 	federation := conn.CreateFederationClient(base, pSessions)
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	rsAPI := roomserver.NewInternalAPI(base)
-
 	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 

build/docker/Dockerfile.demo-pinecone

@@ -17,6 +17,7 @@ RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
+RUN apk --update --no-cache add curl
 LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"

build/gobind-pinecone/monolith.go

@@ -41,13 +41,16 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/producers"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/relayapi"
+	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
@@ -67,24 +70,27 @@ import (
 )
 
 const (
-	PeerTypeRemote    = pineconeRouter.PeerTypeRemote
-	PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
-	PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
-	PeerTypeBonjour   = pineconeRouter.PeerTypeBonjour
+	PeerTypeRemote           = pineconeRouter.PeerTypeRemote
+	PeerTypeMulticast        = pineconeRouter.PeerTypeMulticast
+	PeerTypeBluetooth        = pineconeRouter.PeerTypeBluetooth
+	PeerTypeBonjour          = pineconeRouter.PeerTypeBonjour
+	relayServerRetryInterval = time.Second * 30
 )
 
 type DendriteMonolith struct {
-	logger            logrus.Logger
-	PineconeRouter    *pineconeRouter.Router
-	PineconeMulticast *pineconeMulticast.Multicast
-	PineconeQUIC      *pineconeSessions.Sessions
-	PineconeManager   *pineconeConnections.ConnectionManager
-	StorageDirectory  string
-	CacheDirectory    string
-	listener          net.Listener
-	httpServer        *http.Server
-	processContext    *process.ProcessContext
-	userAPI           userapiAPI.UserInternalAPI
+	logger              logrus.Logger
+	baseDendrite        *base.BaseDendrite
+	PineconeRouter      *pineconeRouter.Router
+	PineconeMulticast   *pineconeMulticast.Multicast
+	PineconeQUIC        *pineconeSessions.Sessions
+	PineconeManager     *pineconeConnections.ConnectionManager
+	StorageDirectory    string
+	CacheDirectory      string
+	listener            net.Listener
+	httpServer          *http.Server
+	userAPI             userapiAPI.UserInternalAPI
+	federationAPI       api.FederationInternalAPI
+	relayServersQueried map[gomatrixserverlib.ServerName]bool
 }
 
 func (m *DendriteMonolith) PublicKey() string {
@@ -326,6 +332,7 @@ func (m *DendriteMonolith) Start() {
 	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix)))
 	cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
 	cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
+	cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(m.StorageDirectory, prefix)))
 	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
 	cfg.ClientAPI.RegistrationDisabled = false
 	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
@@ -335,8 +342,9 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	base := base.NewBaseDendrite(cfg, "Monolith")
-	defer base.Close() // nolint: errcheck
+	base := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
+	m.baseDendrite = base
+	base.ConfigureAdminEndpoints()
 
 	federation := conn.CreateFederationClient(base, m.PineconeQUIC)
 
@@ -345,11 +353,11 @@ func (m *DendriteMonolith) Start() {
 
 	rsAPI := roomserver.NewInternalAPI(base)
 
-	fsAPI := federationapi.NewInternalAPI(
+	m.federationAPI = federationapi.NewInternalAPI(
 		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, m.federationAPI, rsAPI)
 	m.userAPI = userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(m.userAPI)
 
@@ -357,10 +365,24 @@ func (m *DendriteMonolith) Start() {
 
 	// The underlying roomserver implementation needs to be able to call the fedsender.
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
-	rsAPI.SetFederationAPI(fsAPI, keyRing)
+	rsAPI.SetFederationAPI(m.federationAPI, keyRing)
 
 	userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation)
-	roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation)
+	roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, m.federationAPI, federation)
+
+	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	producer := &producers.SyncAPIProducer{
+		JetStream:              js,
+		TopicReceiptEvent:      base.Cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicSendToDeviceEvent: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicTypingEvent:       base.Cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     base.Cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicDeviceListUpdate:  base.Cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
+		TopicSigningKeyUpdate:  base.Cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
+		Config:                 &base.Cfg.FederationAPI,
+		UserAPI:                m.userAPI,
+	}
+	relayAPI := relayapi.NewRelayInternalAPI(base, federation, rsAPI, keyRing, producer)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
@@ -369,10 +391,11 @@ func (m *DendriteMonolith) Start() {
 		KeyRing:   keyRing,
 
 		AppserviceAPI:            asAPI,
-		FederationAPI:            fsAPI,
+		FederationAPI:            m.federationAPI,
 		RoomserverAPI:            rsAPI,
 		UserAPI:                  m.userAPI,
 		KeyAPI:                   keyAPI,
+		RelayAPI:                 relayAPI,
 		ExtPublicRoomsProvider:   roomProvider,
 		ExtUserDirectoryProvider: userProvider,
 	}
@@ -382,6 +405,8 @@ func (m *DendriteMonolith) Start() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 	httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
 
 	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
@@ -408,8 +433,6 @@ func (m *DendriteMonolith) Start() {
 		Handler: h2c.NewHandler(pMux, h2s),
 	}
 
-	m.processContext = base.ProcessContext
-
 	go func() {
 		m.logger.Info("Listening on ", cfg.Global.ServerName)
 
@@ -417,7 +440,7 @@ func (m *DendriteMonolith) Start() {
 		case net.ErrClosed, http.ErrServerClosed:
 			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
 		default:
-			m.logger.Fatal(err)
+			m.logger.Error("Stopped listening on ", cfg.Global.ServerName)
 		}
 	}()
 	go func() {
@@ -427,33 +450,44 @@ func (m *DendriteMonolith) Start() {
 		case net.ErrClosed, http.ErrServerClosed:
 			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
 		default:
-			m.logger.Fatal(err)
+			m.logger.Error("Stopped listening on ", cfg.Global.ServerName)
 		}
 	}()
 
 	go func(ch <-chan pineconeEvents.Event) {
 		eLog := logrus.WithField("pinecone", "events")
+		stopRelayServerSync := make(chan bool)
+
+		relayRetriever := RelayServerRetriever{
+			Context:             context.Background(),
+			ServerName:          gomatrixserverlib.ServerName(m.PineconeRouter.PublicKey().String()),
+			FederationAPI:       m.federationAPI,
+			relayServersQueried: make(map[gomatrixserverlib.ServerName]bool),
+			RelayAPI:            monolith.RelayAPI,
+			running:             *atomic.NewBool(false),
+		}
+		relayRetriever.InitializeRelayServers(eLog)
 
 		for event := range ch {
 			switch e := event.(type) {
 			case pineconeEvents.PeerAdded:
+				if !relayRetriever.running.Load() {
+					go relayRetriever.SyncRelayServers(stopRelayServerSync)
+				}
 			case pineconeEvents.PeerRemoved:
-			case pineconeEvents.TreeParentUpdate:
-			case pineconeEvents.SnakeDescUpdate:
-			case pineconeEvents.TreeRootAnnUpdate:
-			case pineconeEvents.SnakeEntryAdded:
-			case pineconeEvents.SnakeEntryRemoved:
+				if relayRetriever.running.Load() && m.PineconeRouter.TotalPeerCount() == 0 {
+					stopRelayServerSync <- true
+				}
 			case pineconeEvents.BroadcastReceived:
-				eLog.Info("Broadcast received from: ", e.PeerID)
+				// eLog.Info("Broadcast received from: ", e.PeerID)
 
 				req := &api.PerformWakeupServersRequest{
 					ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
 				}
 				res := &api.PerformWakeupServersResponse{}
-				if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
-					logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
+				if err := m.federationAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
+					eLog.WithError(err).Error("Failed to wakeup destination", e.PeerID)
 				}
-			case pineconeEvents.BandwidthReport:
 			default:
 			}
 		}
@@ -461,12 +495,106 @@ func (m *DendriteMonolith) Start() {
 }
 
 func (m *DendriteMonolith) Stop() {
-	m.processContext.ShutdownDendrite()
+	m.baseDendrite.Close()
+	m.baseDendrite.WaitForShutdown()
 	_ = m.listener.Close()
 	m.PineconeMulticast.Stop()
 	_ = m.PineconeQUIC.Close()
 	_ = m.PineconeRouter.Close()
-	m.processContext.WaitForComponentsToFinish()
+}
+
+type RelayServerRetriever struct {
+	Context             context.Context
+	ServerName          gomatrixserverlib.ServerName
+	FederationAPI       api.FederationInternalAPI
+	RelayAPI            relayServerAPI.RelayInternalAPI
+	relayServersQueried map[gomatrixserverlib.ServerName]bool
+	queriedServersMutex sync.Mutex
+	running             atomic.Bool
+}
+
+func (m *RelayServerRetriever) InitializeRelayServers(eLog *logrus.Entry) {
+	request := api.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(m.ServerName)}
+	response := api.P2PQueryRelayServersResponse{}
+	err := m.FederationAPI.P2PQueryRelayServers(m.Context, &request, &response)
+	if err != nil {
+		eLog.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
+	}
+	for _, server := range response.RelayServers {
+		m.relayServersQueried[server] = false
+	}
+
+	eLog.Infof("Registered relay servers: %v", response.RelayServers)
+}
+
+func (m *RelayServerRetriever) SyncRelayServers(stop <-chan bool) {
+	defer m.running.Store(false)
+
+	t := time.NewTimer(relayServerRetryInterval)
+	for {
+		relayServersToQuery := []gomatrixserverlib.ServerName{}
+		func() {
+			m.queriedServersMutex.Lock()
+			defer m.queriedServersMutex.Unlock()
+			for server, complete := range m.relayServersQueried {
+				if !complete {
+					relayServersToQuery = append(relayServersToQuery, server)
+				}
+			}
+		}()
+		if len(relayServersToQuery) == 0 {
+			// All relay servers have been synced.
+			return
+		}
+		m.queryRelayServers(relayServersToQuery)
+		t.Reset(relayServerRetryInterval)
+
+		select {
+		case <-stop:
+			if !t.Stop() {
+				<-t.C
+			}
+			return
+		case <-t.C:
+		}
+	}
+}
+
+func (m *RelayServerRetriever) GetQueriedServerStatus() map[gomatrixserverlib.ServerName]bool {
+	m.queriedServersMutex.Lock()
+	defer m.queriedServersMutex.Unlock()
+
+	result := map[gomatrixserverlib.ServerName]bool{}
+	for server, queried := range m.relayServersQueried {
+		result[server] = queried
+	}
+	return result
+}
+
+func (m *RelayServerRetriever) queryRelayServers(relayServers []gomatrixserverlib.ServerName) {
+	logrus.Info("querying relay servers for any available transactions")
+	for _, server := range relayServers {
+		userID, err := gomatrixserverlib.NewUserID("@user:"+string(m.ServerName), false)
+		if err != nil {
+			return
+		}
+		err = m.RelayAPI.PerformRelayServerSync(context.Background(), *userID, server)
+		if err == nil {
+			func() {
+				m.queriedServersMutex.Lock()
+				defer m.queriedServersMutex.Unlock()
+				m.relayServersQueried[server] = true
+			}()
+			// TODO : What happens if your relay receives new messages after this point?
+			// Should you continue to check with them, or should they try and contact you?
+			// They could send a "new_async_events" message your way maybe?
+			// Then you could mark them as needing to be queried again.
+			// What if you miss this message?
+			// Maybe you should try querying them again after a certain period of time as a backup?
+		} else {
+			logrus.Errorf("Failed querying relay server: %s", err.Error())
+		}
+	}
 }
 
 const MaxFrameSize = types.MaxFrameSize

build/gobind-pinecone/monolith_test.go

@@ -0,0 +1,198 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gobind
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/federationapi/api"
+	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"gotest.tools/v3/poll"
+)
+
+var TestBuf = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+
+type TestNetConn struct {
+	net.Conn
+	shouldFail bool
+}
+
+func (t *TestNetConn) Read(b []byte) (int, error) {
+	if t.shouldFail {
+		return 0, fmt.Errorf("Failed")
+	} else {
+		n := copy(b, TestBuf)
+		return n, nil
+	}
+}
+
+func (t *TestNetConn) Write(b []byte) (int, error) {
+	if t.shouldFail {
+		return 0, fmt.Errorf("Failed")
+	} else {
+		return len(b), nil
+	}
+}
+
+func (t *TestNetConn) Close() error {
+	if t.shouldFail {
+		return fmt.Errorf("Failed")
+	} else {
+		return nil
+	}
+}
+
+func TestConduitStoresPort(t *testing.T) {
+	conduit := Conduit{port: 7}
+	assert.Equal(t, 7, conduit.Port())
+}
+
+func TestConduitRead(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	b := make([]byte, len(TestBuf))
+	bytes, err := conduit.Read(b)
+	assert.NoError(t, err)
+	assert.Equal(t, len(TestBuf), bytes)
+	assert.Equal(t, TestBuf, b)
+}
+
+func TestConduitReadCopy(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	result, err := conduit.ReadCopy()
+	assert.NoError(t, err)
+	assert.Equal(t, TestBuf, result)
+}
+
+func TestConduitWrite(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	bytes, err := conduit.Write(TestBuf)
+	assert.NoError(t, err)
+	assert.Equal(t, len(TestBuf), bytes)
+}
+
+func TestConduitClose(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	assert.True(t, conduit.closed.Load())
+}
+
+func TestConduitReadClosed(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	b := make([]byte, len(TestBuf))
+	_, err = conduit.Read(b)
+	assert.Error(t, err)
+}
+
+func TestConduitReadCopyClosed(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	_, err = conduit.ReadCopy()
+	assert.Error(t, err)
+}
+
+func TestConduitWriteClosed(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	_, err = conduit.Write(TestBuf)
+	assert.Error(t, err)
+}
+
+func TestConduitReadCopyFails(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{shouldFail: true}}
+	_, err := conduit.ReadCopy()
+	assert.Error(t, err)
+}
+
+var testRelayServers = []gomatrixserverlib.ServerName{"relay1", "relay2"}
+
+type FakeFedAPI struct {
+	api.FederationInternalAPI
+}
+
+func (f *FakeFedAPI) P2PQueryRelayServers(ctx context.Context, req *api.P2PQueryRelayServersRequest, res *api.P2PQueryRelayServersResponse) error {
+	res.RelayServers = testRelayServers
+	return nil
+}
+
+type FakeRelayAPI struct {
+	relayServerAPI.RelayInternalAPI
+}
+
+func (r *FakeRelayAPI) PerformRelayServerSync(ctx context.Context, userID gomatrixserverlib.UserID, relayServer gomatrixserverlib.ServerName) error {
+	return nil
+}
+
+func TestRelayRetrieverInitialization(t *testing.T) {
+	retriever := RelayServerRetriever{
+		Context:             context.Background(),
+		ServerName:          "server",
+		relayServersQueried: make(map[gomatrixserverlib.ServerName]bool),
+		FederationAPI:       &FakeFedAPI{},
+		RelayAPI:            &FakeRelayAPI{},
+	}
+
+	retriever.InitializeRelayServers(logrus.WithField("test", "relay"))
+	relayServers := retriever.GetQueriedServerStatus()
+	assert.Equal(t, 2, len(relayServers))
+}
+
+func TestRelayRetrieverSync(t *testing.T) {
+	retriever := RelayServerRetriever{
+		Context:             context.Background(),
+		ServerName:          "server",
+		relayServersQueried: make(map[gomatrixserverlib.ServerName]bool),
+		FederationAPI:       &FakeFedAPI{},
+		RelayAPI:            &FakeRelayAPI{},
+	}
+
+	retriever.InitializeRelayServers(logrus.WithField("test", "relay"))
+	relayServers := retriever.GetQueriedServerStatus()
+	assert.Equal(t, 2, len(relayServers))
+
+	stopRelayServerSync := make(chan bool)
+	go retriever.SyncRelayServers(stopRelayServerSync)
+
+	check := func(log poll.LogT) poll.Result {
+		relayServers := retriever.GetQueriedServerStatus()
+		for _, queried := range relayServers {
+			if !queried {
+				return poll.Continue("waiting for all servers to be queried")
+			}
+		}
+
+		stopRelayServerSync <- true
+		return poll.Success()
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+}
+
+func TestMonolithStarts(t *testing.T) {
+	monolith := DendriteMonolith{}
+	monolith.Start()
+	monolith.PublicKey()
+	monolith.Stop()
+}

build/gobind-yggdrasil/monolith.go

@@ -150,6 +150,7 @@ func (m *DendriteMonolith) Start() {
 	}
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
 	m.processContext = base.ProcessContext
 	defer base.Close() // nolint: errcheck
 
@@ -164,7 +165,7 @@ func (m *DendriteMonolith) Start() {
 		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI)
 	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
@@ -196,6 +197,8 @@ func (m *DendriteMonolith) Start() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 
 	yggRouter := mux.NewRouter()
 	yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)

build/scripts/Complement.Dockerfile

@@ -16,13 +16,16 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
+    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server && \
+    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \
+    cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
 ENV API=0
+ENV COVER=0
 EXPOSE 8008 8448
 
 # At runtime, generate TLS cert based on the CA now mounted at /ca
@@ -30,4 +33,4 @@ EXPOSE 8008 8448
 CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    exec /complement-cmd.sh

build/scripts/ComplementLocal.Dockerfile

@@ -12,18 +12,20 @@ FROM golang:1.18-stretch
 RUN apt-get update && apt-get install -y sqlite3
 
 ENV SERVER_NAME=localhost
+ENV COVER=0
 EXPOSE 8008 8448
 
 WORKDIR /runtime
 # This script compiles Dendrite for us.
 RUN echo '\
     #!/bin/bash -eux \n\
-    if test -f "/runtime/dendrite-monolith-server"; then \n\
+    if test -f "/runtime/dendrite-monolith-server" && test -f "/runtime/dendrite-monolith-server-cover"; then \n\
     echo "Skipping compilation; binaries exist" \n\
     exit 0 \n\
     fi \n\
     cd /dendrite \n\
     go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
+    go test -c -cover -covermode=atomic -o /runtime/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite-monolith-server \n\
     ' > compile.sh && chmod +x compile.sh
 
 # This script runs Dendrite for us. Must be run in the /runtime directory.
@@ -33,6 +35,7 @@ RUN echo '\
     ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
+    [ ${COVER} -eq 1 ] && exec ./dendrite-monolith-server-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
     exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
     ' > run.sh && chmod +x run.sh
 

build/scripts/ComplementPostgres.Dockerfile

@@ -34,13 +34,16 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
+    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server && \
+    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \
+    cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite
 RUN ./generate-keys --private-key matrix_key.pem
 
 ENV SERVER_NAME=localhost
 ENV API=0
+ENV COVER=0
 EXPOSE 8008 8448
 
 
@@ -51,4 +54,4 @@ CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NA
     # Bump max_open_conns up here in the global database config
     sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
-    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
+    exec /complement-cmd.sh

build/scripts/complement-cmd.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -e
+
+# This script is intended to be used inside a docker container for Complement
+
+if [[ "${COVER}" -eq 1 ]]; then
+  echo "Running with coverage"
+  exec /dendrite/dendrite-monolith-server-cover \
+    --really-enable-open-registration \
+    --tls-cert server.crt \
+    --tls-key server.key \
+    --config dendrite.yaml \
+    -api=${API:-0} \
+    --test.coverprofile=integrationcover.log
+else
+  echo "Not running with coverage"
+  exec /dendrite/dendrite-monolith-server \
+    --really-enable-open-registration \
+    --tls-cert server.crt \
+    --tls-key server.key \
+    --config dendrite.yaml \
+    -api=${API:-0}
+fi

clientapi/admin_test.go

@@ -0,0 +1,235 @@
+package clientapi
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/tidwall/gjson"
+
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+func TestAdminResetPassword(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
+	vhUser := &test.User{ID: "@vhuser:vh1"}
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		// add a vhost
+		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+		})
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		// Needed for changing the password/login
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, nil)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]string{
+			aliceAdmin: "",
+			bob:        "",
+			vhUser:     "",
+		}
+		for u := range accessTokens {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+			password := util.RandomString(8)
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+
+			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+				"type": authtypes.LoginTypePassword,
+				"identifier": map[string]interface{}{
+					"type": "m.id.user",
+					"user": u.ID,
+				},
+				"password": password,
+			}))
+			rec := httptest.NewRecorder()
+			base.PublicClientAPIMux.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to login: %s", rec.Body.String())
+			}
+			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
+		}
+
+		testCases := []struct {
+			name           string
+			requestingUser *test.User
+			userID         string
+			requestOpt     test.HTTPRequestOpt
+			wantOK         bool
+			withHeader     bool
+		}{
+			{name: "Missing auth", requestingUser: bob, wantOK: false, userID: bob.ID},
+			{name: "Bob is denied access", requestingUser: bob, wantOK: false, withHeader: true, userID: bob.ID},
+			{name: "Alice is allowed access", requestingUser: aliceAdmin, wantOK: true, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(8),
+			})},
+			{name: "missing userID does not call function", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: ""}, // this 404s
+			{name: "rejects empty password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": "",
+			})},
+			{name: "rejects unknown server name", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "@doesnotexist:localhost", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
+			{name: "rejects unknown user", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "@doesnotexist:test", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
+			{name: "allows changing password for different vhost", requestingUser: aliceAdmin, wantOK: true, withHeader: true, userID: vhUser.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(8),
+			})},
+			{name: "rejects existing user, missing body", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID},
+			{name: "rejects invalid userID", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "!notauserid:test", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
+			{name: "rejects invalid json", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, `{invalidJSON}`)},
+			{name: "rejects too weak password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(6),
+			})},
+			{name: "rejects too long password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
+				"password": util.RandomString(513),
+			})},
+		}
+
+		for _, tc := range testCases {
+			tc := tc // ensure we don't accidentally only test the last test case
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/resetPassword/"+tc.userID)
+				if tc.requestOpt != nil {
+					req = test.NewRequest(t, http.MethodPost, "/_dendrite/admin/resetPassword/"+tc.userID, tc.requestOpt)
+				}
+
+				if tc.withHeader {
+					req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser])
+				}
+
+				rec := httptest.NewRecorder()
+				base.DendriteAdminMux.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+	})
+}
+
+func TestPurgeRoom(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin, test.RoomPreset(test.PresetTrustedPrivateChat))
+
+	// Invite Bob
+	room.CreateAndInsert(t, aliceAdmin, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "invite",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		fedClient := base.CreateFederationClient()
+		rsAPI := roomserver.NewInternalAPI(base)
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fedClient, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		syncapi.AddPublicRoutes(base, userAPI, rsAPI, keyAPI)
+		federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true)
+		rsAPI.SetFederationAPI(nil, nil)
+		keyAPI.SetUserAPI(userAPI)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, nil)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]string{
+			aliceAdmin: "",
+		}
+		for u := range accessTokens {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+			password := util.RandomString(8)
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+
+			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+				"type": authtypes.LoginTypePassword,
+				"identifier": map[string]interface{}{
+					"type": "m.id.user",
+					"user": u.ID,
+				},
+				"password": password,
+			}))
+			rec := httptest.NewRecorder()
+			base.PublicClientAPIMux.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to login: %s", rec.Body.String())
+			}
+			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
+		}
+
+		testCases := []struct {
+			name   string
+			roomID string
+			wantOK bool
+		}{
+			{name: "Can purge existing room", wantOK: true, roomID: room.ID},
+			{name: "Can not purge non-existent room", wantOK: false, roomID: "!doesnotexist:localhost"},
+			{name: "rejects invalid room ID", wantOK: false, roomID: "@doesnotexist:localhost"},
+		}
+
+		for _, tc := range testCases {
+			tc := tc // ensure we don't accidentally only test the last test case
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
+
+				rec := httptest.NewRecorder()
+				base.DendriteAdminMux.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+
+	})
+}

clientapi/auth/login_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -47,7 +48,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 				"password": "herpassword",
 				"device_id": "adevice"
             }`,
-			WantUsername: "alice",
+			WantUsername: "@alice:example.com",
 			WantDeviceID: "adevice",
 		},
 		{
@@ -174,7 +175,7 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 		return nil
 	}
 	res.Exists = true
-	res.Account = &uapi.Account{}
+	res.Account = &uapi.Account{UserID: userutil.MakeUserID(req.Localpart, req.ServerName)}
 	return nil
 }
 

clientapi/auth/password.go

@@ -101,6 +101,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		}
 	}
 
+	// If we couldn't find the user by the lower cased localpart, try the provided
+	// localpart as is.
 	if !res.Exists {
 		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
 			Localpart:         localpart,
@@ -122,5 +124,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			}
 		}
 	}
+	// Set the user, so login.Username() can do the right thing
+	r.Identifier.User = res.Account.UserID
+	r.User = res.Account.UserID
 	return &r.Login, nil
 }

clientapi/clientapi.go

@@ -15,6 +15,8 @@
 package clientapi
 
 import (
+	"github.com/matrix-org/gomatrixserverlib"
+
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/api"
 	"github.com/matrix-org/dendrite/clientapi/producers"
@@ -26,7 +28,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
@@ -57,10 +58,7 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		base.PublicClientAPIMux,
-		base.PublicWellKnownAPIMux,
-		base.SynapseAdminMux,
-		base.DendriteAdminMux,
+		base,
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI, keyAPI,

clientapi/routing/admin.go

@@ -1,12 +1,14 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
@@ -97,26 +99,77 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi
 	}
 }
 
-func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
 	}
-	serverName := cfg.Matrix.ServerName
-	localpart, ok := vars["localpart"]
+	roomID, ok := vars["roomID"]
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.MissingArgument("Expecting user localpart."),
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
 		}
 	}
-	if l, s, err := cfg.Matrix.SplitLocalID('@', localpart); err == nil {
-		localpart, serverName = l, s
+	res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
+	if err := rsAPI.PerformAdminPurgeRoom(
+		context.Background(),
+		&roomserverAPI.PerformAdminPurgeRoomRequest{
+			RoomID: roomID,
+		},
+		res,
+	); err != nil {
+		return util.ErrorResponse(err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: res,
+	}
+}
+
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+	if req.Body == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("Missing request body"),
+		}
+	}
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	var localpart string
+	userID := vars["userID"]
+	localpart, serverName, err := cfg.Matrix.SplitLocalID('@', userID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+	accAvailableResp := &userapi.QueryAccountAvailabilityResponse{}
+	if err = userAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{
+		Localpart:  localpart,
+		ServerName: serverName,
+	}, accAvailableResp); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.InternalAPIError(req.Context(), err),
+		}
+	}
+	if accAvailableResp.Available {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.Unknown("User does not exist"),
+		}
 	}
 	request := struct {
 		Password string `json:"password"`
 	}{}
-	if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+	if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
@@ -128,6 +181,11 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 			JSON: jsonerror.MissingArgument("Expecting non-empty password."),
 		}
 	}
+
+	if err = internal.ValidatePassword(request.Password); err != nil {
+		return *internal.PasswordResponse(err)
+	}
+
 	updateReq := &userapi.PerformPasswordUpdateRequest{
 		Localpart:     localpart,
 		ServerName:    serverName,

clientapi/routing/auth_fallback.go

@@ -15,11 +15,11 @@
 package routing
 
 import (
+	"fmt"
 	"html/template"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/util"
 )
@@ -101,14 +101,28 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s
 func AuthFallback(
 	w http.ResponseWriter, req *http.Request, authType string,
 	cfg *config.ClientAPI,
-) *util.JSONResponse {
-	sessionID := req.URL.Query().Get("session")
+) {
+	// We currently only support "m.login.recaptcha", so fail early if that's not requested
+	if authType == authtypes.LoginTypeRecaptcha {
+		if !cfg.RecaptchaEnabled {
+			writeHTTPMessage(w, req,
+				"Recaptcha login is disabled on this Homeserver",
+				http.StatusBadRequest,
+			)
+			return
+		}
+	} else {
+		writeHTTPMessage(w, req, fmt.Sprintf("Unknown authtype %q", authType), http.StatusNotImplemented)
+		return
+	}
 
+	sessionID := req.URL.Query().Get("session")
 	if sessionID == "" {
-		return writeHTTPMessage(w, req,
+		writeHTTPMessage(w, req,
 			"Session ID not provided",
 			http.StatusBadRequest,
 		)
+		return
 	}
 
 	serveRecaptcha := func() {
@@ -130,70 +144,44 @@ func AuthFallback(
 
 	if req.Method == http.MethodGet {
 		// Handle Recaptcha
-		if authType == authtypes.LoginTypeRecaptcha {
-			if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
-				return err
-			}
-
-			serveRecaptcha()
-			return nil
-		}
-		return &util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Unknown auth stage type"),
-		}
+		serveRecaptcha()
+		return
 	} else if req.Method == http.MethodPost {
 		// Handle Recaptcha
-		if authType == authtypes.LoginTypeRecaptcha {
-			if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
-				return err
-			}
-
-			clientIP := req.RemoteAddr
-			err := req.ParseForm()
-			if err != nil {
-				util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed")
-				res := jsonerror.InternalServerError()
-				return &res
-			}
-
-			response := req.Form.Get(cfg.RecaptchaFormField)
-			if err := validateRecaptcha(cfg, response, clientIP); err != nil {
-				util.GetLogger(req.Context()).Error(err)
-				return err
-			}
-
-			// Success. Add recaptcha as a completed login flow
-			sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
-
-			serveSuccess()
-			return nil
+		clientIP := req.RemoteAddr
+		err := req.ParseForm()
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed")
+			w.WriteHeader(http.StatusBadRequest)
+			serveRecaptcha()
+			return
 		}
 
-		return &util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Unknown auth stage type"),
+		response := req.Form.Get(cfg.RecaptchaFormField)
+		err = validateRecaptcha(cfg, response, clientIP)
+		switch err {
+		case ErrMissingResponse:
+			w.WriteHeader(http.StatusBadRequest)
+			serveRecaptcha() // serve the initial page again, instead of nothing
+			return
+		case ErrInvalidCaptcha:
+			w.WriteHeader(http.StatusUnauthorized)
+			serveRecaptcha()
+			return
+		case nil:
+		default: // something else failed
+			util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
+			serveRecaptcha()
+			return
 		}
-	}
-	return &util.JSONResponse{
-		Code: http.StatusMethodNotAllowed,
-		JSON: jsonerror.NotFound("Bad method"),
-	}
-}
 
-// checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver.
-func checkRecaptchaEnabled(
-	cfg *config.ClientAPI,
-	w http.ResponseWriter,
-	req *http.Request,
-) *util.JSONResponse {
-	if !cfg.RecaptchaEnabled {
-		return writeHTTPMessage(w, req,
-			"Recaptcha login is disabled on this Homeserver",
-			http.StatusBadRequest,
-		)
+		// Success. Add recaptcha as a completed login flow
+		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
+
+		serveSuccess()
+		return
 	}
-	return nil
+	writeHTTPMessage(w, req, "Bad method", http.StatusMethodNotAllowed)
 }
 
 // writeHTTPMessage writes the given header and message to the HTTP response writer.
@@ -201,13 +189,10 @@ func checkRecaptchaEnabled(
 func writeHTTPMessage(
 	w http.ResponseWriter, req *http.Request,
 	message string, header int,
-) *util.JSONResponse {
+) {
 	w.WriteHeader(header)
 	_, err := w.Write([]byte(message))
 	if err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("w.Write failed")
-		res := jsonerror.InternalServerError()
-		return &res
 	}
-	return nil
 }

clientapi/routing/auth_fallback_test.go

@@ -0,0 +1,149 @@
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test/testrig"
+)
+
+func Test_AuthFallback(t *testing.T) {
+	base, _, _ := testrig.Base(nil)
+	defer base.Close()
+
+	for _, useHCaptcha := range []bool{false, true} {
+		for _, recaptchaEnabled := range []bool{false, true} {
+			for _, wantErr := range []bool{false, true} {
+				t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) {
+					// Set the defaults for each test
+					base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, Monolithic: true})
+					base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
+					base.Cfg.ClientAPI.RecaptchaPublicKey = "pub"
+					base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv"
+					if useHCaptcha {
+						base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
+						base.Cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
+						base.Cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
+						base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
+					}
+					cfgErrs := &config.ConfigErrors{}
+					base.Cfg.ClientAPI.Verify(cfgErrs, true)
+					if len(*cfgErrs) > 0 {
+						t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error())
+					}
+
+					req := httptest.NewRequest(http.MethodGet, "/?session=1337", nil)
+					rec := httptest.NewRecorder()
+
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					if !recaptchaEnabled {
+						if rec.Code != http.StatusBadRequest {
+							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
+						}
+						if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
+							t.Fatalf("unexpected response body: %s", rec.Body.String())
+						}
+					} else {
+						if !strings.Contains(rec.Body.String(), base.Cfg.ClientAPI.RecaptchaSitekeyClass) {
+							t.Fatalf("body does not contain %s: %s", base.Cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
+						}
+					}
+
+					srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+						if wantErr {
+							_, _ = w.Write([]byte(`{"success":false}`))
+							return
+						}
+						_, _ = w.Write([]byte(`{"success":true}`))
+					}))
+					defer srv.Close() // nolint: errcheck
+
+					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+
+					// check the result after sending the captcha
+					req = httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
+					req.Form = url.Values{}
+					req.Form.Add(base.Cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
+					rec = httptest.NewRecorder()
+					AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+					if recaptchaEnabled {
+						if !wantErr {
+							if rec.Code != http.StatusOK {
+								t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusOK)
+							}
+							if rec.Body.String() != successTemplate {
+								t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), successTemplate)
+							}
+						} else {
+							if rec.Code != http.StatusUnauthorized {
+								t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusUnauthorized)
+							}
+							wantString := "Authentication"
+							if !strings.Contains(rec.Body.String(), wantString) {
+								t.Fatalf("expected response to contain '%s', but didn't: %s", wantString, rec.Body.String())
+							}
+						}
+					} else {
+						if rec.Code != http.StatusBadRequest {
+							t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
+						}
+						if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
+							t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), "successTemplate")
+						}
+					}
+				})
+			}
+		}
+	}
+
+	t.Run("unknown fallbacks are handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, "DoesNotExist", &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusNotImplemented {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusNotImplemented)
+		}
+	})
+
+	t.Run("unknown methods are handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodDelete, "/?session=1337", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusMethodNotAllowed {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusMethodNotAllowed)
+		}
+	})
+
+	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
+		}
+	})
+
+	t.Run("missing session parameter is handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
+		}
+	})
+
+	t.Run("missing 'response' is handled correctly", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
+		rec := httptest.NewRecorder()
+		AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
+		}
+	})
+}

clientapi/routing/joinroom.go

@@ -37,6 +37,7 @@ func JoinRoomByIDOrAlias(
 	joinReq := roomserverAPI.PerformJoinRequest{
 		RoomIDOrAlias: roomIDOrAlias,
 		UserID:        device.UserID,
+		IsGuest:       device.AccountType == api.AccountTypeGuest,
 		Content:       map[string]interface{}{},
 	}
 	joinRes := roomserverAPI.PerformJoinResponse{}
@@ -84,7 +85,14 @@ func JoinRoomByIDOrAlias(
 		if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
 			done <- jsonerror.InternalAPIError(req.Context(), err)
 		} else if joinRes.Error != nil {
-			done <- joinRes.Error.JSONResponse()
+			if joinRes.Error.Code == roomserverAPI.PerformErrorNotAllowed && device.AccountType == api.AccountTypeGuest {
+				done <- util.JSONResponse{
+					Code: http.StatusForbidden,
+					JSON: jsonerror.GuestAccessForbidden(joinRes.Error.Msg),
+				}
+			} else {
+				done <- joinRes.Error.JSONResponse()
+			}
 		} else {
 			done <- util.JSONResponse{
 				Code: http.StatusOK,

clientapi/routing/joinroom_test.go

@@ -0,0 +1,158 @@
+package routing
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+func TestJoinRoomByIDOrAlias(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t, test.WithAccountType(uapi.AccountTypeGuest))
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
+
+		// Create the users in the userapi
+		for _, u := range []*test.User{alice, bob, charlie} {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    "someRandomPassword",
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+
+		}
+
+		aliceDev := &uapi.Device{UserID: alice.ID}
+		bobDev := &uapi.Device{UserID: bob.ID}
+		charlieDev := &uapi.Device{UserID: charlie.ID, AccountType: uapi.AccountTypeGuest}
+
+		// create a room with disabled guest access and invite Bob
+		resp := createRoom(ctx, createRoomRequest{
+			Name:          "testing",
+			IsDirect:      true,
+			Topic:         "testing",
+			Visibility:    "public",
+			Preset:        presetPublicChat,
+			RoomAliasName: "alias",
+			Invite:        []string{bob.ID},
+			GuestCanJoin:  false,
+		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		crResp, ok := resp.JSON.(createRoomResponse)
+		if !ok {
+			t.Fatalf("response is not a createRoomResponse: %+v", resp)
+		}
+
+		// create a room with guest access enabled and invite Charlie
+		resp = createRoom(ctx, createRoomRequest{
+			Name:         "testing",
+			IsDirect:     true,
+			Topic:        "testing",
+			Visibility:   "public",
+			Preset:       presetPublicChat,
+			Invite:       []string{charlie.ID},
+			GuestCanJoin: true,
+		}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
+		crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse)
+		if !ok {
+			t.Fatalf("response is not a createRoomResponse: %+v", resp)
+		}
+
+		// Dummy request
+		body := &bytes.Buffer{}
+		req, err := http.NewRequest(http.MethodPost, "/?server_name=test", body)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		testCases := []struct {
+			name        string
+			device      *uapi.Device
+			roomID      string
+			wantHTTP200 bool
+		}{
+			{
+				name:        "User can join successfully by alias",
+				device:      bobDev,
+				roomID:      crResp.RoomAlias,
+				wantHTTP200: true,
+			},
+			{
+				name:        "User can join successfully by roomID",
+				device:      bobDev,
+				roomID:      crResp.RoomID,
+				wantHTTP200: true,
+			},
+			{
+				name:   "join is forbidden if user is guest",
+				device: charlieDev,
+				roomID: crResp.RoomID,
+			},
+			{
+				name:   "room does not exist",
+				device: aliceDev,
+				roomID: "!doesnotexist:test",
+			},
+			{
+				name:   "user from different server",
+				device: &uapi.Device{UserID: "@wrong:server"},
+				roomID: crResp.RoomAlias,
+			},
+			{
+				name:   "user doesn't exist locally",
+				device: &uapi.Device{UserID: "@doesnotexist:test"},
+				roomID: crResp.RoomAlias,
+			},
+			{
+				name:   "invalid room ID",
+				device: aliceDev,
+				roomID: "invalidRoomID",
+			},
+			{
+				name:   "roomAlias does not exist",
+				device: aliceDev,
+				roomID: "#doesnotexist:test",
+			},
+			{
+				name:   "room with guest_access event",
+				device: charlieDev,
+				roomID: crRespWithGuestAccess.RoomID,
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				joinResp := JoinRoomByIDOrAlias(req, tc.device, rsAPI, userAPI, tc.roomID)
+				if tc.wantHTTP200 && !joinResp.Is2xx() {
+					t.Fatalf("expected join room to succeed, but didn't: %+v", joinResp)
+				}
+			})
+		}
+	})
+}

clientapi/routing/login.go

@@ -23,15 +23,13 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
 type loginResponse struct {
-	UserID      string                       `json:"user_id"`
-	AccessToken string                       `json:"access_token"`
-	HomeServer  gomatrixserverlib.ServerName `json:"home_server"`
-	DeviceID    string                       `json:"device_id"`
+	UserID      string `json:"user_id"`
+	AccessToken string `json:"access_token"`
+	DeviceID    string `json:"device_id"`
 }
 
 type flows struct {
@@ -116,7 +114,6 @@ func completeAuth(
 		JSON: loginResponse{
 			UserID:      performRes.Device.UserID,
 			AccessToken: performRes.Device.AccessToken,
-			HomeServer:  serverName,
 			DeviceID:    performRes.Device.ID,
 		},
 	}

clientapi/routing/login_test.go

@@ -0,0 +1,152 @@
+package routing
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+func TestLogin(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bobUser := &test.User{ID: "@bob:test", AccountType: uapi.AccountTypeUser}
+	charlie := &test.User{ID: "@Charlie:test", AccountType: uapi.AccountTypeUser}
+	vhUser := &test.User{ID: "@vhuser:vh1"}
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.ClientAPI.RateLimiting.Enabled = false
+		// add a vhost
+		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+		})
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		// Needed for /login
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+
+		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
+		Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, keyAPI, nil, &base.Cfg.MSCs, nil)
+
+		// Create password
+		password := util.RandomString(8)
+
+		// create the users
+		for _, u := range []*test.User{aliceAdmin, bobUser, vhUser, charlie} {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+			if !userRes.AccountCreated {
+				t.Fatalf("account not created")
+			}
+		}
+
+		testCases := []struct {
+			name   string
+			userID string
+			wantOK bool
+		}{
+			{
+				name:   "aliceAdmin can login",
+				userID: aliceAdmin.ID,
+				wantOK: true,
+			},
+			{
+				name:   "bobUser can login",
+				userID: bobUser.ID,
+				wantOK: true,
+			},
+			{
+				name:   "vhuser can login",
+				userID: vhUser.ID,
+				wantOK: true,
+			},
+			{
+				name:   "bob with uppercase can login",
+				userID: "@Bob:test",
+				wantOK: true,
+			},
+			{
+				name:   "Charlie can login (existing uppercase)",
+				userID: charlie.ID,
+				wantOK: true,
+			},
+			{
+				name:   "Charlie can not login with lowercase userID",
+				userID: strings.ToLower(charlie.ID),
+				wantOK: false,
+			},
+		}
+
+		ctx := context.Background()
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+					"type": authtypes.LoginTypePassword,
+					"identifier": map[string]interface{}{
+						"type": "m.id.user",
+						"user": tc.userID,
+					},
+					"password": password,
+				}))
+				rec := httptest.NewRecorder()
+				base.PublicClientAPIMux.ServeHTTP(rec, req)
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("failed to login: %s", rec.Body.String())
+				}
+
+				t.Logf("Response: %s", rec.Body.String())
+				// get the response
+				resp := loginResponse{}
+				if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+					t.Fatal(err)
+				}
+				// everything OK
+				if !tc.wantOK && resp.AccessToken == "" {
+					return
+				}
+				if tc.wantOK && resp.AccessToken == "" {
+					t.Fatalf("expected accessToken after successful login but got none: %+v", resp)
+				}
+
+				devicesResp := &uapi.QueryDevicesResponse{}
+				if err := userAPI.QueryDevices(ctx, &uapi.QueryDevicesRequest{UserID: resp.UserID}, devicesResp); err != nil {
+					t.Fatal(err)
+				}
+				for _, dev := range devicesResp.Devices {
+					// We expect the userID on the device to be the same as resp.UserID
+					if dev.UserID != resp.UserID {
+						t.Fatalf("unexpected userID on device: %s", dev.UserID)
+					}
+				}
+			})
+		}
+	})
+}

clientapi/routing/password.go

@@ -7,6 +7,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -81,8 +82,8 @@ func Password(
 	sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
 
 	// Check the new password strength.
-	if resErr = validatePassword(r.NewPassword); resErr != nil {
-		return *resErr
+	if err := internal.ValidatePassword(r.NewPassword); err != nil {
+		return *internal.PasswordResponse(err)
 	}
 
 	// Get the local part.

clientapi/routing/register.go

@@ -18,18 +18,19 @@ package routing
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
@@ -59,12 +60,7 @@ var (
 	)
 )
 
-const (
-	minPasswordLength = 8   // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
-	maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
-	maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain
-	sessionIDLength   = 24
-)
+const sessionIDLength = 24
 
 // sessionsDict keeps track of completed auth stages for each session.
 // It shouldn't be passed by value because it contains a mutex.
@@ -199,8 +195,7 @@ func (d *sessionsDict) getDeviceToDelete(sessionID string) (string, bool) {
 }
 
 var (
-	sessions           = newSessionsDict()
-	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
+	sessions = newSessionsDict()
 )
 
 // registerRequest represents the submitted registration request.
@@ -263,10 +258,9 @@ func newUserInteractiveResponse(
 
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 type registerResponse struct {
-	UserID      string                       `json:"user_id"`
-	AccessToken string                       `json:"access_token,omitempty"`
-	HomeServer  gomatrixserverlib.ServerName `json:"home_server"`
-	DeviceID    string                       `json:"device_id,omitempty"`
+	UserID      string `json:"user_id"`
+	AccessToken string `json:"access_token,omitempty"`
+	DeviceID    string `json:"device_id,omitempty"`
 }
 
 // recaptchaResponse represents the HTTP response from a Google Recaptcha server
@@ -277,83 +271,28 @@ type recaptchaResponse struct {
 	ErrorCodes  []int     `json:"error-codes"`
 }
 
-// validateUsername returns an error response if the username is invalid
-func validateUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
-	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
-	if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
-		}
-	} else if !validUsernameRegex.MatchString(localpart) {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
-		}
-	} else if localpart[0] == '_' { // Regex checks its not a zero length string
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username cannot start with a '_'"),
-		}
-	}
-	return nil
-}
-
-// validateApplicationServiceUsername returns an error response if the username is invalid for an application service
-func validateApplicationServiceUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
-	if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
-		}
-	} else if !validUsernameRegex.MatchString(localpart) {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
-		}
-	}
-	return nil
-}
-
-// validatePassword returns an error response if the password is invalid
-func validatePassword(password string) *util.JSONResponse {
-	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
-	if len(password) > maxPasswordLength {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(fmt.Sprintf("'password' >%d characters", maxPasswordLength)),
-		}
-	} else if len(password) > 0 && len(password) < minPasswordLength {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.WeakPassword(fmt.Sprintf("password too weak: min %d chars", minPasswordLength)),
-		}
-	}
-	return nil
-}
+var (
+	ErrInvalidCaptcha  = errors.New("invalid captcha response")
+	ErrMissingResponse = errors.New("captcha response is required")
+	ErrCaptchaDisabled = errors.New("captcha registration is disabled")
+)
 
 // validateRecaptcha returns an error response if the captcha response is invalid
 func validateRecaptcha(
 	cfg *config.ClientAPI,
 	response string,
 	clientip string,
-) *util.JSONResponse {
+) error {
 	ip, _, _ := net.SplitHostPort(clientip)
 	if !cfg.RecaptchaEnabled {
-		return &util.JSONResponse{
-			Code: http.StatusConflict,
-			JSON: jsonerror.Unknown("Captcha registration is disabled"),
-		}
+		return ErrCaptchaDisabled
 	}
 
 	if response == "" {
-		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Captcha response is required"),
-		}
+		return ErrMissingResponse
 	}
 
-	// Make a POST request to Google's API to check the captcha response
+	// Make a POST request to the captcha provider API to check the captcha response
 	resp, err := http.PostForm(cfg.RecaptchaSiteVerifyAPI,
 		url.Values{
 			"secret":   {cfg.RecaptchaPrivateKey},
@@ -363,10 +302,7 @@ func validateRecaptcha(
 	)
 
 	if err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: jsonerror.BadJSON("Error in requesting validation of captcha response"),
-		}
+		return err
 	}
 
 	// Close the request once we're finishing reading from it
@@ -376,25 +312,16 @@ func validateRecaptcha(
 	var r recaptchaResponse
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusGatewayTimeout,
-			JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()),
-		}
+		return err
 	}
 	err = json.Unmarshal(body, &r)
 	if err != nil {
-		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: jsonerror.BadJSON("Error in unmarshaling captcha server's response: " + err.Error()),
-		}
+		return err
 	}
 
 	// Check that we received a "success"
 	if !r.Success {
-		return &util.JSONResponse{
-			Code: http.StatusUnauthorized,
-			JSON: jsonerror.BadJSON("Invalid captcha response. Please try again."),
-		}
+		return ErrInvalidCaptcha
 	}
 	return nil
 }
@@ -526,8 +453,8 @@ func validateApplicationService(
 	}
 
 	// Check username application service is trying to register is valid
-	if err := validateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
-		return "", err
+	if err := internal.ValidateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
+		return "", internal.UsernameResponse(err)
 	}
 
 	// No errors, registration valid
@@ -582,15 +509,12 @@ func Register(
 	if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil {
 		return *resErr
 	}
-	if l, d, err := cfg.Matrix.SplitLocalID('@', r.Username); err == nil {
-		r.Username, r.ServerName = l, d
-	}
 	if req.URL.Query().Get("kind") == "guest" {
 		return handleGuestRegistration(req, r, cfg, userAPI)
 	}
 
 	// Don't allow numeric usernames less than MAX_INT64.
-	if _, err := strconv.ParseInt(r.Username, 10, 64); err == nil {
+	if _, err = strconv.ParseInt(r.Username, 10, 64); err == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
@@ -602,7 +526,7 @@ func Register(
 			ServerName: r.ServerName,
 		}
 		nres := &userapi.QueryNumericLocalpartResponse{}
-		if err := userAPI.QueryNumericLocalpart(req.Context(), nreq, nres); err != nil {
+		if err = userAPI.QueryNumericLocalpart(req.Context(), nreq, nres); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryNumericLocalpart failed")
 			return jsonerror.InternalServerError()
 		}
@@ -619,8 +543,8 @@ func Register(
 	case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil:
 		// Spec-compliant case (the access_token is specified and the login type
 		// is correctly set, so it's an appservice registration)
-		if resErr := validateApplicationServiceUsername(r.Username, r.ServerName); resErr != nil {
-			return *resErr
+		if err = internal.ValidateApplicationServiceUsername(r.Username, r.ServerName); err != nil {
+			return *internal.UsernameResponse(err)
 		}
 	case accessTokenErr == nil:
 		// Non-spec-compliant case (the access_token is specified but the login
@@ -632,12 +556,12 @@ func Register(
 	default:
 		// Spec-compliant case (neither the access_token nor the login type are
 		// specified, so it's a normal user registration)
-		if resErr := validateUsername(r.Username, r.ServerName); resErr != nil {
-			return *resErr
+		if err = internal.ValidateUsername(r.Username, r.ServerName); err != nil {
+			return *internal.UsernameResponse(err)
 		}
 	}
-	if resErr := validatePassword(r.Password); resErr != nil {
-		return *resErr
+	if err = internal.ValidatePassword(r.Password); err != nil {
+		return *internal.PasswordResponse(err)
 	}
 
 	logger := util.GetLogger(req.Context())
@@ -715,7 +639,6 @@ func handleGuestRegistration(
 		JSON: registerResponse{
 			UserID:      devRes.Device.UserID,
 			AccessToken: devRes.Device.AccessToken,
-			HomeServer:  res.Account.ServerName,
 			DeviceID:    devRes.Device.ID,
 		},
 	}
@@ -779,9 +702,18 @@ func handleRegistrationFlow(
 	switch r.Auth.Type {
 	case authtypes.LoginTypeRecaptcha:
 		// Check given captcha response
-		resErr := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
-		if resErr != nil {
-			return *resErr
+		err := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
+		switch err {
+		case ErrCaptchaDisabled:
+			return util.JSONResponse{Code: http.StatusForbidden, JSON: jsonerror.Unknown(err.Error())}
+		case ErrMissingResponse:
+			return util.JSONResponse{Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error())}
+		case ErrInvalidCaptcha:
+			return util.JSONResponse{Code: http.StatusUnauthorized, JSON: jsonerror.BadJSON(err.Error())}
+		case nil:
+		default:
+			util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
+			return util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()}
 		}
 
 		// Add Recaptcha to the list of completed registration stages
@@ -848,7 +780,7 @@ func handleApplicationServiceRegistration(
 	// Don't need to worry about appending to registration stages as
 	// application service registration is entirely separate.
 	return completeRegistration(
-		req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr,
+		req.Context(), userAPI, r.Username, r.ServerName, "", "", appserviceID, req.RemoteAddr,
 		req.UserAgent(), r.Auth.Session, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 		userapi.AccountTypeAppService,
 	)
@@ -868,7 +800,7 @@ func checkAndCompleteFlow(
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
-			req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr,
+			req.Context(), userAPI, r.Username, r.ServerName, "", r.Password, "", req.RemoteAddr,
 			req.UserAgent(), sessionID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 			userapi.AccountTypeUser,
 		)
@@ -892,10 +824,10 @@ func checkAndCompleteFlow(
 func completeRegistration(
 	ctx context.Context,
 	userAPI userapi.ClientUserAPI,
-	username string, serverName gomatrixserverlib.ServerName,
+	username string, serverName gomatrixserverlib.ServerName, displayName string,
 	password, appserviceID, ipAddr, userAgent, sessionID string,
 	inhibitLogin eventutil.WeakBoolean,
-	displayName, deviceID *string,
+	deviceDisplayName, deviceID *string,
 	accType userapi.AccountType,
 ) util.JSONResponse {
 	if username == "" {
@@ -942,8 +874,7 @@ func completeRegistration(
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: registerResponse{
-				UserID:     userutil.MakeUserID(username, accRes.Account.ServerName),
-				HomeServer: accRes.Account.ServerName,
+				UserID: userutil.MakeUserID(username, accRes.Account.ServerName),
 			},
 		}
 	}
@@ -956,12 +887,28 @@ func completeRegistration(
 		}
 	}
 
+	if displayName != "" {
+		nameReq := userapi.PerformUpdateDisplayNameRequest{
+			Localpart:   username,
+			ServerName:  serverName,
+			DisplayName: displayName,
+		}
+		var nameRes userapi.PerformUpdateDisplayNameResponse
+		err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("failed to set display name: " + err.Error()),
+			}
+		}
+	}
+
 	var devRes userapi.PerformDeviceCreationResponse
 	err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
 		Localpart:         username,
 		ServerName:        serverName,
 		AccessToken:       token,
-		DeviceDisplayName: displayName,
+		DeviceDisplayName: deviceDisplayName,
 		DeviceID:          deviceID,
 		IPAddr:            ipAddr,
 		UserAgent:         userAgent,
@@ -976,7 +923,6 @@ func completeRegistration(
 	result := registerResponse{
 		UserID:      devRes.Device.UserID,
 		AccessToken: devRes.Device.AccessToken,
-		HomeServer:  accRes.Account.ServerName,
 		DeviceID:    devRes.Device.ID,
 	}
 	sessions.addCompletedRegistration(sessionID, result)
@@ -1072,8 +1018,8 @@ func RegisterAvailable(
 		}
 	}
 
-	if err := validateUsername(username, domain); err != nil {
-		return *err
+	if err := internal.ValidateUsername(username, domain); err != nil {
+		return *internal.UsernameResponse(err)
 	}
 
 	// Check if this username is reserved by an application service
@@ -1135,11 +1081,11 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	// downcase capitals
 	ssrr.User = strings.ToLower(ssrr.User)
 
-	if resErr := validateUsername(ssrr.User, cfg.Matrix.ServerName); resErr != nil {
-		return *resErr
+	if err = internal.ValidateUsername(ssrr.User, cfg.Matrix.ServerName); err != nil {
+		return *internal.UsernameResponse(err)
 	}
-	if resErr := validatePassword(ssrr.Password); resErr != nil {
-		return *resErr
+	if err = internal.ValidatePassword(ssrr.Password); err != nil {
+		return *internal.PasswordResponse(err)
 	}
 	deviceID := "shared_secret_registration"
 
@@ -1147,5 +1093,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	if ssrr.Admin {
 		accType = userapi.AccountTypeAdmin
 	}
-	return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
+	return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.DisplayName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
 }

clientapi/routing/register_secret.go

@@ -18,12 +18,13 @@ import (
 )
 
 type SharedSecretRegistrationRequest struct {
-	User     string `json:"username"`
-	Password string `json:"password"`
-	Nonce    string `json:"nonce"`
-	MacBytes []byte
-	MacStr   string `json:"mac"`
-	Admin    bool   `json:"admin"`
+	User        string `json:"username"`
+	Password    string `json:"password"`
+	Nonce       string `json:"nonce"`
+	MacBytes    []byte
+	MacStr      string `json:"mac"`
+	Admin       bool   `json:"admin"`
+	DisplayName string `json:"displayname,omitempty"`
 }
 
 func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) {

clientapi/routing/register_secret_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestSharedSecretRegister(t *testing.T) {
 	// these values have come from a local synapse instance to ensure compatibility
-	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
+	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
 	sharedSecret := "dendritetest"
 
 	req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))

clientapi/routing/register_test.go

@@ -15,12 +15,31 @@
 package routing
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
 	"regexp"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+	"github.com/patrickmn/go-cache"
+	"github.com/stretchr/testify/assert"
 )
 
 var (
@@ -264,3 +283,383 @@ func TestSessionCleanUp(t *testing.T) {
 		}
 	})
 }
+
+func Test_register(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		kind                 string
+		password             string
+		username             string
+		loginType            string
+		forceEmpty           bool
+		registrationDisabled bool
+		guestsDisabled       bool
+		enableRecaptcha      bool
+		captchaBody          string
+		wantResponse         util.JSONResponse
+	}{
+		{
+			name:           "disallow guests",
+			kind:           "guest",
+			guestsDisabled: true,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden(`Guest registration is disabled on "test"`),
+			},
+		},
+		{
+			name: "allow guests",
+			kind: "guest",
+		},
+		{
+			name:      "unknown login type",
+			loginType: "im.not.known",
+			wantResponse: util.JSONResponse{
+				Code: http.StatusNotImplemented,
+				JSON: jsonerror.Unknown("unknown/unimplemented auth type"),
+			},
+		},
+		{
+			name:                 "disabled registration",
+			registrationDisabled: true,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden(`Registration is disabled on "test"`),
+			},
+		},
+		{
+			name:       "successful registration, numeric ID",
+			username:   "",
+			password:   "someRandomPassword",
+			forceEmpty: true,
+		},
+		{
+			name:     "successful registration",
+			username: "success",
+		},
+		{
+			name:     "failing registration - user already exists",
+			username: "success",
+			wantResponse: util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+			},
+		},
+		{
+			name:     "successful registration uppercase username",
+			username: "LOWERCASED", // this is going to be lower-cased
+		},
+		{
+			name:         "invalid username",
+			username:     "#totalyNotValid",
+			wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
+		},
+		{
+			name:     "numeric username is forbidden",
+			username: "1337",
+			wantResponse: util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
+			},
+		},
+		{
+			name:      "disabled recaptcha login",
+			loginType: authtypes.LoginTypeRecaptcha,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Unknown(ErrCaptchaDisabled.Error()),
+			},
+		},
+		{
+			name:            "enabled recaptcha, no response defined",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON(ErrMissingResponse.Error()),
+			},
+		},
+		{
+			name:            "invalid captcha response",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			captchaBody:     `notvalid`,
+			wantResponse: util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.BadJSON(ErrInvalidCaptcha.Error()),
+			},
+		},
+		{
+			name:            "valid captcha response",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			captchaBody:     `success`,
+		},
+		{
+			name:            "captcha invalid from remote",
+			enableRecaptcha: true,
+			loginType:       authtypes.LoginTypeRecaptcha,
+			captchaBody:     `i should fail for other reasons`,
+			wantResponse:    util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()},
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				if tc.enableRecaptcha {
+					srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+						if err := r.ParseForm(); err != nil {
+							t.Fatal(err)
+						}
+						response := r.Form.Get("response")
+
+						// Respond with valid JSON or no JSON at all to test happy/error cases
+						switch response {
+						case "success":
+							json.NewEncoder(w).Encode(recaptchaResponse{Success: true})
+						case "notvalid":
+							json.NewEncoder(w).Encode(recaptchaResponse{Success: false})
+						default:
+
+						}
+					}))
+					defer srv.Close()
+					base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
+				}
+
+				if err := base.Cfg.Derive(); err != nil {
+					t.Fatalf("failed to derive config: %s", err)
+				}
+
+				base.Cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha
+				base.Cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled
+				base.Cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled
+
+				if tc.kind == "" {
+					tc.kind = "user"
+				}
+				if tc.password == "" && !tc.forceEmpty {
+					tc.password = "someRandomPassword"
+				}
+				if tc.username == "" && !tc.forceEmpty {
+					tc.username = "valid"
+				}
+				if tc.loginType == "" {
+					tc.loginType = "m.login.dummy"
+				}
+
+				reg := registerRequest{
+					Password: tc.password,
+					Username: tc.username,
+				}
+
+				body := &bytes.Buffer{}
+				err := json.NewEncoder(body).Encode(reg)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/?kind=%s", tc.kind), body)
+
+				resp := Register(req, userAPI, &base.Cfg.ClientAPI)
+				t.Logf("Resp: %+v", resp)
+
+				// The first request should return a userInteractiveResponse
+				switch r := resp.JSON.(type) {
+				case userInteractiveResponse:
+					// Check that the flows are the ones we configured
+					if !reflect.DeepEqual(r.Flows, base.Cfg.Derived.Registration.Flows) {
+						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, base.Cfg.Derived.Registration.Flows)
+					}
+				case *jsonerror.MatrixError:
+					if !reflect.DeepEqual(tc.wantResponse, resp) {
+						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse)
+					}
+					return
+				case registerResponse:
+					// this should only be possible on guest user registration, never for normal users
+					if tc.kind != "guest" {
+						t.Fatalf("got register response on first request: %+v", r)
+					}
+					// assert we've got a UserID, AccessToken and DeviceID
+					if r.UserID == "" {
+						t.Fatalf("missing userID in response")
+					}
+					if r.AccessToken == "" {
+						t.Fatalf("missing accessToken in response")
+					}
+					if r.DeviceID == "" {
+						t.Fatalf("missing deviceID in response")
+					}
+					return
+				default:
+					t.Logf("Got response: %T", resp.JSON)
+				}
+
+				// If we reached this, we should have received a UIA response
+				uia, ok := resp.JSON.(userInteractiveResponse)
+				if !ok {
+					t.Fatalf("did not receive a userInteractiveResponse: %T", resp.JSON)
+				}
+				t.Logf("%+v", uia)
+
+				// Register the user
+				reg.Auth = authDict{
+					Type:    authtypes.LoginType(tc.loginType),
+					Session: uia.Session,
+				}
+
+				if tc.captchaBody != "" {
+					reg.Auth.Response = tc.captchaBody
+				}
+
+				dummy := "dummy"
+				reg.DeviceID = &dummy
+				reg.InitialDisplayName = &dummy
+				reg.Type = authtypes.LoginType(tc.loginType)
+
+				err = json.NewEncoder(body).Encode(reg)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				req = httptest.NewRequest(http.MethodPost, "/", body)
+
+				resp = Register(req, userAPI, &base.Cfg.ClientAPI)
+
+				switch resp.JSON.(type) {
+				case *jsonerror.MatrixError:
+					if !reflect.DeepEqual(tc.wantResponse, resp) {
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+					}
+					return
+				case util.JSONResponse:
+					if !reflect.DeepEqual(tc.wantResponse, resp) {
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+					}
+					return
+				}
+
+				rr, ok := resp.JSON.(registerResponse)
+				if !ok {
+					t.Fatalf("expected a registerresponse, got %T", resp.JSON)
+				}
+
+				// validate the response
+				if tc.forceEmpty {
+					// when not supplying a username, one will be generated. Given this _SHOULD_ be
+					// the second user, set the username accordingly
+					reg.Username = "2"
+				}
+				wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test"))
+				if wantUserID != rr.UserID {
+					t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
+				}
+				if rr.DeviceID != *reg.DeviceID {
+					t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
+				}
+				if rr.AccessToken == "" {
+					t.Fatalf("missing accessToken in response")
+				}
+			})
+		}
+	})
+}
+
+func TestRegisterUserWithDisplayName(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.Global.ServerName = "server"
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+		deviceName, deviceID := "deviceName", "deviceID"
+		expectedDisplayName := "DisplayName"
+		response := completeRegistration(
+			base.Context(),
+			userAPI,
+			"user",
+			"server",
+			expectedDisplayName,
+			"password",
+			"",
+			"localhost",
+			"user agent",
+			"session",
+			false,
+			&deviceName,
+			&deviceID,
+			api.AccountTypeAdmin,
+		)
+
+		assert.Equal(t, http.StatusOK, response.Code)
+
+		req := api.QueryProfileRequest{UserID: "@user:server"}
+		var res api.QueryProfileResponse
+		err := userAPI.QueryProfile(base.Context(), &req, &res)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedDisplayName, res.DisplayName)
+	})
+}
+
+func TestRegisterAdminUsingSharedSecret(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.Global.ServerName = "server"
+		sharedSecret := "dendritetest"
+		base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+
+		expectedDisplayName := "rabbit"
+		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
+		req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))
+		assert.NoError(t, err)
+		if err != nil {
+			t.Fatalf("failed to read request: %s", err)
+		}
+
+		r := NewSharedSecretRegistration(sharedSecret)
+
+		// force the nonce to be known
+		r.nonces.Set(req.Nonce, true, cache.DefaultExpiration)
+
+		_, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
+		assert.NoError(t, err)
+
+		body := &bytes.Buffer{}
+		err = json.NewEncoder(body).Encode(req)
+		assert.NoError(t, err)
+		ssrr := httptest.NewRequest(http.MethodPost, "/", body)
+
+		response := handleSharedSecretRegistration(
+			&base.Cfg.ClientAPI,
+			userAPI,
+			r,
+			ssrr,
+		)
+		assert.Equal(t, http.StatusOK, response.Code)
+
+		profilReq := api.QueryProfileRequest{UserID: "@alice:server"}
+		var profileRes api.QueryProfileResponse
+		err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedDisplayName, profileRes.DisplayName)
+	})
+}

clientapi/routing/routing.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
@@ -49,7 +50,7 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux, wkMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
+	base *base.BaseDendrite,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
@@ -63,7 +64,14 @@ func Setup(
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs, natsClient *nats.Conn,
 ) {
-	prometheus.MustRegister(amtRegUsers, sendEventDuration)
+	publicAPIMux := base.PublicClientAPIMux
+	wkMux := base.PublicWellKnownAPIMux
+	synapseAdminRouter := base.SynapseAdminMux
+	dendriteAdminRouter := base.DendriteAdminMux
+
+	if base.EnableMetrics {
+		prometheus.MustRegister(amtRegUsers, sendEventDuration)
+	}
 
 	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
 	userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)
@@ -157,6 +165,12 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
+		httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminPurgeRoom(req, cfg, device, rsAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	dendriteAdminRouter.Handle("/admin/resetPassword/{userID}",
 		httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return AdminResetPassword(req, cfg, device, userAPI)
@@ -631,9 +645,9 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 
 	v3mux.Handle("/auth/{authType}/fallback/web",
-		httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
+		httputil.MakeHTMLAPI("auth_fallback", base.EnableMetrics, func(w http.ResponseWriter, req *http.Request) {
 			vars := mux.Vars(req)
-			return AuthFallback(w, req, vars["authType"], cfg)
+			AuthFallback(w, req, vars["authType"], cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
 

clientapi/routing/thirdparty.go

@@ -36,9 +36,15 @@ func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, dev
 		return jsonerror.InternalServerError()
 	}
 	if !resp.Exists {
+		if protocol != "" {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: jsonerror.NotFound("The protocol is unknown."),
+			}
+		}
 		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("The protocol is unknown."),
+			Code: http.StatusOK,
+			JSON: struct{}{},
 		}
 	}
 	if protocol != "" {

cmd/create-account/main.go

@@ -25,10 +25,10 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"regexp"
 	"strings"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal"
 	"github.com/tidwall/gjson"
 
 	"github.com/sirupsen/logrus"
@@ -58,15 +58,14 @@ Arguments:
 `
 
 var (
-	username           = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
-	password           = flag.String("password", "", "The password to associate with the account")
-	pwdFile            = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
-	pwdStdin           = flag.Bool("passwordstdin", false, "Reads the password from stdin")
-	isAdmin            = flag.Bool("admin", false, "Create an admin account")
-	resetPassword      = flag.Bool("reset-password", false, "Deprecated")
-	serverURL          = flag.String("url", "http://localhost:8008", "The URL to connect to.")
-	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
-	timeout            = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
+	username      = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
+	password      = flag.String("password", "", "The password to associate with the account")
+	pwdFile       = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
+	pwdStdin      = flag.Bool("passwordstdin", false, "Reads the password from stdin")
+	isAdmin       = flag.Bool("admin", false, "Create an admin account")
+	resetPassword = flag.Bool("reset-password", false, "Deprecated")
+	serverURL     = flag.String("url", "http://localhost:8008", "The URL to connect to.")
+	timeout       = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
 )
 
 var cl = http.Client{
@@ -95,20 +94,21 @@ func main() {
 		os.Exit(1)
 	}
 
-	if !validUsernameRegex.MatchString(*username) {
-		logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
+	if err := internal.ValidateUsername(*username, cfg.Global.ServerName); err != nil {
+		logrus.WithError(err).Error("Specified username is invalid")
 		os.Exit(1)
 	}
 
-	if len(fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName)) > 255 {
-		logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
-	}
-
 	pass, err := getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
 	if err != nil {
 		logrus.Fatalln(err)
 	}
 
+	if err = internal.ValidatePassword(pass); err != nil {
+		logrus.WithError(err).Error("Specified password is invalid")
+		os.Exit(1)
+	}
+
 	cl.Timeout = *timeout
 
 	accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
@@ -177,7 +177,7 @@ func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, a
 	defer regResp.Body.Close() // nolint: errcheck
 	if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
 		body, _ = io.ReadAll(regResp.Body)
-		return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
+		return "", fmt.Errorf("got HTTP %d error from server: %s", regResp.StatusCode, string(body))
 	}
 	r, err := io.ReadAll(regResp.Body)
 	if err != nil {

cmd/dendrite-demo-pinecone/ARCHITECTURE.md

@@ -0,0 +1,59 @@
+## Relay Server Architecture
+
+Relay Servers function similar to the way physical mail drop boxes do. 
+A node can have many associated relay servers. Matrix events can be sent to them instead of to the destination node, and the destination node will eventually retrieve them from the relay server. 
+Nodes that want to send events to an offline node need to know what relay servers are associated with their intended destination. 
+Currently this is manually configured in the dendrite database. In the future this information could be configurable in the app and shared automatically via other means.
+
+Currently events are sent as complete Matrix Transactions. 
+Transactions include a list of PDUs, (which contain, among other things, lists of authorization events, previous events, and signatures) a list of EDUs, and other information about the transaction. 
+There is no additional information sent along with the transaction other than what is typically added to them during Matrix federation today. 
+In the future this will probably need to change in order to handle more complex room state resolution during p2p usage.
+
+### Relay Server Architecture
+
+```
+                     0                            +--------------------+
+ +----------------------------------------+       |     P2P Node A     |
+ |             Relay Server               |       |     +--------+     |
+ |                                        |       |     | Client |     |
+ |                 +--------------------+ |       |     +--------+     |
+ |                 |  Relay Server API  | |       |          |         |
+ |                 |                    | |       |          V         |
+ |  .--------.   2 |   +-------------+  | |   1   |   +------------+   |
+ | |`--------`| <----- |  Forwarder  | <------------- | Homeserver |   |
+ | | Database |    |   +-------------+  | |       |   +------------+   |
+ | `----------`    |                    | |       +--------------------+
+ |       ^         |                    | |       
+ |       |     4   |   +-------------+  | |  
+ |       `------------ |  Retriever  | <------.   +--------------------+
+ |                 |   +-------------+  | |   |   |     P2P Node B     |
+ |                 |                    | |   |   |     +--------+     |
+ |                 +--------------------+ |   |   |     | Client |     |
+ |                                        |   |   |     +--------+     |
+ +----------------------------------------+   |   |          |         |
+                                              |   |          V         |
+                                            3 |   |   +------------+   |
+                                              `------ | Homeserver |   |
+                                                  |   +------------+   |
+                                                  +--------------------+
+```
+
+-   0: This relay server is currently only acting on behalf of `P2P Node B`. It will only receive, and later forward events that are destined for `P2P Node B`.
+-   1: When `P2P Node A` fails sending directly to `P2P Node B` (after a configurable number of attempts), it checks for any known relay servers associated with `P2P Node B` and sends to all of them.
+    - If sending to any of the relay servers succeeds, that transaction is considered to be successfully sent.      
+-   2: The relay server `forwarder` stores the transaction json in its database and marks it as destined for `P2P Node B`.
+-   3: When `P2P Node B` comes online, it queries all its relay servers for any missed messages.
+-   4: The relay server `retriever` will look in its database for any transactions that are destined for `P2P Node B` and returns them one at a time.
+
+For now, it is important that we don’t design out a hybrid approach of having both sender-side and recipient-side relay servers. 
+Both approaches make sense and determining which makes for a better experience depends on the use case.
+
+#### Sender-Side Relay Servers
+
+If we are running around truly ad-hoc, and I don't know when or where you will be able to pick up messages, then having a sender designated server makes sense to give things the best chance at making their way to the destination. 
+But in order to achieve this, you are either relying on p2p presence broadcasts for the relay to know when to try forwarding (which means you are in a pretty small network), or the relay just keeps on periodically attempting to forward to the destination which will lead to a lot of extra traffic on the network.
+
+#### Recipient-Side Relay Servers
+
+If we have agreed to some static relay server before going off and doing other things, or if we are talking about more global p2p federation, then having a recipient designated relay server can cut down on redundant traffic since it will sit there idle until the recipient pulls events from it.

cmd/dendrite-demo-pinecone/README.md

@@ -24,3 +24,42 @@ Then point your favourite Matrix client to  the homeserver URL`http://localhost:
 If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
 
 Once logged in, you should be able to open the room directory or join a room by its ID.
+
+## Store & Forward Relays
+
+To test out the store & forward relay functionality, you need a minimum of 3 instances. 
+One instance will act as the relay, and the other two instances will be the users trying to communicate.
+Then you can send messages between the two nodes and watch as the relay is used if the receiving node is offline.
+
+### Launching the Nodes
+
+Relay Server:
+```
+go run cmd/dendrite-demo-pinecone/main.go -dir relay/ -listen "[::]:49000"
+```
+
+Node 1:
+```
+go run cmd/dendrite-demo-pinecone/main.go -dir node-1/ -peer "[::]:49000" -port 8007
+```
+
+Node 2:
+```
+go run cmd/dendrite-demo-pinecone/main.go -dir node-2/ -peer "[::]:49000" -port 8009
+```
+
+### Database Setup
+
+At the moment, the database must be manually configured.
+For both `Node 1` and `Node 2` add the following entries to their respective `relay_server` table in the federationapi database:
+```
+server_name: {node_1_public_key}, relay_server_name: {relay_public_key}
+server_name: {node_2_public_key}, relay_server_name: {relay_public_key}
+```
+
+After editing the database you will need to relaunch the nodes for the changes to be picked up by dendrite.
+
+### Testing
+
+Now you can run two separate instances of element and connect them to `Node 1` and `Node 2`.
+You can shutdown one of the nodes and continue sending messages. If you wait long enough, the message will be sent to the relay server. (you can see this in the log output of the relay server) 

cmd/dendrite-demo-pinecone/main.go

@@ -38,16 +38,21 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/producers"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/relayapi"
+	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
+	"go.uber.org/atomic"
 
 	pineconeConnections "github.com/matrix-org/pinecone/connections"
 	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
@@ -66,6 +71,8 @@ var (
 	instanceDir    = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
 )
 
+const relayServerRetryInterval = time.Second * 30
+
 // nolint:gocyclo
 func main() {
 	flag.Parse()
@@ -139,6 +146,7 @@ func main() {
 		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
+		cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
 		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
 		cfg.ClientAPI.RegistrationDisabled = false
@@ -155,6 +163,7 @@ func main() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
 	defer base.Close() // nolint: errcheck
 
 	pineconeEventChannel := make(chan pineconeEvents.Event)
@@ -212,7 +221,7 @@ func main() {
 		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsComponent)
 	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
 	keyAPI.SetUserAPI(userAPI)
 
@@ -223,6 +232,20 @@ func main() {
 	userProvider := users.NewPineconeUserProvider(pRouter, pQUIC, userAPI, federation)
 	roomProvider := rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation)
 
+	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
+	producer := &producers.SyncAPIProducer{
+		JetStream:              js,
+		TopicReceiptEvent:      base.Cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicSendToDeviceEvent: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicTypingEvent:       base.Cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     base.Cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicDeviceListUpdate:  base.Cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
+		TopicSigningKeyUpdate:  base.Cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
+		Config:                 &base.Cfg.FederationAPI,
+		UserAPI:                userAPI,
+	}
+	relayAPI := relayapi.NewRelayInternalAPI(base, federation, rsAPI, keyRing, producer)
+
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
 		Client:    conn.CreateClient(base, pQUIC),
@@ -234,6 +257,7 @@ func main() {
 		RoomserverAPI:            rsAPI,
 		UserAPI:                  userAPI,
 		KeyAPI:                   keyAPI,
+		RelayAPI:                 relayAPI,
 		ExtPublicRoomsProvider:   roomProvider,
 		ExtUserDirectoryProvider: userProvider,
 	}
@@ -248,6 +272,8 @@ func main() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 	httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
 		c, err := wsUpgrader.Upgrade(w, r, nil)
 		if err != nil {
@@ -302,27 +328,38 @@ func main() {
 
 	go func(ch <-chan pineconeEvents.Event) {
 		eLog := logrus.WithField("pinecone", "events")
+		relayServerSyncRunning := atomic.NewBool(false)
+		stopRelayServerSync := make(chan bool)
+
+		m := RelayServerRetriever{
+			Context:             context.Background(),
+			ServerName:          gomatrixserverlib.ServerName(pRouter.PublicKey().String()),
+			FederationAPI:       fsAPI,
+			RelayServersQueried: make(map[gomatrixserverlib.ServerName]bool),
+			RelayAPI:            monolith.RelayAPI,
+		}
+		m.InitializeRelayServers(eLog)
 
 		for event := range ch {
 			switch e := event.(type) {
 			case pineconeEvents.PeerAdded:
+				if !relayServerSyncRunning.Load() {
+					go m.syncRelayServers(stopRelayServerSync, *relayServerSyncRunning)
+				}
 			case pineconeEvents.PeerRemoved:
-			case pineconeEvents.TreeParentUpdate:
-			case pineconeEvents.SnakeDescUpdate:
-			case pineconeEvents.TreeRootAnnUpdate:
-			case pineconeEvents.SnakeEntryAdded:
-			case pineconeEvents.SnakeEntryRemoved:
+				if relayServerSyncRunning.Load() && pRouter.TotalPeerCount() == 0 {
+					stopRelayServerSync <- true
+				}
 			case pineconeEvents.BroadcastReceived:
-				eLog.Info("Broadcast received from: ", e.PeerID)
+				// eLog.Info("Broadcast received from: ", e.PeerID)
 
 				req := &api.PerformWakeupServersRequest{
 					ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
 				}
 				res := &api.PerformWakeupServersResponse{}
 				if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
-					logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
+					eLog.WithError(err).Error("Failed to wakeup destination", e.PeerID)
 				}
-			case pineconeEvents.BandwidthReport:
 			default:
 			}
 		}
@@ -330,3 +367,78 @@ func main() {
 
 	base.WaitForShutdown()
 }
+
+type RelayServerRetriever struct {
+	Context             context.Context
+	ServerName          gomatrixserverlib.ServerName
+	FederationAPI       api.FederationInternalAPI
+	RelayServersQueried map[gomatrixserverlib.ServerName]bool
+	RelayAPI            relayServerAPI.RelayInternalAPI
+}
+
+func (m *RelayServerRetriever) InitializeRelayServers(eLog *logrus.Entry) {
+	request := api.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(m.ServerName)}
+	response := api.P2PQueryRelayServersResponse{}
+	err := m.FederationAPI.P2PQueryRelayServers(m.Context, &request, &response)
+	if err != nil {
+		eLog.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
+	}
+	for _, server := range response.RelayServers {
+		m.RelayServersQueried[server] = false
+	}
+
+	eLog.Infof("Registered relay servers: %v", response.RelayServers)
+}
+
+func (m *RelayServerRetriever) syncRelayServers(stop <-chan bool, running atomic.Bool) {
+	defer running.Store(false)
+
+	t := time.NewTimer(relayServerRetryInterval)
+	for {
+		relayServersToQuery := []gomatrixserverlib.ServerName{}
+		for server, complete := range m.RelayServersQueried {
+			if !complete {
+				relayServersToQuery = append(relayServersToQuery, server)
+			}
+		}
+		if len(relayServersToQuery) == 0 {
+			// All relay servers have been synced.
+			return
+		}
+		m.queryRelayServers(relayServersToQuery)
+		t.Reset(relayServerRetryInterval)
+
+		select {
+		case <-stop:
+			// We have been asked to stop syncing, drain the timer and return.
+			if !t.Stop() {
+				<-t.C
+			}
+			return
+		case <-t.C:
+			// The timer has expired. Continue to the next loop iteration.
+		}
+	}
+}
+
+func (m *RelayServerRetriever) queryRelayServers(relayServers []gomatrixserverlib.ServerName) {
+	logrus.Info("querying relay servers for any available transactions")
+	for _, server := range relayServers {
+		userID, err := gomatrixserverlib.NewUserID("@user:"+string(m.ServerName), false)
+		if err != nil {
+			return
+		}
+		err = m.RelayAPI.PerformRelayServerSync(context.Background(), *userID, server)
+		if err == nil {
+			m.RelayServersQueried[server] = true
+			// TODO : What happens if your relay receives new messages after this point?
+			// Should you continue to check with them, or should they try and contact you?
+			// They could send a "new_async_events" message your way maybe?
+			// Then you could mark them as needing to be queried again.
+			// What if you miss this message?
+			// Maybe you should try querying them again after a certain period of time as a backup?
+		} else {
+			logrus.Errorf("Failed querying relay server: %s", err.Error())
+		}
+	}
+}

cmd/dendrite-demo-yggdrasil/main.go

@@ -144,6 +144,7 @@ func main() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
+	base.ConfigureAdminEndpoints()
 	defer base.Close() // nolint: errcheck
 
 	ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
@@ -156,11 +157,12 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
-
 	rsComponent := roomserver.NewInternalAPI(
 		base,
 	)
+
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsComponent)
+
 	rsAPI := rsComponent
 
 	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
@@ -198,6 +200,8 @@ func main() {
 	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
+	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
 	embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
 
 	yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()

cmd/dendrite-monolith-server/main.go

@@ -18,6 +18,8 @@ import (
 	"flag"
 	"os"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/keyserver"
@@ -29,7 +31,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/userapi"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -75,7 +76,7 @@ func main() {
 	// call functions directly on the impl unless running in HTTP mode
 	rsAPI := rsImpl
 	if base.UseHTTPAPIs {
-		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl)
+		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics)
 		rsAPI = base.RoomserverHTTPClient()
 	}
 	if traceInternal {
@@ -89,15 +90,15 @@ func main() {
 	)
 	fsImplAPI := fsAPI
 	if base.UseHTTPAPIs {
-		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
+		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
 		fsAPI = base.FederationAPIHTTPClient()
 	}
 	keyRing := fsAPI.KeyRing()
 
-	keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
 	keyAPI := keyImpl
 	if base.UseHTTPAPIs {
-		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics)
 		keyAPI = base.KeyServerHTTPClient()
 	}
 
@@ -105,7 +106,7 @@ func main() {
 	userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
 	userAPI := userImpl
 	if base.UseHTTPAPIs {
-		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
 		userAPI = base.UserAPIClient()
 	}
 	if traceInternal {
@@ -119,7 +120,7 @@ func main() {
 	// before the listeners are up.
 	asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
 	if base.UseHTTPAPIs {
-		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
+		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics)
 		asAPI = base.AppserviceHTTPClient()
 	}
 

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -26,7 +26,7 @@ func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) {
 	rsAPI := base.RoomserverHTTPClient()
 
 	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
+	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -34,7 +34,7 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 		rsAPI, fsAPI, keyAPI, nil,
 	)
 
-	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
+	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.FederationAPI.InternalAPI.Listen,

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -22,10 +22,11 @@ import (
 
 func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	fsAPI := base.FederationAPIHTTPClient()
-	intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	rsAPI := base.RoomserverHTTPClient()
+	intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
-	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
+	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.KeyServer.InternalAPI.Listen, // internal listener

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -26,7 +26,7 @@ func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	rsAPI := roomserver.NewInternalAPI(base)
 	rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
 	rsAPI.SetAppserviceAPI(asAPI)
-	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
+	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.RoomServer.InternalAPI.Listen, // internal listener

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -27,7 +27,7 @@ func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 		base.PushGatewayHTTPClient(),
 	)
 
-	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.UserAPI.InternalAPI.Listen, // internal listener

cmd/dendrite-upgrade-tests/main.go

@@ -7,6 +7,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -61,6 +62,7 @@ COPY . .
 RUN go build ./cmd/dendrite-monolith-server
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
+RUN go build ./cmd/create-account
 RUN ./generate-config --ci > dendrite.yaml
 RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 
@@ -104,6 +106,7 @@ COPY . .
 RUN go build ./cmd/dendrite-monolith-server
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
+RUN go build ./cmd/create-account
 RUN ./generate-config --ci > dendrite.yaml
 RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
 
@@ -458,6 +461,46 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
 	if err = runTests(csAPIURL, v); err != nil {
 		return fmt.Errorf("failed to run tests on version %s: %s", v, err)
 	}
+
+	err = testCreateAccount(dockerClient, v, containerID)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// test that create-account is working
+func testCreateAccount(dockerClient *client.Client, v string, containerID string) error {
+	createUser := strings.ToLower("createaccountuser-" + v)
+	log.Printf("%s: Creating account %s with create-account\n", v, createUser)
+
+	respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
+		AttachStderr: true,
+		AttachStdout: true,
+		Cmd: []string{
+			"/build/create-account",
+			"-username", createUser,
+			"-password", "someRandomPassword",
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to ContainerExecCreate: %w", err)
+	}
+
+	response, err := dockerClient.ContainerExecAttach(context.Background(), respID.ID, types.ExecStartCheck{})
+	if err != nil {
+		return fmt.Errorf("failed to attach to container: %w", err)
+	}
+	defer response.Close()
+
+	data, err := ioutil.ReadAll(response.Reader)
+	if err != nil {
+		return err
+	}
+
+	if !bytes.Contains(data, []byte("AccessToken")) {
+		return fmt.Errorf("failed to create-account: %s", string(data))
+	}
 	return nil
 }
 

cmd/generate-config/main.go

@@ -54,6 +54,9 @@ func main() {
 		} else {
 			cfg.Global.DatabaseOptions.ConnectionString = uri
 		}
+		cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
+		cfg.Global.JetStream.StoragePath = config.Path(*dirPath)
+		cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(*dirPath, "searchindex"))
 		cfg.Logging = []config.LogrusHook{
 			{
 				Type:  "file",
@@ -67,6 +70,7 @@ func main() {
 			cfg.AppServiceAPI.DisableTLSValidation = true
 			cfg.ClientAPI.RateLimiting.Enabled = false
 			cfg.FederationAPI.DisableTLSValidation = false
+			cfg.FederationAPI.DisableHTTPKeepalives = true
 			// don't hit matrix.org when running tests!!!
 			cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
 			cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))

docs/FAQ.md

@@ -6,6 +6,12 @@ permalink: /faq
 
 # FAQ
 
+## Why does Dendrite exist?
+
+Dendrite aims to provide a matrix compatible server that has low resource usage compared to [Synapse](https://github.com/matrix-org/synapse).
+It also aims to provide more flexibility when scaling either up or down.
+Dendrite's code is also very easy to hack on which makes it suitable for experimenting with new matrix features such as peer-to-peer.
+
 ## Is Dendrite stable?
 
 Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead.
@@ -34,6 +40,10 @@ No, Dendrite has a very different database schema to Synapse and the two are not
 Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't
 recommend.
 
+## Can I configure which port Dendrite listens on?
+
+Yes, use the cli flag `-http-bind-address`. 
+
 ## I've installed Dendrite but federation isn't working
 
 Check the [Federation Tester](https://federationtester.matrix.org). You need at least:
@@ -42,6 +52,10 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at
 * A valid TLS certificate for that DNS name
 * Either DNS SRV records or well-known files
 
+## Whenever I try to connect from Element it says unable to connect to homeserver
+
+Check that your dendrite instance is running. Otherwise this is most likely due to a reverse proxy misconfiguration.
+
 ## Does Dendrite work with my favourite client?
 
 It should do, although we are aware of some minor issues:
@@ -49,6 +63,10 @@ It should do, although we are aware of some minor issues:
 * **Element Android**: registration does not work, but logging in with an existing account does
 * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
 
+## Is there a public instance of Dendrite I can try out?
+
+Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially support.
+
 ## Does Dendrite support Space Summaries?
 
 Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
@@ -84,14 +102,46 @@ Remember to add the config file(s) to the `app_service_api` section of the confi
 
 Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file.
 
+## How can I migrate a room in order to change the internal ID?
+
+This can be done by performing a room upgrade. Use the command `/upgraderoom <version>` in Element to do this.
+
+## How do I reset somebody's password on my server?
+
+Use the admin endpoint [resetpassword](https://matrix-org.github.io/dendrite/administration/adminapi#post-_dendriteadminresetpassworduserid)
+
 ## Should I use PostgreSQL or SQLite for my databases?
 
 Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users.
 
+## What data needs to be kept if transferring/backing up Dendrite?
+
+The list of files that need to be stored is:
+- matrix-key.pem
+- dendrite.yaml
+- the postgres or sqlite DB
+- the media store
+- the search index (although this can be regenerated)
+
+Note that this list may change / be out of date. We don't officially maintain instructions for migrations like this.
+
+## How can I prepare enough storage for media caches?
+
+This might be what you want: [matrix-media-repo](https://github.com/turt2live/matrix-media-repo)
+We don't officially support this or any other dedicated media storage solutions.
+
+## Is there an upgrade guide for Dendrite?
+
+Run a newer docker image. We don't officially support deployments other than Docker. 
+Most of the time you should be able to just 
+- stop 
+- replace binary
+- start
+
 ## Dendrite is using a lot of CPU
 
 Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the
-CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](PROFILING.md) then that would
+CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](development/PROFILING.md) then that would
 be a huge help too, as that will help us to understand where the CPU time is going.
 
 ## Dendrite is using a lot of RAM
@@ -99,9 +149,13 @@ be a huge help too, as that will help us to understand where the CPU time is goi
 As above with CPU usage, some memory spikes are expected if Dendrite is doing particularly heavy work
 at a given instant. However, if it is using more RAM than you expect for a long time, that's probably
 not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doing when the memory usage
-ballooned, or file a GitHub issue if you can. If you can take a [memory profile](PROFILING.md) then that
+ballooned, or file a GitHub issue if you can. If you can take a [memory profile](development/PROFILING.md) then that
 would be a huge help too, as that will help us to understand where the memory usage is happening.
 
+## Do I need to generate the self-signed certificate if I'm going to use a reverse proxy?
+
+No, if you already have a proper certificate from some provider, like Let's Encrypt, and use that on your reverse proxy, and the reverse proxy does TLS termination, then you’re good and can use HTTP to the dendrite process.
+
 ## Dendrite is running out of PostgreSQL database connections
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!

docs/Gemfile.lock

@@ -14,7 +14,7 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.6)
+    commonmarker (0.23.7)
     concurrent-ruby (1.1.10)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)
@@ -231,9 +231,9 @@ GEM
       jekyll-seo-tag (~> 2.1)
     minitest (5.15.0)
     multipart-post (2.1.1)
-    nokogiri (1.13.9-arm64-darwin)
+    nokogiri (1.13.10-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.13.9-x86_64-linux)
+    nokogiri (1.13.10-x86_64-linux)
       racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
@@ -241,7 +241,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
-    racc (1.6.0)
+    racc (1.6.1)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)

docs/administration/4_adminapi.md

@@ -44,7 +44,9 @@ This endpoint will instruct Dendrite to part the given local `userID` in the URL
 all rooms which they are currently joined. A JSON body will be returned containing
 the room IDs of all affected rooms.
 
-## POST `/_dendrite/admin/resetPassword/{localpart}`
+## POST `/_dendrite/admin/resetPassword/{userID}`
+
+Reset the password of a local user.
 
 Request body format:
 
@@ -54,9 +56,6 @@ Request body format:
 }
 ```
 
-Reset the password of a local user. The `localpart` is the username only, i.e. if
-the full user ID is `@alice:domain.com` then the local part is `alice`.
-
 ## GET `/_dendrite/admin/fulltext/reindex`
 
 This endpoint instructs Dendrite to reindex all searchable events (`m.room.message`, `m.room.topic` and `m.room.name`). An empty JSON body will be returned immediately.

docs/development/CONTRIBUTING.md

@@ -9,6 +9,28 @@ permalink: /development/contributing
 Everyone is welcome to contribute to Dendrite! We aim to make it as easy as
 possible to get started.
 
+## Contribution types
+
+We are a small team maintaining a large project. As a result, we cannot merge every feature, even if it
+is bug-free and useful, because we then commit to maintaining it indefinitely. We will always accept:
+ - bug fixes
+ - security fixes (please responsibly disclose via security@matrix.org *before* creating pull requests)
+
+We will accept the following with caveats:
+ - documentation fixes, provided they do not add additional instructions which can end up going out-of-date,
+   e.g example configs, shell commands.
+ - performance fixes, provided they do not add significantly more maintenance burden.
+ - additional functionality on existing features, provided the functionality is small and maintainable.
+ - additional functionality that, in its absence, would impact the ecosystem e.g spam and abuse mitigations
+ - test-only changes, provided they help improve coverage or test tricky code.
+
+The following items are at risk of not being accepted:
+ - Configuration or CLI changes, particularly ones which increase the overall configuration surface.
+
+The following items are unlikely to be accepted into a main Dendrite release for now:
+ - New MSC implementations.
+ - New features which are not in the specification.
+
 ## Sign off
 
 We require that everyone who contributes to the project signs off their contributions
@@ -35,7 +57,7 @@ to do so for future contributions.
 
 ## Getting up and running
 
-See the [Installation](installation) section for information on how to build an
+See the [Installation](../installation) section for information on how to build an
 instance of Dendrite. You will likely need this in order to test your changes.
 
 ## Code style
@@ -75,7 +97,20 @@ comment. Please avoid doing this if you can.
 We also have unit tests which we run via:
 
 ```bash
-go test --race ./...
+DENDRITE_TEST_SKIP_NODB=1 go test --race ./...
+```
+
+This only runs SQLite database tests. If you wish to execute Postgres tests as well, you'll either need to 
+have Postgres installed locally (`createdb` will be used) or have a remote/containerized Postgres instance
+available.
+
+To configure the connection to a remote Postgres, you can use the following enviroment variables:
+
+```bash
+POSTGRES_USER=postgres
+POSTGERS_PASSWORD=yourPostgresPassword
+POSTGRES_HOST=localhost
+POSTGRES_DB=postgres # the superuser database to use
 ```
 
 In general, we like submissions that come with tests. Anything that proves that the
@@ -116,7 +151,7 @@ significant amount of CPU and RAM.
 
 Once the code builds, run [Sytest](https://github.com/matrix-org/sytest)
 according to the guide in
-[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/main/docs/sytest.md#using-a-sytest-docker-image)
+[docs/development/sytest.md](https://github.com/matrix-org/dendrite/blob/main/docs/development/sytest.md#using-a-sytest-docker-image)
 so you can see whether something is being broken and whether there are newly
 passing tests.
 

federationapi/api/api.go

@@ -18,6 +18,7 @@ type FederationInternalAPI interface {
 	gomatrixserverlib.KeyDatabase
 	ClientFederationAPI
 	RoomserverFederationAPI
+	P2PFederationAPI
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
 	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
@@ -30,7 +31,6 @@ type FederationInternalAPI interface {
 		request *PerformBroadcastEDURequest,
 		response *PerformBroadcastEDUResponse,
 	) error
-
 	PerformWakeupServers(
 		ctx context.Context,
 		request *PerformWakeupServersRequest,
@@ -71,6 +71,15 @@ type RoomserverFederationAPI interface {
 	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
+type P2PFederationAPI interface {
+	// Relay Server sync api used in the pinecone demos.
+	P2PQueryRelayServers(
+		ctx context.Context,
+		request *P2PQueryRelayServersRequest,
+		response *P2PQueryRelayServersResponse,
+	) error
+}
+
 // KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver
 // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
 // this interface are of type FederationClientError
@@ -82,6 +91,7 @@ type KeyserverFederationAPI interface {
 
 // an interface for gmsl.FederationClient - contains functions called by federationapi only.
 type FederationClient interface {
+	P2PFederationClient
 	gomatrixserverlib.KeyClient
 	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
 
@@ -110,6 +120,11 @@ type FederationClient interface {
 	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
+type P2PFederationClient interface {
+	P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error)
+	P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev gomatrixserverlib.RelayEntry, relayServer gomatrixserverlib.ServerName) (res gomatrixserverlib.RespGetRelayTransaction, err error)
+}
+
 // FederationClientError is returned from FederationClient methods in the event of a problem.
 type FederationClientError struct {
 	Err         string
@@ -233,3 +248,11 @@ type InputPublicKeysRequest struct {
 
 type InputPublicKeysResponse struct {
 }
+
+type P2PQueryRelayServersRequest struct {
+	Server gomatrixserverlib.ServerName
+}
+
+type P2PQueryRelayServersResponse struct {
+	RelayServers []gomatrixserverlib.ServerName
+}

federationapi/consumers/roomserver.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/federationapi/queue"
@@ -90,8 +91,10 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
 
-	// Only handle events we care about
-	if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek {
+	// Only handle events we care about, avoids unneeded unmarshalling
+	switch receivedType {
+	case api.OutputTypeNewRoomEvent, api.OutputTypeNewInboundPeek, api.OutputTypePurgeRoom:
+	default:
 		return true
 	}
 
@@ -126,6 +129,14 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
 			return false
 		}
 
+	case api.OutputTypePurgeRoom:
+		log.WithField("room_id", output.PurgeRoom.RoomID).Warn("Purging room from federation API")
+		if err := s.db.PurgeRoom(ctx, output.PurgeRoom.RoomID); err != nil {
+			logrus.WithField("room_id", output.PurgeRoom.RoomID).WithError(err).Error("Failed to purge room from federation API")
+		} else {
+			logrus.WithField("room_id", output.PurgeRoom.RoomID).Warn("Room purged from federation API")
+		}
+
 	default:
 		log.WithField("type", output.Type).Debug(
 			"roomserver output log: ignoring unknown output type",
@@ -195,7 +206,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	}
 
 	// If we added new hosts, inform them about our known presence events for this room
-	if len(addsJoinedHosts) > 0 && ore.Event.Type() == gomatrixserverlib.MRoomMember && ore.Event.StateKey() != nil {
+	if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == gomatrixserverlib.MRoomMember && ore.Event.StateKey() != nil {
 		membership, _ := ore.Event.Membership()
 		if membership == gomatrixserverlib.Join {
 			s.sendPresence(ore.Event.RoomID(), addsJoinedHosts)
@@ -232,7 +243,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 }
 
 func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []types.JoinedHost) {
-	joined := make([]gomatrixserverlib.ServerName, len(addedJoined))
+	joined := make([]gomatrixserverlib.ServerName, 0, len(addedJoined))
 	for _, added := range addedJoined {
 		joined = append(joined, added.ServerName)
 	}

federationapi/federationapi.go

@@ -43,8 +43,8 @@ import (
 
 // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
 // on the given input API.
-func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) {
-	inthttp.AddRoutes(intAPI, router)
+func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI, enableMetrics bool) {
+	inthttp.AddRoutes(intAPI, router, enableMetrics)
 }
 
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
@@ -85,10 +85,7 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		cfg,
+		base,
 		rsAPI, f, keyRing,
 		federation, userAPI, keyAPI, mscCfg,
 		servers, producer,
@@ -116,7 +113,10 @@ func NewInternalAPI(
 		_ = federationDB.RemoveAllServersFromBlacklist()
 	}
 
-	stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1)
+	stats := statistics.NewStatistics(
+		federationDB,
+		cfg.FederationMaxRetries+1,
+		cfg.P2PFederationRetriesUntilAssumedOffline+1)
 
 	js, nats := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 

federationapi/federationapi_test.go

@@ -273,12 +273,12 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
 	cfg.Global.PrivateKey = privKey
 	cfg.Global.JetStream.InMemory = true
-	base := base.NewBaseDendrite(cfg, "Monolith")
+	b := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
 	keyRing := &test.NopJSONVerifier{}
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, nil)
-	baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true)
+	federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, nil)
+	baseURL, cancel := test.ListenAndServe(t, b.PublicFederationAPIMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
 

federationapi/internal/api.go

@@ -109,13 +109,14 @@ func NewFederationInternalAPI(
 
 func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s gomatrixserverlib.ServerName) (*statistics.ServerStatistics, error) {
 	stats := a.statistics.ForServer(s)
-	until, blacklisted := stats.BackoffInfo()
-	if blacklisted {
+	if stats.Blacklisted() {
 		return stats, &api.FederationClientError{
 			Blacklisted: true,
 		}
 	}
+
 	now := time.Now()
+	until := stats.BackoffInfo()
 	if until != nil && now.Before(*until) {
 		return stats, &api.FederationClientError{
 			RetryAfter: time.Until(*until),
@@ -163,7 +164,7 @@ func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
 			RetryAfter:  retryAfter,
 		}
 	}
-	stats.Success()
+	stats.Success(statistics.SendDirect)
 	return res, nil
 }
 
@@ -171,7 +172,7 @@ func (a *FederationInternalAPI) doRequestIfNotBlacklisted(
 	s gomatrixserverlib.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
 	stats := a.statistics.ForServer(s)
-	if _, blacklisted := stats.BackoffInfo(); blacklisted {
+	if blacklisted := stats.Blacklisted(); blacklisted {
 		return stats, &api.FederationClientError{
 			Err:         fmt.Sprintf("server %q is blacklisted", s),
 			Blacklisted: true,

federationapi/internal/federationclient_test.go

@@ -0,0 +1,202 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	FailuresUntilAssumedOffline = 3
+	FailuresUntilBlacklist      = 8
+)
+
+func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) {
+	t.queryKeysCalled = true
+	if t.shouldFail {
+		return gomatrixserverlib.RespQueryKeys{}, fmt.Errorf("Failure")
+	}
+	return gomatrixserverlib.RespQueryKeys{}, nil
+}
+
+func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) {
+	t.claimKeysCalled = true
+	if t.shouldFail {
+		return gomatrixserverlib.RespClaimKeys{}, fmt.Errorf("Failure")
+	}
+	return gomatrixserverlib.RespClaimKeys{}, nil
+}
+
+func TestFederationClientQueryKeys(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.Nil(t, err)
+	assert.True(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+	testDB.AddServerToBlacklist("server")
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.False(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientQueryKeysFailure(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{shouldFail: true}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.True(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientClaimKeys(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.ClaimKeys(context.Background(), "origin", "server", nil)
+	assert.Nil(t, err)
+	assert.True(t, fedClient.claimKeysCalled)
+}
+
+func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+	testDB.AddServerToBlacklist("server")
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.ClaimKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.False(t, fedClient.claimKeysCalled)
+}

federationapi/internal/perform.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 )
@@ -24,6 +25,10 @@ func (r *FederationInternalAPI) PerformDirectoryLookup(
 	request *api.PerformDirectoryLookupRequest,
 	response *api.PerformDirectoryLookupResponse,
 ) (err error) {
+	if !r.shouldAttemptDirectFederation(request.ServerName) {
+		return fmt.Errorf("relay servers have no meaningful response for directory lookup.")
+	}
+
 	dir, err := r.federation.LookupRoomAlias(
 		ctx,
 		r.cfg.Matrix.ServerName,
@@ -36,7 +41,7 @@ func (r *FederationInternalAPI) PerformDirectoryLookup(
 	}
 	response.RoomID = dir.RoomID
 	response.ServerNames = dir.Servers
-	r.statistics.ForServer(request.ServerName).Success()
+	r.statistics.ForServer(request.ServerName).Success(statistics.SendDirect)
 	return nil
 }
 
@@ -144,6 +149,10 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	supportedVersions []gomatrixserverlib.RoomVersion,
 	unsigned map[string]interface{},
 ) error {
+	if !r.shouldAttemptDirectFederation(serverName) {
+		return fmt.Errorf("relay servers have no meaningful response for join.")
+	}
+
 	_, origin, err := r.cfg.Matrix.SplitLocalID('@', userID)
 	if err != nil {
 		return err
@@ -164,7 +173,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.MakeJoin: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// Set all the fields to be what they should be, this should be a no-op
 	// but it's possible that the remote server returned us something "odd"
@@ -219,7 +228,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.SendJoin: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// If the remote server returned an event in the "event" key of
 	// the send_join request then we should use that instead. It may
@@ -407,6 +416,10 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 	serverName gomatrixserverlib.ServerName,
 	supportedVersions []gomatrixserverlib.RoomVersion,
 ) error {
+	if !r.shouldAttemptDirectFederation(serverName) {
+		return fmt.Errorf("relay servers have no meaningful response for outbound peek.")
+	}
+
 	// create a unique ID for this peek.
 	// for now we just use the room ID again. In future, if we ever
 	// support concurrent peeks to the same room with different filters
@@ -446,7 +459,7 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.Peek: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// Work out if we support the room version that has been supplied in
 	// the peek response.
@@ -516,6 +529,10 @@ func (r *FederationInternalAPI) PerformLeave(
 	// Try each server that we were provided until we land on one that
 	// successfully completes the make-leave send-leave dance.
 	for _, serverName := range request.ServerNames {
+		if !r.shouldAttemptDirectFederation(serverName) {
+			continue
+		}
+
 		// Try to perform a make_leave using the information supplied in the
 		// request.
 		respMakeLeave, err := r.federation.MakeLeave(
@@ -585,7 +602,7 @@ func (r *FederationInternalAPI) PerformLeave(
 			continue
 		}
 
-		r.statistics.ForServer(serverName).Success()
+		r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 		return nil
 	}
 
@@ -616,6 +633,12 @@ func (r *FederationInternalAPI) PerformInvite(
 		return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 	}
 
+	// TODO (devon): This should be allowed via a relay. Currently only transactions
+	// can be sent to relays. Would need to extend relays to handle invites.
+	if !r.shouldAttemptDirectFederation(destination) {
+		return fmt.Errorf("relay servers have no meaningful response for invite.")
+	}
+
 	logrus.WithFields(logrus.Fields{
 		"event_id":     request.Event.EventID(),
 		"user_id":      *request.Event.StateKey(),
@@ -682,12 +705,8 @@ func (r *FederationInternalAPI) PerformWakeupServers(
 
 func (r *FederationInternalAPI) MarkServersAlive(destinations []gomatrixserverlib.ServerName) {
 	for _, srv := range destinations {
-		// Check the statistics cache for the blacklist status to prevent hitting
-		// the database unnecessarily.
-		if r.queues.IsServerBlacklisted(srv) {
-			_ = r.db.RemoveServerFromBlacklist(srv)
-		}
-		r.queues.RetryServer(srv)
+		wasBlacklisted := r.statistics.ForServer(srv).MarkServerAlive()
+		r.queues.RetryServer(srv, wasBlacklisted)
 	}
 }
 
@@ -719,7 +738,9 @@ func sanityCheckAuthChain(authChain []*gomatrixserverlib.Event) error {
 	return fmt.Errorf("auth chain response is missing m.room.create event")
 }
 
-func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder) gomatrixserverlib.RoomVersion {
+func setDefaultRoomVersionFromJoinEvent(
+	joinEvent gomatrixserverlib.EventBuilder,
+) gomatrixserverlib.RoomVersion {
 	// if auth events are not event references we know it must be v3+
 	// we have to do these shenanigans to satisfy sytest, specifically for:
 	// "Outbound federation rejects m.room.create events with an unknown room version"
@@ -802,3 +823,31 @@ func federatedAuthProvider(
 		return returning, nil
 	}
 }
+
+// P2PQueryRelayServers implements api.FederationInternalAPI
+func (r *FederationInternalAPI) P2PQueryRelayServers(
+	ctx context.Context,
+	request *api.P2PQueryRelayServersRequest,
+	response *api.P2PQueryRelayServersResponse,
+) error {
+	logrus.Infof("Getting relay servers for: %s", request.Server)
+	relayServers, err := r.db.P2PGetRelayServersForServer(ctx, request.Server)
+	if err != nil {
+		return err
+	}
+
+	response.RelayServers = relayServers
+	return nil
+}
+
+func (r *FederationInternalAPI) shouldAttemptDirectFederation(
+	destination gomatrixserverlib.ServerName,
+) bool {
+	var shouldRelay bool
+	stats := r.statistics.ForServer(destination)
+	if stats.AssumedOffline() && len(stats.KnownRelayServers()) > 0 {
+		shouldRelay = true
+	}
+
+	return !shouldRelay
+}

federationapi/internal/perform_test.go

@@ -0,0 +1,190 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+type testFedClient struct {
+	api.FederationClient
+	queryKeysCalled bool
+	claimKeysCalled bool
+	shouldFail      bool
+}
+
+func (t *testFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) {
+	return gomatrixserverlib.RespDirectory{}, nil
+}
+
+func TestPerformWakeupServers(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	testDB.AddServerToBlacklist(server)
+	testDB.SetServerAssumedOffline(context.Background(), server)
+	blacklisted, err := testDB.IsServerBlacklisted(server)
+	assert.NoError(t, err)
+	assert.True(t, blacklisted)
+	offline, err := testDB.IsServerAssumedOffline(context.Background(), server)
+	assert.NoError(t, err)
+	assert.True(t, offline)
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformWakeupServersRequest{
+		ServerNames: []gomatrixserverlib.ServerName{server},
+	}
+	res := api.PerformWakeupServersResponse{}
+	err = fedAPI.PerformWakeupServers(context.Background(), &req, &res)
+	assert.NoError(t, err)
+
+	blacklisted, err = testDB.IsServerBlacklisted(server)
+	assert.NoError(t, err)
+	assert.False(t, blacklisted)
+	offline, err = testDB.IsServerAssumedOffline(context.Background(), server)
+	assert.NoError(t, err)
+	assert.False(t, offline)
+}
+
+func TestQueryRelayServers(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	relayServers := []gomatrixserverlib.ServerName{"relay1", "relay2"}
+	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
+	assert.NoError(t, err)
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.P2PQueryRelayServersRequest{
+		Server: server,
+	}
+	res := api.P2PQueryRelayServersResponse{}
+	err = fedAPI.P2PQueryRelayServers(context.Background(), &req, &res)
+	assert.NoError(t, err)
+
+	assert.Equal(t, len(relayServers), len(res.RelayServers))
+}
+
+func TestPerformDirectoryLookup(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformDirectoryLookupRequest{
+		RoomAlias:  "room",
+		ServerName: "server",
+	}
+	res := api.PerformDirectoryLookupResponse{}
+	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	assert.NoError(t, err)
+}
+
+func TestPerformDirectoryLookupRelaying(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	testDB.SetServerAssumedOffline(context.Background(), server)
+	testDB.P2PAddRelayServersForServer(context.Background(), server, []gomatrixserverlib.ServerName{"relay"})
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: server,
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformDirectoryLookupRequest{
+		RoomAlias:  "room",
+		ServerName: server,
+	}
+	res := api.PerformDirectoryLookupResponse{}
+	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	assert.Error(t, err)
+}

federationapi/inthttp/client.go

@@ -24,6 +24,7 @@ const (
 	FederationAPIPerformOutboundPeekRequestPath    = "/federationapi/performOutboundPeekRequest"
 	FederationAPIPerformBroadcastEDUPath           = "/federationapi/performBroadcastEDU"
 	FederationAPIPerformWakeupServers              = "/federationapi/performWakeupServers"
+	FederationAPIQueryRelayServers                 = "/federationapi/queryRelayServers"
 
 	FederationAPIGetUserDevicesPath      = "/federationapi/client/getUserDevices"
 	FederationAPIClaimKeysPath           = "/federationapi/client/claimKeys"
@@ -510,3 +511,14 @@ func (h *httpFederationInternalAPI) QueryPublicKeys(
 		h.httpClient, ctx, request, response,
 	)
 }
+
+func (h *httpFederationInternalAPI) P2PQueryRelayServers(
+	ctx context.Context,
+	request *api.P2PQueryRelayServersRequest,
+	response *api.P2PQueryRelayServersResponse,
+) error {
+	return httputil.CallInternalRPCAPI(
+		"QueryRelayServers", h.federationAPIURL+FederationAPIQueryRelayServers,
+		h.httpClient, ctx, request, response,
+	)
+}

federationapi/inthttp/server.go

@@ -17,41 +17,41 @@ import (
 
 // AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux.
 // nolint:gocyclo
-func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
+func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
 	internalAPIMux.Handle(
 		FederationAPIQueryJoinedHostServerNamesInRoomPath,
-		httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", intAPI.QueryJoinedHostServerNamesInRoom),
+		httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", enableMetrics, intAPI.QueryJoinedHostServerNamesInRoom),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformInviteRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", intAPI.PerformInvite),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", enableMetrics, intAPI.PerformInvite),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformLeaveRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", intAPI.PerformLeave),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", enableMetrics, intAPI.PerformLeave),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformDirectoryLookupRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", intAPI.PerformDirectoryLookup),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", enableMetrics, intAPI.PerformDirectoryLookup),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformBroadcastEDUPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", intAPI.PerformBroadcastEDU),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", enableMetrics, intAPI.PerformBroadcastEDU),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformWakeupServers,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", intAPI.PerformWakeupServers),
+		httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", enableMetrics, intAPI.PerformWakeupServers),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPIPerformJoinRequestPath,
 		httputil.MakeInternalRPCAPI(
-			"FederationAPIPerformJoinRequest",
+			"FederationAPIPerformJoinRequest", enableMetrics,
 			func(ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse) error {
 				intAPI.PerformJoin(ctx, req, res)
 				return nil
@@ -62,7 +62,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIGetUserDevicesPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetUserDevices",
+			"FederationAPIGetUserDevices", enableMetrics,
 			func(ctx context.Context, req *getUserDevices) (*gomatrixserverlib.RespUserDevices, error) {
 				res, err := intAPI.GetUserDevices(ctx, req.Origin, req.S, req.UserID)
 				return &res, federationClientError(err)
@@ -73,7 +73,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIClaimKeysPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIClaimKeys",
+			"FederationAPIClaimKeys", enableMetrics,
 			func(ctx context.Context, req *claimKeys) (*gomatrixserverlib.RespClaimKeys, error) {
 				res, err := intAPI.ClaimKeys(ctx, req.Origin, req.S, req.OneTimeKeys)
 				return &res, federationClientError(err)
@@ -84,7 +84,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIQueryKeysPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIQueryKeys",
+			"FederationAPIQueryKeys", enableMetrics,
 			func(ctx context.Context, req *queryKeys) (*gomatrixserverlib.RespQueryKeys, error) {
 				res, err := intAPI.QueryKeys(ctx, req.Origin, req.S, req.Keys)
 				return &res, federationClientError(err)
@@ -95,7 +95,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIBackfillPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIBackfill",
+			"FederationAPIBackfill", enableMetrics,
 			func(ctx context.Context, req *backfill) (*gomatrixserverlib.Transaction, error) {
 				res, err := intAPI.Backfill(ctx, req.Origin, req.S, req.RoomID, req.Limit, req.EventIDs)
 				return &res, federationClientError(err)
@@ -106,7 +106,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPILookupStatePath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupState",
+			"FederationAPILookupState", enableMetrics,
 			func(ctx context.Context, req *lookupState) (*gomatrixserverlib.RespState, error) {
 				res, err := intAPI.LookupState(ctx, req.Origin, req.S, req.RoomID, req.EventID, req.RoomVersion)
 				return &res, federationClientError(err)
@@ -117,7 +117,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPILookupStateIDsPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupStateIDs",
+			"FederationAPILookupStateIDs", enableMetrics,
 			func(ctx context.Context, req *lookupStateIDs) (*gomatrixserverlib.RespStateIDs, error) {
 				res, err := intAPI.LookupStateIDs(ctx, req.Origin, req.S, req.RoomID, req.EventID)
 				return &res, federationClientError(err)
@@ -128,7 +128,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPILookupMissingEventsPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupMissingEvents",
+			"FederationAPILookupMissingEvents", enableMetrics,
 			func(ctx context.Context, req *lookupMissingEvents) (*gomatrixserverlib.RespMissingEvents, error) {
 				res, err := intAPI.LookupMissingEvents(ctx, req.Origin, req.S, req.RoomID, req.Missing, req.RoomVersion)
 				return &res, federationClientError(err)
@@ -139,7 +139,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIGetEventPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetEvent",
+			"FederationAPIGetEvent", enableMetrics,
 			func(ctx context.Context, req *getEvent) (*gomatrixserverlib.Transaction, error) {
 				res, err := intAPI.GetEvent(ctx, req.Origin, req.S, req.EventID)
 				return &res, federationClientError(err)
@@ -150,7 +150,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIGetEventAuthPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetEventAuth",
+			"FederationAPIGetEventAuth", enableMetrics,
 			func(ctx context.Context, req *getEventAuth) (*gomatrixserverlib.RespEventAuth, error) {
 				res, err := intAPI.GetEventAuth(ctx, req.Origin, req.S, req.RoomVersion, req.RoomID, req.EventID)
 				return &res, federationClientError(err)
@@ -160,13 +160,13 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 
 	internalAPIMux.Handle(
 		FederationAPIQueryServerKeysPath,
-		httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", intAPI.QueryServerKeys),
+		httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", enableMetrics, intAPI.QueryServerKeys),
 	)
 
 	internalAPIMux.Handle(
 		FederationAPILookupServerKeysPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupServerKeys",
+			"FederationAPILookupServerKeys", enableMetrics,
 			func(ctx context.Context, req *lookupServerKeys) (*[]gomatrixserverlib.ServerKeys, error) {
 				res, err := intAPI.LookupServerKeys(ctx, req.S, req.KeyRequests)
 				return &res, federationClientError(err)
@@ -177,7 +177,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPIEventRelationshipsPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIMSC2836EventRelationships",
+			"FederationAPIMSC2836EventRelationships", enableMetrics,
 			func(ctx context.Context, req *eventRelationships) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) {
 				res, err := intAPI.MSC2836EventRelationships(ctx, req.Origin, req.S, req.Req, req.RoomVer)
 				return &res, federationClientError(err)
@@ -188,7 +188,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 	internalAPIMux.Handle(
 		FederationAPISpacesSummaryPath,
 		httputil.MakeInternalProxyAPI(
-			"FederationAPIMSC2946SpacesSummary",
+			"FederationAPIMSC2946SpacesSummary", enableMetrics,
 			func(ctx context.Context, req *spacesReq) (*gomatrixserverlib.MSC2946SpacesResponse, error) {
 				res, err := intAPI.MSC2946Spaces(ctx, req.Origin, req.S, req.RoomID, req.SuggestedOnly)
 				return &res, federationClientError(err)
@@ -198,7 +198,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 
 	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIQueryPublicKeyPath,
-		httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
 			request := api.QueryPublicKeysRequest{}
 			response := api.QueryPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
@@ -215,7 +215,7 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
 
 	// TODO: Look at this shape
 	internalAPIMux.Handle(FederationAPIInputPublicKeyPath,
-		httputil.MakeInternalAPI("FederationAPIInputPublicKeys", func(req *http.Request) util.JSONResponse {
+		httputil.MakeInternalAPI("FederationAPIInputPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
 			request := api.InputPublicKeysRequest{}
 			response := api.InputPublicKeysResponse{}
 			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {

federationapi/queue/destinationqueue.go

@@ -29,7 +29,7 @@ import (
 	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -70,7 +70,7 @@ type destinationQueue struct {
 // Send event adds the event to the pending queue for the destination.
 // If the queue is empty then it starts a background goroutine to
 // start sending events to that destination.
-func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, receipt *shared.Receipt) {
+func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, dbReceipt *receipt.Receipt) {
 	if event == nil {
 		logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination)
 		return
@@ -84,8 +84,8 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re
 		oq.pendingMutex.Lock()
 		if len(oq.pendingPDUs) < maxPDUsInMemory {
 			oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{
-				pdu:     event,
-				receipt: receipt,
+				pdu:       event,
+				dbReceipt: dbReceipt,
 			})
 		} else {
 			oq.overflowed.Store(true)
@@ -101,7 +101,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re
 // sendEDU adds the EDU event to the pending queue for the destination.
 // If the queue is empty then it starts a background goroutine to
 // start sending events to that destination.
-func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *shared.Receipt) {
+func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, dbReceipt *receipt.Receipt) {
 	if event == nil {
 		logrus.Errorf("attempt to send nil EDU with destination %q", oq.destination)
 		return
@@ -115,8 +115,8 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 		oq.pendingMutex.Lock()
 		if len(oq.pendingEDUs) < maxEDUsInMemory {
 			oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{
-				edu:     event,
-				receipt: receipt,
+				edu:       event,
+				dbReceipt: dbReceipt,
 			})
 		} else {
 			oq.overflowed.Store(true)
@@ -210,10 +210,10 @@ func (oq *destinationQueue) getPendingFromDatabase() {
 	gotPDUs := map[string]struct{}{}
 	gotEDUs := map[string]struct{}{}
 	for _, pdu := range oq.pendingPDUs {
-		gotPDUs[pdu.receipt.String()] = struct{}{}
+		gotPDUs[pdu.dbReceipt.String()] = struct{}{}
 	}
 	for _, edu := range oq.pendingEDUs {
-		gotEDUs[edu.receipt.String()] = struct{}{}
+		gotEDUs[edu.dbReceipt.String()] = struct{}{}
 	}
 
 	overflowed := false
@@ -371,7 +371,7 @@ func (oq *destinationQueue) backgroundSend() {
 
 		// If we have pending PDUs or EDUs then construct a transaction.
 		// Try sending the next transaction and see what happens.
-		terr := oq.nextTransaction(toSendPDUs, toSendEDUs)
+		terr, sendMethod := oq.nextTransaction(toSendPDUs, toSendEDUs)
 		if terr != nil {
 			// We failed to send the transaction. Mark it as a failure.
 			_, blacklisted := oq.statistics.Failure()
@@ -388,18 +388,19 @@ func (oq *destinationQueue) backgroundSend() {
 				return
 			}
 		} else {
-			oq.handleTransactionSuccess(pduCount, eduCount)
+			oq.handleTransactionSuccess(pduCount, eduCount, sendMethod)
 		}
 	}
 }
 
 // nextTransaction creates a new transaction from the pending event
 // queue and sends it.
-// Returns an error if the transaction wasn't sent.
+// Returns an error if the transaction wasn't sent. And whether the success
+// was to a relay server or not.
 func (oq *destinationQueue) nextTransaction(
 	pdus []*queuedPDU,
 	edus []*queuedEDU,
-) error {
+) (err error, sendMethod statistics.SendMethod) {
 	// Create the transaction.
 	t, pduReceipts, eduReceipts := oq.createTransaction(pdus, edus)
 	logrus.WithField("server_name", oq.destination).Debugf("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs))
@@ -407,7 +408,37 @@ func (oq *destinationQueue) nextTransaction(
 	// Try to send the transaction to the destination server.
 	ctx, cancel := context.WithTimeout(oq.process.Context(), time.Minute*5)
 	defer cancel()
-	_, err := oq.client.SendTransaction(ctx, t)
+
+	relayServers := oq.statistics.KnownRelayServers()
+	if oq.statistics.AssumedOffline() && len(relayServers) > 0 {
+		sendMethod = statistics.SendViaRelay
+		relaySuccess := false
+		logrus.Infof("Sending to relay servers: %v", relayServers)
+		// TODO : how to pass through actual userID here?!?!?!?!
+		userID, userErr := gomatrixserverlib.NewUserID("@user:"+string(oq.destination), false)
+		if userErr != nil {
+			return userErr, sendMethod
+		}
+
+		// Attempt sending to each known relay server.
+		for _, relayServer := range relayServers {
+			_, relayErr := oq.client.P2PSendTransactionToRelay(ctx, *userID, t, relayServer)
+			if relayErr != nil {
+				err = relayErr
+			} else {
+				// If sending to one of the relay servers succeeds, consider the send successful.
+				relaySuccess = true
+			}
+		}
+
+		// Clear the error if sending to any of the relay servers succeeded.
+		if relaySuccess {
+			err = nil
+		}
+	} else {
+		sendMethod = statistics.SendDirect
+		_, err = oq.client.SendTransaction(ctx, t)
+	}
 	switch errResponse := err.(type) {
 	case nil:
 		// Clean up the transaction in the database.
@@ -427,7 +458,7 @@ func (oq *destinationQueue) nextTransaction(
 		oq.transactionIDMutex.Lock()
 		oq.transactionID = ""
 		oq.transactionIDMutex.Unlock()
-		return nil
+		return nil, sendMethod
 	case gomatrix.HTTPError:
 		// Report that we failed to send the transaction and we
 		// will retry again, subject to backoff.
@@ -437,13 +468,13 @@ func (oq *destinationQueue) nextTransaction(
 		// to a 400-ish error
 		code := errResponse.Code
 		logrus.Debug("Transaction failed with HTTP", code)
-		return err
+		return err, sendMethod
 	default:
 		logrus.WithFields(logrus.Fields{
 			"destination":   oq.destination,
 			logrus.ErrorKey: err,
 		}).Debugf("Failed to send transaction %q", t.TransactionID)
-		return err
+		return err, sendMethod
 	}
 }
 
@@ -453,7 +484,7 @@ func (oq *destinationQueue) nextTransaction(
 func (oq *destinationQueue) createTransaction(
 	pdus []*queuedPDU,
 	edus []*queuedEDU,
-) (gomatrixserverlib.Transaction, []*shared.Receipt, []*shared.Receipt) {
+) (gomatrixserverlib.Transaction, []*receipt.Receipt, []*receipt.Receipt) {
 	// If there's no projected transaction ID then generate one. If
 	// the transaction succeeds then we'll set it back to "" so that
 	// we generate a new one next time. If it fails, we'll preserve
@@ -474,8 +505,8 @@ func (oq *destinationQueue) createTransaction(
 	t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now())
 	t.TransactionID = oq.transactionID
 
-	var pduReceipts []*shared.Receipt
-	var eduReceipts []*shared.Receipt
+	var pduReceipts []*receipt.Receipt
+	var eduReceipts []*receipt.Receipt
 
 	// Go through PDUs that we retrieved from the database, if any,
 	// and add them into the transaction.
@@ -487,7 +518,7 @@ func (oq *destinationQueue) createTransaction(
 		// Append the JSON of the event, since this is a json.RawMessage type in the
 		// gomatrixserverlib.Transaction struct
 		t.PDUs = append(t.PDUs, pdu.pdu.JSON())
-		pduReceipts = append(pduReceipts, pdu.receipt)
+		pduReceipts = append(pduReceipts, pdu.dbReceipt)
 	}
 
 	// Do the same for pending EDUS in the queue.
@@ -497,7 +528,7 @@ func (oq *destinationQueue) createTransaction(
 			continue
 		}
 		t.EDUs = append(t.EDUs, *edu.edu)
-		eduReceipts = append(eduReceipts, edu.receipt)
+		eduReceipts = append(eduReceipts, edu.dbReceipt)
 	}
 
 	return t, pduReceipts, eduReceipts
@@ -530,10 +561,11 @@ func (oq *destinationQueue) blacklistDestination() {
 
 // handleTransactionSuccess updates the cached event queues as well as the success and
 // backoff information for this server.
-func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int) {
+func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int, sendMethod statistics.SendMethod) {
 	// If we successfully sent the transaction then clear out
 	// the pending events and EDUs, and wipe our transaction ID.
-	oq.statistics.Success()
+
+	oq.statistics.Success(sendMethod)
 	oq.pendingMutex.Lock()
 	defer oq.pendingMutex.Unlock()
 

federationapi/queue/queue.go

@@ -30,7 +30,7 @@ import (
 	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -138,13 +138,13 @@ func NewOutgoingQueues(
 }
 
 type queuedPDU struct {
-	receipt *shared.Receipt
-	pdu     *gomatrixserverlib.HeaderedEvent
+	dbReceipt *receipt.Receipt
+	pdu       *gomatrixserverlib.HeaderedEvent
 }
 
 type queuedEDU struct {
-	receipt *shared.Receipt
-	edu     *gomatrixserverlib.EDU
+	dbReceipt *receipt.Receipt
+	edu       *gomatrixserverlib.EDU
 }
 
 func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue {
@@ -374,24 +374,13 @@ func (oqs *OutgoingQueues) SendEDU(
 	return nil
 }
 
-// IsServerBlacklisted returns whether or not the provided server is currently
-// blacklisted.
-func (oqs *OutgoingQueues) IsServerBlacklisted(srv gomatrixserverlib.ServerName) bool {
-	return oqs.statistics.ForServer(srv).Blacklisted()
-}
-
 // RetryServer attempts to resend events to the given server if we had given up.
-func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) {
+func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName, wasBlacklisted bool) {
 	if oqs.disabled {
 		return
 	}
 
-	serverStatistics := oqs.statistics.ForServer(srv)
-	forceWakeup := serverStatistics.Blacklisted()
-	serverStatistics.RemoveBlacklist()
-	serverStatistics.ClearBackoff()
-
 	if queue := oqs.getQueue(srv); queue != nil {
-		queue.wakeQueueIfEventsPending(forceWakeup)
+		queue.wakeQueueIfEventsPending(wasBlacklisted)
 	}
 }

federationapi/queue/queue_test.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"sync"
 	"testing"
 	"time"
 
@@ -26,13 +25,11 @@ import (
 	"gotest.tools/v3/poll"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -57,7 +54,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
 		}
 	} else {
 		// Fake Database
-		db := createDatabase()
+		db := test.NewInMemoryFederationDatabase()
 		b := struct {
 			ProcessContext *process.ProcessContext
 		}{ProcessContext: process.NewProcessContext()}
@@ -65,242 +62,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
 	}
 }
 
-func createDatabase() storage.Database {
-	return &fakeDatabase{
-		pendingPDUServers:  make(map[gomatrixserverlib.ServerName]struct{}),
-		pendingEDUServers:  make(map[gomatrixserverlib.ServerName]struct{}),
-		blacklistedServers: make(map[gomatrixserverlib.ServerName]struct{}),
-		pendingPDUs:        make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent),
-		pendingEDUs:        make(map[*shared.Receipt]*gomatrixserverlib.EDU),
-		associatedPDUs:     make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}),
-		associatedEDUs:     make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}),
-	}
-}
-
-type fakeDatabase struct {
-	storage.Database
-	dbMutex            sync.Mutex
-	pendingPDUServers  map[gomatrixserverlib.ServerName]struct{}
-	pendingEDUServers  map[gomatrixserverlib.ServerName]struct{}
-	blacklistedServers map[gomatrixserverlib.ServerName]struct{}
-	pendingPDUs        map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent
-	pendingEDUs        map[*shared.Receipt]*gomatrixserverlib.EDU
-	associatedPDUs     map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}
-	associatedEDUs     map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}
-}
-
-var nidMutex sync.Mutex
-var nid = int64(0)
-
-func (d *fakeDatabase) StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	var event gomatrixserverlib.HeaderedEvent
-	if err := json.Unmarshal([]byte(js), &event); err == nil {
-		nidMutex.Lock()
-		defer nidMutex.Unlock()
-		nid++
-		receipt := shared.NewReceipt(nid)
-		d.pendingPDUs[&receipt] = &event
-		return &receipt, nil
-	}
-
-	var edu gomatrixserverlib.EDU
-	if err := json.Unmarshal([]byte(js), &edu); err == nil {
-		nidMutex.Lock()
-		defer nidMutex.Unlock()
-		nid++
-		receipt := shared.NewReceipt(nid)
-		d.pendingEDUs[&receipt] = &edu
-		return &receipt, nil
-	}
-
-	return nil, errors.New("Failed to determine type of json to store")
-}
-
-func (d *fakeDatabase) GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	pduCount := 0
-	pdus = make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent)
-	if receipts, ok := d.associatedPDUs[serverName]; ok {
-		for receipt := range receipts {
-			if event, ok := d.pendingPDUs[receipt]; ok {
-				pdus[receipt] = event
-				pduCount++
-				if pduCount == limit {
-					break
-				}
-			}
-		}
-	}
-	return pdus, nil
-}
-
-func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	eduCount := 0
-	edus = make(map[*shared.Receipt]*gomatrixserverlib.EDU)
-	if receipts, ok := d.associatedEDUs[serverName]; ok {
-		for receipt := range receipts {
-			if event, ok := d.pendingEDUs[receipt]; ok {
-				edus[receipt] = event
-				eduCount++
-				if eduCount == limit {
-					break
-				}
-			}
-		}
-	}
-	return edus, nil
-}
-
-func (d *fakeDatabase) AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if _, ok := d.pendingPDUs[receipt]; ok {
-		for destination := range destinations {
-			if _, ok := d.associatedPDUs[destination]; !ok {
-				d.associatedPDUs[destination] = make(map[*shared.Receipt]struct{})
-			}
-			d.associatedPDUs[destination][receipt] = struct{}{}
-		}
-
-		return nil
-	} else {
-		return errors.New("PDU doesn't exist")
-	}
-}
-
-func (d *fakeDatabase) AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if _, ok := d.pendingEDUs[receipt]; ok {
-		for destination := range destinations {
-			if _, ok := d.associatedEDUs[destination]; !ok {
-				d.associatedEDUs[destination] = make(map[*shared.Receipt]struct{})
-			}
-			d.associatedEDUs[destination][receipt] = struct{}{}
-		}
-
-		return nil
-	} else {
-		return errors.New("EDU doesn't exist")
-	}
-}
-
-func (d *fakeDatabase) CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if pdus, ok := d.associatedPDUs[serverName]; ok {
-		for _, receipt := range receipts {
-			delete(pdus, receipt)
-		}
-	}
-
-	return nil
-}
-
-func (d *fakeDatabase) CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if edus, ok := d.associatedEDUs[serverName]; ok {
-		for _, receipt := range receipts {
-			delete(edus, receipt)
-		}
-	}
-
-	return nil
-}
-
-func (d *fakeDatabase) GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	var count int64
-	if pdus, ok := d.associatedPDUs[serverName]; ok {
-		count = int64(len(pdus))
-	}
-	return count, nil
-}
-
-func (d *fakeDatabase) GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	var count int64
-	if edus, ok := d.associatedEDUs[serverName]; ok {
-		count = int64(len(edus))
-	}
-	return count, nil
-}
-
-func (d *fakeDatabase) GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	servers := []gomatrixserverlib.ServerName{}
-	for server := range d.pendingPDUServers {
-		servers = append(servers, server)
-	}
-	return servers, nil
-}
-
-func (d *fakeDatabase) GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	servers := []gomatrixserverlib.ServerName{}
-	for server := range d.pendingEDUServers {
-		servers = append(servers, server)
-	}
-	return servers, nil
-}
-
-func (d *fakeDatabase) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	d.blacklistedServers[serverName] = struct{}{}
-	return nil
-}
-
-func (d *fakeDatabase) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	delete(d.blacklistedServers, serverName)
-	return nil
-}
-
-func (d *fakeDatabase) RemoveAllServersFromBlacklist() error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	d.blacklistedServers = make(map[gomatrixserverlib.ServerName]struct{})
-	return nil
-}
-
-func (d *fakeDatabase) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	isBlacklisted := false
-	if _, ok := d.blacklistedServers[serverName]; ok {
-		isBlacklisted = true
-	}
-
-	return isBlacklisted, nil
-}
-
 type stubFederationRoomServerAPI struct {
 	rsapi.FederationRoomserverAPI
 }
@@ -312,8 +73,10 @@ func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Cont
 
 type stubFederationClient struct {
 	api.FederationClient
-	shouldTxSucceed bool
-	txCount         atomic.Uint32
+	shouldTxSucceed      bool
+	shouldTxRelaySucceed bool
+	txCount              atomic.Uint32
+	txRelayCount         atomic.Uint32
 }
 
 func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
@@ -326,6 +89,16 @@ func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixse
 	return gomatrixserverlib.RespSend{}, result
 }
 
+func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error) {
+	var result error
+	if !f.shouldTxRelaySucceed {
+		result = fmt.Errorf("relay transaction failed")
+	}
+
+	f.txRelayCount.Add(1)
+	return gomatrixserverlib.EmptyResp{}, result
+}
+
 func mustCreatePDU(t *testing.T) *gomatrixserverlib.HeaderedEvent {
 	t.Helper()
 	content := `{"type":"m.room.message"}`
@@ -341,15 +114,18 @@ func mustCreateEDU(t *testing.T) *gomatrixserverlib.EDU {
 	return &gomatrixserverlib.EDU{Type: gomatrixserverlib.MTyping}
 }
 
-func testSetup(failuresUntilBlacklist uint32, shouldTxSucceed bool, t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *stubFederationClient, *OutgoingQueues, *process.ProcessContext, func()) {
+func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32, shouldTxSucceed bool, shouldTxRelaySucceed bool, t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *stubFederationClient, *OutgoingQueues, *process.ProcessContext, func()) {
 	db, processContext, close := mustCreateFederationDatabase(t, dbType, realDatabase)
 
 	fc := &stubFederationClient{
-		shouldTxSucceed: shouldTxSucceed,
-		txCount:         *atomic.NewUint32(0),
+		shouldTxSucceed:      shouldTxSucceed,
+		shouldTxRelaySucceed: shouldTxRelaySucceed,
+		txCount:              *atomic.NewUint32(0),
+		txRelayCount:         *atomic.NewUint32(0),
 	}
 	rs := &stubFederationRoomServerAPI{}
-	stats := statistics.NewStatistics(db, failuresUntilBlacklist)
+
+	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
 	signingInfo := []*gomatrixserverlib.SigningIdentity{
 		{
 			KeyID:      "ed21019:auto",
@@ -366,7 +142,7 @@ func TestSendPDUOnSuccessRemovedFromDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -395,7 +171,7 @@ func TestSendEDUOnSuccessRemovedFromDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -424,7 +200,7 @@ func TestSendPDUOnFailStoredInDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -454,7 +230,7 @@ func TestSendEDUOnFailStoredInDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -484,7 +260,7 @@ func TestSendPDUAgainDoesntInterruptBackoff(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -535,7 +311,7 @@ func TestSendEDUAgainDoesntInterruptBackoff(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -586,7 +362,7 @@ func TestSendPDUMultipleFailuresBlacklisted(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -618,7 +394,7 @@ func TestSendEDUMultipleFailuresBlacklisted(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -650,7 +426,7 @@ func TestSendPDUBlacklistedWithPriorExternalFailure(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -684,7 +460,7 @@ func TestSendEDUBlacklistedWithPriorExternalFailure(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -718,7 +494,7 @@ func TestRetryServerSendsPDUSuccessfully(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(1)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -752,8 +528,8 @@ func TestRetryServerSendsPDUSuccessfully(t *testing.T) {
 	poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
 
 	fc.shouldTxSucceed = true
-	db.RemoveServerFromBlacklist(destination)
-	queues.RetryServer(destination)
+	wasBlacklisted := dest.statistics.MarkServerAlive()
+	queues.RetryServer(destination, wasBlacklisted)
 	checkRetry := func(log poll.LogT) poll.Result {
 		data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100)
 		assert.NoError(t, dbErr)
@@ -769,7 +545,7 @@ func TestRetryServerSendsEDUSuccessfully(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(1)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -803,8 +579,8 @@ func TestRetryServerSendsEDUSuccessfully(t *testing.T) {
 	poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
 
 	fc.shouldTxSucceed = true
-	db.RemoveServerFromBlacklist(destination)
-	queues.RetryServer(destination)
+	wasBlacklisted := dest.statistics.MarkServerAlive()
+	queues.RetryServer(destination, wasBlacklisted)
 	checkRetry := func(log poll.LogT) poll.Result {
 		data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100)
 		assert.NoError(t, dbErr)
@@ -823,7 +599,7 @@ func TestSendPDUBatches(t *testing.T) {
 
 	// test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 	// db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true)
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -867,7 +643,7 @@ func TestSendEDUBatches(t *testing.T) {
 
 	// test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 	// db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true)
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -911,7 +687,7 @@ func TestSendPDUAndEDUBatches(t *testing.T) {
 
 	// test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 	// db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true)
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -962,7 +738,7 @@ func TestExternalFailureBackoffDoesntStartQueue(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -1000,7 +776,7 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) {
 	destination := gomatrixserverlib.ServerName("remotehost")
 	destinations := map[gomatrixserverlib.ServerName]struct{}{destination: {}}
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, dbType, true)
+		db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, dbType, true)
 		// NOTE : These defers aren't called if go test is killed so the dbs may not get cleaned up.
 		defer close()
 		defer func() {
@@ -1045,8 +821,8 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) {
 		poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(10*time.Second), poll.WithDelay(100*time.Millisecond))
 
 		fc.shouldTxSucceed = true
-		db.RemoveServerFromBlacklist(destination)
-		queues.RetryServer(destination)
+		wasBlacklisted := dest.statistics.MarkServerAlive()
+		queues.RetryServer(destination, wasBlacklisted)
 		checkRetry := func(log poll.LogT) poll.Result {
 			pduData, dbErrPDU := db.GetPendingPDUs(pc.Context(), destination, 200)
 			assert.NoError(t, dbErrPDU)
@@ -1060,3 +836,147 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) {
 		poll.WaitOn(t, checkRetry, poll.WithTimeout(10*time.Second), poll.WithDelay(100*time.Millisecond))
 	})
 }
+
+func TestSendPDUMultipleFailuresAssumedOffline(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(7)
+	failuresUntilAssumedOffline := uint32(2)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, false, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	ev := mustCreatePDU(t)
+	err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() == failuresUntilAssumedOffline {
+			data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100)
+			assert.NoError(t, dbErr)
+			if len(data) == 1 {
+				if val, _ := db.IsServerAssumedOffline(context.Background(), destination); val {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for server to be assumed offline")
+			}
+			return poll.Continue("waiting for event to be added to database. Currently present PDU: %d", len(data))
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+}
+
+func TestSendEDUMultipleFailuresAssumedOffline(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(7)
+	failuresUntilAssumedOffline := uint32(2)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, false, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	ev := mustCreateEDU(t)
+	err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() == failuresUntilAssumedOffline {
+			data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100)
+			assert.NoError(t, dbErr)
+			if len(data) == 1 {
+				if val, _ := db.IsServerAssumedOffline(context.Background(), destination); val {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for server to be assumed offline")
+			}
+			return poll.Continue("waiting for event to be added to database. Currently present EDU: %d", len(data))
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+}
+
+func TestSendPDUOnRelaySuccessRemovedFromDB(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(16)
+	failuresUntilAssumedOffline := uint32(1)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, true, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	relayServers := []gomatrixserverlib.ServerName{"relayserver"}
+	queues.statistics.ForServer(destination).AddRelayServers(relayServers)
+
+	ev := mustCreatePDU(t)
+	err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() == 1 {
+			if fc.txRelayCount.Load() == 1 {
+				data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100)
+				assert.NoError(t, dbErr)
+				if len(data) == 0 {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for event to be removed from database. Currently present PDU: %d", len(data))
+			}
+			return poll.Continue("waiting for more relay send attempts before checking database. Currently %d", fc.txRelayCount.Load())
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+
+	assumedOffline, _ := db.IsServerAssumedOffline(context.Background(), destination)
+	assert.Equal(t, true, assumedOffline)
+}
+
+func TestSendEDUOnRelaySuccessRemovedFromDB(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(16)
+	failuresUntilAssumedOffline := uint32(1)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, true, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	relayServers := []gomatrixserverlib.ServerName{"relayserver"}
+	queues.statistics.ForServer(destination).AddRelayServers(relayServers)
+
+	ev := mustCreateEDU(t)
+	err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() == 1 {
+			if fc.txRelayCount.Load() == 1 {
+				data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100)
+				assert.NoError(t, dbErr)
+				if len(data) == 0 {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for event to be removed from database. Currently present EDU: %d", len(data))
+			}
+			return poll.Continue("waiting for more relay send attempts before checking database. Currently %d", fc.txRelayCount.Load())
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+
+	assumedOffline, _ := db.IsServerAssumedOffline(context.Background(), destination)
+	assert.Equal(t, true, assumedOffline)
+}

federationapi/routing/profile_test.go

@@ -0,0 +1,94 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing_test
+
+import (
+	"context"
+	"encoding/hex"
+	"io"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	fedAPI "github.com/matrix-org/dendrite/federationapi"
+	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
+	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ed25519"
+)
+
+type fakeUserAPI struct {
+	userAPI.FederationUserAPI
+}
+
+func (u *fakeUserAPI) QueryProfile(ctx context.Context, req *userAPI.QueryProfileRequest, res *userAPI.QueryProfileResponse) error {
+	return nil
+}
+
+func TestHandleQueryProfile(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
+
+		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
+		base.PublicFederationAPIMux = fedMux
+		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		fedClient := fakeFedClient{}
+		serverKeyAPI := &signing.YggdrasilKeys{}
+		keyRing := serverKeyAPI.KeyRing()
+		fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true)
+		userapi := fakeUserAPI{}
+		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+		if !ok {
+			panic("This is a programming error.")
+		}
+		routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, nil, &base.Cfg.MSCs, nil, nil)
+
+		handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP
+		_, sk, _ := ed25519.GenerateKey(nil)
+		keyID := signing.KeyID
+		pk := sk.Public().(ed25519.PublicKey)
+		serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+		req := gomatrixserverlib.NewFederationRequest("GET", serverName, testOrigin, "/query/profile?user_id="+url.QueryEscape("@user:"+string(testOrigin)))
+		type queryContent struct{}
+		content := queryContent{}
+		err := req.SetContent(content)
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
+		httpReq, err := req.HTTPRequest()
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		// vars := map[string]string{"room_alias": "#room:server"}
+		w := httptest.NewRecorder()
+		// httpReq = mux.SetURLVars(httpReq, vars)
+		handler(w, httpReq)
+
+		res := w.Result()
+		data, _ := io.ReadAll(res.Body)
+		println(string(data))
+		assert.Equal(t, 200, res.StatusCode)
+	})
+}

federationapi/routing/query_test.go

@@ -0,0 +1,94 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing_test
+
+import (
+	"context"
+	"encoding/hex"
+	"io"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	fedAPI "github.com/matrix-org/dendrite/federationapi"
+	fedclient "github.com/matrix-org/dendrite/federationapi/api"
+	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
+	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ed25519"
+)
+
+type fakeFedClient struct {
+	fedclient.FederationClient
+}
+
+func (f *fakeFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) {
+	return
+}
+
+func TestHandleQueryDirectory(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
+
+		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
+		base.PublicFederationAPIMux = fedMux
+		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		fedClient := fakeFedClient{}
+		serverKeyAPI := &signing.YggdrasilKeys{}
+		keyRing := serverKeyAPI.KeyRing()
+		fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true)
+		userapi := fakeUserAPI{}
+		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+		if !ok {
+			panic("This is a programming error.")
+		}
+		routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, nil, &base.Cfg.MSCs, nil, nil)
+
+		handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP
+		_, sk, _ := ed25519.GenerateKey(nil)
+		keyID := signing.KeyID
+		pk := sk.Public().(ed25519.PublicKey)
+		serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+		req := gomatrixserverlib.NewFederationRequest("GET", serverName, testOrigin, "/query/directory?room_alias="+url.QueryEscape("#room:server"))
+		type queryContent struct{}
+		content := queryContent{}
+		err := req.SetContent(content)
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
+		httpReq, err := req.HTTPRequest()
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		// vars := map[string]string{"room_alias": "#room:server"}
+		w := httptest.NewRecorder()
+		// httpReq = mux.SetURLVars(httpReq, vars)
+		handler(w, httpReq)
+
+		res := w.Result()
+		data, _ := io.ReadAll(res.Body)
+		println(string(data))
+		assert.Equal(t, 200, res.StatusCode)
+	})
+}

federationapi/routing/routing.go

@@ -32,6 +32,7 @@ import (
 	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -40,6 +41,12 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	SendRouteName           = "Send"
+	QueryDirectoryRouteName = "QueryDirectory"
+	QueryProfileRouteName   = "QueryProfile"
+)
+
 // Setup registers HTTP handlers with the given ServeMux.
 // The provided publicAPIMux MUST have `UseEncodedPath()` enabled or else routes will incorrectly
 // path unescape twice (once from the router, once from MakeFedAPI). We need to have this enabled
@@ -49,8 +56,7 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	fedMux, keyMux, wkMux *mux.Router,
-	cfg *config.FederationAPI,
+	base *base.BaseDendrite,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fsAPI *fedInternal.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
@@ -61,9 +67,16 @@ func Setup(
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,
 ) {
-	prometheus.MustRegister(
-		pduCountTotal, eduCountTotal,
-	)
+	fedMux := base.PublicFederationAPIMux
+	keyMux := base.PublicKeyAPIMux
+	wkMux := base.PublicWellKnownAPIMux
+	cfg := &base.Cfg.FederationAPI
+
+	if base.EnableMetrics {
+		prometheus.MustRegister(
+			internal.PDUCountTotal, internal.EDUCountTotal,
+		)
+	}
 
 	v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
 	v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
@@ -131,7 +144,7 @@ func Setup(
 				cfg, rsAPI, keyAPI, keys, federation, mu, servers, producer,
 			)
 		},
-	)).Methods(http.MethodPut, http.MethodOptions)
+	)).Methods(http.MethodPut, http.MethodOptions).Name(SendRouteName)
 
 	v1fedmux.Handle("/invite/{roomID}/{eventID}", MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
@@ -241,7 +254,7 @@ func Setup(
 				httpReq, federation, cfg, rsAPI, fsAPI,
 			)
 		},
-	)).Methods(http.MethodGet)
+	)).Methods(http.MethodGet).Name(QueryDirectoryRouteName)
 
 	v1fedmux.Handle("/query/profile", MakeFedAPI(
 		"federation_query_profile", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
@@ -250,7 +263,7 @@ func Setup(
 				httpReq, userAPI, cfg,
 			)
 		},
-	)).Methods(http.MethodGet)
+	)).Methods(http.MethodGet).Name(QueryProfileRouteName)
 
 	v1fedmux.Handle("/user/devices/{userID}", MakeFedAPI(
 		"federation_user_devices", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,

federationapi/routing/send.go

@@ -17,26 +17,20 @@ package routing
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"sync"
 	"time"
 
-	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/producers"
-	"github.com/matrix-org/dendrite/federationapi/types"
 	"github.com/matrix-org/dendrite/internal"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
 )
 
 const (
@@ -56,26 +50,6 @@ const (
 	MetricsWorkMissingPrevEvents = "missing_prev_events"
 )
 
-var (
-	pduCountTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "dendrite",
-			Subsystem: "federationapi",
-			Name:      "recv_pdus",
-			Help:      "Number of incoming PDUs from remote servers with labels for success",
-		},
-		[]string{"status"}, // 'success' or 'total'
-	)
-	eduCountTotal = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: "dendrite",
-			Subsystem: "federationapi",
-			Name:      "recv_edus",
-			Help:      "Number of incoming EDUs from remote servers",
-		},
-	)
-)
-
 var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse
 
 // Send implements /_matrix/federation/v1/send/{txnID}
@@ -123,18 +97,6 @@ func Send(
 	defer close(ch)
 	defer inFlightTxnsPerOrigin.Delete(index)
 
-	t := txnReq{
-		rsAPI:                  rsAPI,
-		keys:                   keys,
-		ourServerName:          cfg.Matrix.ServerName,
-		federation:             federation,
-		servers:                servers,
-		keyAPI:                 keyAPI,
-		roomsMu:                mu,
-		producer:               producer,
-		inboundPresenceEnabled: cfg.Matrix.Presence.EnableInbound,
-	}
-
 	var txnEvents struct {
 		PDUs []json.RawMessage       `json:"pdus"`
 		EDUs []gomatrixserverlib.EDU `json:"edus"`
@@ -155,16 +117,23 @@ func Send(
 		}
 	}
 
-	// TODO: Really we should have a function to convert FederationRequest to txnReq
-	t.PDUs = txnEvents.PDUs
-	t.EDUs = txnEvents.EDUs
-	t.Origin = request.Origin()
-	t.TransactionID = txnID
-	t.Destination = cfg.Matrix.ServerName
+	t := internal.NewTxnReq(
+		rsAPI,
+		keyAPI,
+		cfg.Matrix.ServerName,
+		keys,
+		mu,
+		producer,
+		cfg.Matrix.Presence.EnableInbound,
+		txnEvents.PDUs,
+		txnEvents.EDUs,
+		request.Origin(),
+		txnID,
+		cfg.Matrix.ServerName)
 
 	util.GetLogger(httpReq.Context()).Debugf("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs))
 
-	resp, jsonErr := t.processTransaction(httpReq.Context())
+	resp, jsonErr := t.ProcessTransaction(httpReq.Context())
 	if jsonErr != nil {
 		util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed")
 		return *jsonErr
@@ -181,283 +150,3 @@ func Send(
 	ch <- res
 	return res
 }
-
-type txnReq struct {
-	gomatrixserverlib.Transaction
-	rsAPI                  api.FederationRoomserverAPI
-	keyAPI                 keyapi.FederationKeyAPI
-	ourServerName          gomatrixserverlib.ServerName
-	keys                   gomatrixserverlib.JSONVerifier
-	federation             txnFederationClient
-	roomsMu                *internal.MutexByRoom
-	servers                federationAPI.ServersInRoomProvider
-	producer               *producers.SyncAPIProducer
-	inboundPresenceEnabled bool
-}
-
-// A subset of FederationClient functionality that txn requires. Useful for testing.
-type txnFederationClient interface {
-	LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (
-		res gomatrixserverlib.RespState, err error,
-	)
-	LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
-	GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
-	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents,
-		roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
-}
-
-func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.RespSend, *util.JSONResponse) {
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		t.processEDUs(ctx)
-	}()
-
-	results := make(map[string]gomatrixserverlib.PDUResult)
-	roomVersions := make(map[string]gomatrixserverlib.RoomVersion)
-	getRoomVersion := func(roomID string) gomatrixserverlib.RoomVersion {
-		if v, ok := roomVersions[roomID]; ok {
-			return v
-		}
-		verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
-		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := t.rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
-			util.GetLogger(ctx).WithError(err).Debug("Transaction: Failed to query room version for room", verReq.RoomID)
-			return ""
-		}
-		roomVersions[roomID] = verRes.RoomVersion
-		return verRes.RoomVersion
-	}
-
-	for _, pdu := range t.PDUs {
-		pduCountTotal.WithLabelValues("total").Inc()
-		var header struct {
-			RoomID string `json:"room_id"`
-		}
-		if err := json.Unmarshal(pdu, &header); err != nil {
-			util.GetLogger(ctx).WithError(err).Debug("Transaction: Failed to extract room ID from event")
-			// We don't know the event ID at this point so we can't return the
-			// failure in the PDU results
-			continue
-		}
-		roomVersion := getRoomVersion(header.RoomID)
-		event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion)
-		if err != nil {
-			if _, ok := err.(gomatrixserverlib.BadJSONError); ok {
-				// Room version 6 states that homeservers should strictly enforce canonical JSON
-				// on PDUs.
-				//
-				// This enforces that the entire transaction is rejected if a single bad PDU is
-				// sent. It is unclear if this is the correct behaviour or not.
-				//
-				// See https://github.com/matrix-org/synapse/issues/7543
-				return nil, &util.JSONResponse{
-					Code: 400,
-					JSON: jsonerror.BadJSON("PDU contains bad JSON"),
-				}
-			}
-			util.GetLogger(ctx).WithError(err).Debugf("Transaction: Failed to parse event JSON of event %s", string(pdu))
-			continue
-		}
-		if event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("") {
-			continue
-		}
-		if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID(), t.Origin) {
-			results[event.EventID()] = gomatrixserverlib.PDUResult{
-				Error: "Forbidden by server ACLs",
-			}
-			continue
-		}
-		if err = event.VerifyEventSignatures(ctx, t.keys); err != nil {
-			util.GetLogger(ctx).WithError(err).Debugf("Transaction: Couldn't validate signature of event %q", event.EventID())
-			results[event.EventID()] = gomatrixserverlib.PDUResult{
-				Error: err.Error(),
-			}
-			continue
-		}
-
-		// pass the event to the roomserver which will do auth checks
-		// If the event fail auth checks, gmsl.NotAllowed error will be returned which we be silently
-		// discarded by the caller of this function
-		if err = api.SendEvents(
-			ctx,
-			t.rsAPI,
-			api.KindNew,
-			[]*gomatrixserverlib.HeaderedEvent{
-				event.Headered(roomVersion),
-			},
-			t.Destination,
-			t.Origin,
-			api.DoNotSendToOtherServers,
-			nil,
-			true,
-		); err != nil {
-			util.GetLogger(ctx).WithError(err).Errorf("Transaction: Couldn't submit event %q to input queue: %s", event.EventID(), err)
-			results[event.EventID()] = gomatrixserverlib.PDUResult{
-				Error: err.Error(),
-			}
-			continue
-		}
-
-		results[event.EventID()] = gomatrixserverlib.PDUResult{}
-		pduCountTotal.WithLabelValues("success").Inc()
-	}
-
-	wg.Wait()
-	return &gomatrixserverlib.RespSend{PDUs: results}, nil
-}
-
-// nolint:gocyclo
-func (t *txnReq) processEDUs(ctx context.Context) {
-	for _, e := range t.EDUs {
-		eduCountTotal.Inc()
-		switch e.Type {
-		case gomatrixserverlib.MTyping:
-			// https://matrix.org/docs/spec/server_server/latest#typing-notifications
-			var typingPayload struct {
-				RoomID string `json:"room_id"`
-				UserID string `json:"user_id"`
-				Typing bool   `json:"typing"`
-			}
-			if err := json.Unmarshal(e.Content, &typingPayload); err != nil {
-				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal typing event")
-				continue
-			}
-			if _, serverName, err := gomatrixserverlib.SplitID('@', typingPayload.UserID); err != nil {
-				continue
-			} else if serverName == t.ourServerName {
-				continue
-			} else if serverName != t.Origin {
-				continue
-			}
-			if err := t.producer.SendTyping(ctx, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to JetStream")
-			}
-		case gomatrixserverlib.MDirectToDevice:
-			// https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema
-			var directPayload gomatrixserverlib.ToDeviceMessage
-			if err := json.Unmarshal(e.Content, &directPayload); err != nil {
-				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal send-to-device events")
-				continue
-			}
-			if _, serverName, err := gomatrixserverlib.SplitID('@', directPayload.Sender); err != nil {
-				continue
-			} else if serverName == t.ourServerName {
-				continue
-			} else if serverName != t.Origin {
-				continue
-			}
-			for userID, byUser := range directPayload.Messages {
-				for deviceID, message := range byUser {
-					// TODO: check that the user and the device actually exist here
-					if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil {
-						sentry.CaptureException(err)
-						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
-							"sender":    directPayload.Sender,
-							"user_id":   userID,
-							"device_id": deviceID,
-						}).Error("Failed to send send-to-device event to JetStream")
-					}
-				}
-			}
-		case gomatrixserverlib.MDeviceListUpdate:
-			if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil {
-				sentry.CaptureException(err)
-				util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate")
-			}
-		case gomatrixserverlib.MReceipt:
-			// https://matrix.org/docs/spec/server_server/r0.1.4#receipts
-			payload := map[string]types.FederationReceiptMRead{}
-
-			if err := json.Unmarshal(e.Content, &payload); err != nil {
-				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal receipt event")
-				continue
-			}
-
-			for roomID, receipt := range payload {
-				for userID, mread := range receipt.User {
-					_, domain, err := gomatrixserverlib.SplitID('@', userID)
-					if err != nil {
-						util.GetLogger(ctx).WithError(err).Debug("Failed to split domain from receipt event sender")
-						continue
-					}
-					if t.Origin != domain {
-						util.GetLogger(ctx).Debugf("Dropping receipt event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
-						continue
-					}
-					if err := t.processReceiptEvent(ctx, userID, roomID, "m.read", mread.Data.TS, mread.EventIDs); err != nil {
-						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
-							"sender":  t.Origin,
-							"user_id": userID,
-							"room_id": roomID,
-							"events":  mread.EventIDs,
-						}).Error("Failed to send receipt event to JetStream")
-						continue
-					}
-				}
-			}
-		case types.MSigningKeyUpdate:
-			if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil {
-				sentry.CaptureException(err)
-				logrus.WithError(err).Errorf("Failed to process signing key update")
-			}
-		case gomatrixserverlib.MPresence:
-			if t.inboundPresenceEnabled {
-				if err := t.processPresence(ctx, e); err != nil {
-					logrus.WithError(err).Errorf("Failed to process presence update")
-				}
-			}
-		default:
-			util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU")
-		}
-	}
-}
-
-// processPresence handles m.receipt events
-func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) error {
-	payload := types.Presence{}
-	if err := json.Unmarshal(e.Content, &payload); err != nil {
-		return err
-	}
-	for _, content := range payload.Push {
-		if _, serverName, err := gomatrixserverlib.SplitID('@', content.UserID); err != nil {
-			continue
-		} else if serverName == t.ourServerName {
-			continue
-		} else if serverName != t.Origin {
-			continue
-		}
-		presence, ok := syncTypes.PresenceFromString(content.Presence)
-		if !ok {
-			continue
-		}
-		if err := t.producer.SendPresence(ctx, content.UserID, presence, content.StatusMsg, content.LastActiveAgo); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// processReceiptEvent sends receipt events to JetStream
-func (t *txnReq) processReceiptEvent(ctx context.Context,
-	userID, roomID, receiptType string,
-	timestamp gomatrixserverlib.Timestamp,
-	eventIDs []string,
-) error {
-	if _, serverName, err := gomatrixserverlib.SplitID('@', userID); err != nil {
-		return nil
-	} else if serverName == t.ourServerName {
-		return nil
-	} else if serverName != t.Origin {
-		return nil
-	}
-	// store every event
-	for _, eventID := range eventIDs {
-		if err := t.producer.SendReceipt(ctx, userID, roomID, eventID, receiptType, timestamp); err != nil {
-			return fmt.Errorf("unable to set receipt event: %w", err)
-		}
-	}
-
-	return nil
-}

federationapi/routing/send_test.go

@@ -1,552 +1,87 @@
-package routing
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing_test
 
 import (
-	"context"
+	"encoding/hex"
 	"encoding/json"
-	"fmt"
+	"net/http/httptest"
 	"testing"
-	"time"
 
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	fedAPI "github.com/matrix-org/dendrite/federationapi"
+	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
+	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ed25519"
 )
 
 const (
-	testOrigin      = gomatrixserverlib.ServerName("kaer.morhen")
-	testDestination = gomatrixserverlib.ServerName("white.orchard")
+	testOrigin = gomatrixserverlib.ServerName("kaer.morhen")
 )
 
-var (
-	testRoomVersion = gomatrixserverlib.RoomVersionV1
-	testData        = []json.RawMessage{
-		[]byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$0ok8ynDp7kjc95e3:kaer.morhen","hashes":{"sha256":"17kPoH+h0Dk4Omn7Sus0qMb6+oGcf+CZFEgDhv7UKWs"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"jP4a04f5/F10Pw95FPpdCyKAO44JOwUQ/MZOOeA/RTU1Dn+AHPMzGSaZnuGjRr/xQuADt+I3ctb5ZQfLKNzHDw"}},"state_key":"","type":"m.room.create"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"content":{"membership":"join"},"depth":1,"event_id":"$LEwEu0kxrtu5fOiS:kaer.morhen","hashes":{"sha256":"B7M88PhXf3vd1LaFtjQutFu4x/w7fHD28XKZ4sAsJTo"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"p2vqmuJn7ZBRImctSaKbXCAxCcBlIjPH9JHte1ouIUGy84gpu4eLipOvSBCLL26hXfC0Zrm4WUto6Hr+ohdrCg"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"join_rule":"public"},"depth":2,"event_id":"$SMHlqUrNhhBBRLeN:kaer.morhen","hashes":{"sha256":"vIuJQvmMjrGxshAkj1SXe0C4RqvMbv4ZADDw9pFCWqQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"hBMsb3Qppo3RaqqAl4JyTgaiWEbW5hlckATky6PrHun+F3YM203TzG7w9clwuQU5F5pZoB1a6nw+to0hN90FAw"}},"state_key":"","type":"m.room.join_rules"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"history_visibility":"shared"},"depth":3,"event_id":"$6F1yGIbO0J7TM93h:kaer.morhen","hashes":{"sha256":"Mr23GKSlZW7UCCYLgOWawI2Sg6KIoMjUWO2TDenuOgw"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$SMHlqUrNhhBBRLeN:kaer.morhen",{"sha256":"SylzE8U02I+6eyEHgL+FlU0L5YdqrVp8OOlxKS9VQW0"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sHLKrFI3hKGrEJfpMVZSDS3LvLasQsy50CTsOwru9XTVxgRsPo6wozNtRVjxo1J3Rk18RC9JppovmQ5VR5EcDw"}},"state_key":"","type":"m.room.history_visibility"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"ban":50,"events":null,"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":null,"users_default":0},"depth":4,"event_id":"$UKNe10XzYzG0TeA9:kaer.morhen","hashes":{"sha256":"ngbP3yja9U5dlckKerUs/fSOhtKxZMCVvsfhPURSS28"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$6F1yGIbO0J7TM93h:kaer.morhen",{"sha256":"A4CucrKSoWX4IaJXhq02mBg1sxIyZEftbC+5p3fZAvk"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"zOmwlP01QL3yFchzuR9WHvogOoBZA3oVtNIF3lM0ZfDnqlSYZB9sns27G/4HVq0k7alaK7ZE3oGoCrVnMkPNCw"}},"state_key":"","type":"m.room.power_levels"}`),
-		// messages
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":6,"event_id":"$MYSbs8m4rEbsCWXD:kaer.morhen","hashes":{"sha256":"kgbYM7v4Ud2YaBsjBTolM4ySg6rHcJNYI6nWhMSdFUA"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$gl2T9l3qm0kUbiIJ:kaer.morhen",{"sha256":"C/rD04h9wGxRdN2G/IBfrgoE1UovzLZ+uskwaKZ37/Q"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"x0UoKh968jj/F5l1/R7Ew0T6CTKuew3PLNHASNxqck/bkNe8yYQiDHXRr+kZxObeqPZZTpaF1+EI+bLU9W8GDQ"}},"type":"m.room.message"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":7,"event_id":"$N5x9WJkl9ClPrAEg:kaer.morhen","hashes":{"sha256":"FWM8oz4yquTunRZ67qlW2gzPDzdWfBP6RPHXhK1I/x8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$MYSbs8m4rEbsCWXD:kaer.morhen",{"sha256":"fatqgW+SE8mb2wFn3UN+drmluoD4UJ/EcSrL6Ur9q1M"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"Y+LX/xcyufoXMOIoqQBNOzy6lZfUGB1ffgXIrSugk6obMiyAsiRejHQN/pciZXsHKxMJLYRFAz4zSJoS/LGPAA"}},"type":"m.room.message"}`),
-	}
-	testEvents      = []*gomatrixserverlib.HeaderedEvent{}
-	testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent)
-)
+type sendContent struct {
+	PDUs []json.RawMessage       `json:"pdus"`
+	EDUs []gomatrixserverlib.EDU `json:"edus"`
+}
 
-func init() {
-	for _, j := range testData {
-		e, err := gomatrixserverlib.NewEventFromTrustedJSON(j, false, testRoomVersion)
+func TestHandleSend(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
+
+		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
+		base.PublicFederationAPIMux = fedMux
+		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)
+		serverKeyAPI := &signing.YggdrasilKeys{}
+		keyRing := serverKeyAPI.KeyRing()
+		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+		if !ok {
+			panic("This is a programming error.")
+		}
+		routing.Setup(base, nil, r, keyRing, nil, nil, nil, &base.Cfg.MSCs, nil, nil)
+
+		handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
+		_, sk, _ := ed25519.GenerateKey(nil)
+		keyID := signing.KeyID
+		pk := sk.Public().(ed25519.PublicKey)
+		serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+		req := gomatrixserverlib.NewFederationRequest("PUT", serverName, testOrigin, "/send/1234")
+		content := sendContent{}
+		err := req.SetContent(content)
 		if err != nil {
-			panic("cannot load test data: " + err.Error())
+			t.Fatalf("Error: %s", err.Error())
 		}
-		h := e.Headered(testRoomVersion)
-		testEvents = append(testEvents, h)
-		if e.StateKey() != nil {
-			testStateEvents[gomatrixserverlib.StateKeyTuple{
-				EventType: e.Type(),
-				StateKey:  *e.StateKey(),
-			}] = h
+		req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
+		httpReq, err := req.HTTPRequest()
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
 		}
-	}
+		vars := map[string]string{"txnID": "1234"}
+		w := httptest.NewRecorder()
+		httpReq = mux.SetURLVars(httpReq, vars)
+		handler(w, httpReq)
+
+		res := w.Result()
+		assert.Equal(t, 200, res.StatusCode)
+	})
 }
-
-type testRoomserverAPI struct {
-	api.RoomserverInternalAPITrace
-	inputRoomEvents           []api.InputRoomEvent
-	queryStateAfterEvents     func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse
-	queryEventsByID           func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse
-	queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse
-}
-
-func (t *testRoomserverAPI) InputRoomEvents(
-	ctx context.Context,
-	request *api.InputRoomEventsRequest,
-	response *api.InputRoomEventsResponse,
-) error {
-	t.inputRoomEvents = append(t.inputRoomEvents, request.InputRoomEvents...)
-	for _, ire := range request.InputRoomEvents {
-		fmt.Println("InputRoomEvents: ", ire.Event.EventID())
-	}
-	return nil
-}
-
-// Query the latest events and state for a room from the room server.
-func (t *testRoomserverAPI) QueryLatestEventsAndState(
-	ctx context.Context,
-	request *api.QueryLatestEventsAndStateRequest,
-	response *api.QueryLatestEventsAndStateResponse,
-) error {
-	r := t.queryLatestEventsAndState(request)
-	response.RoomExists = r.RoomExists
-	response.RoomVersion = testRoomVersion
-	response.LatestEvents = r.LatestEvents
-	response.StateEvents = r.StateEvents
-	response.Depth = r.Depth
-	return nil
-}
-
-// Query the state after a list of events in a room from the room server.
-func (t *testRoomserverAPI) QueryStateAfterEvents(
-	ctx context.Context,
-	request *api.QueryStateAfterEventsRequest,
-	response *api.QueryStateAfterEventsResponse,
-) error {
-	response.RoomVersion = testRoomVersion
-	res := t.queryStateAfterEvents(request)
-	response.PrevEventsExist = res.PrevEventsExist
-	response.RoomExists = res.RoomExists
-	response.StateEvents = res.StateEvents
-	return nil
-}
-
-// Query a list of events by event ID.
-func (t *testRoomserverAPI) QueryEventsByID(
-	ctx context.Context,
-	request *api.QueryEventsByIDRequest,
-	response *api.QueryEventsByIDResponse,
-) error {
-	res := t.queryEventsByID(request)
-	response.Events = res.Events
-	return nil
-}
-
-// Query if a server is joined to a room
-func (t *testRoomserverAPI) QueryServerJoinedToRoom(
-	ctx context.Context,
-	request *api.QueryServerJoinedToRoomRequest,
-	response *api.QueryServerJoinedToRoomResponse,
-) error {
-	response.RoomExists = true
-	response.IsInRoom = true
-	return nil
-}
-
-// Asks for the room version for a given room.
-func (t *testRoomserverAPI) QueryRoomVersionForRoom(
-	ctx context.Context,
-	request *api.QueryRoomVersionForRoomRequest,
-	response *api.QueryRoomVersionForRoomResponse,
-) error {
-	response.RoomVersion = testRoomVersion
-	return nil
-}
-
-func (t *testRoomserverAPI) QueryServerBannedFromRoom(
-	ctx context.Context, req *api.QueryServerBannedFromRoomRequest, res *api.QueryServerBannedFromRoomResponse,
-) error {
-	res.Banned = false
-	return nil
-}
-
-type txnFedClient struct {
-	state            map[string]gomatrixserverlib.RespState    // event_id to response
-	stateIDs         map[string]gomatrixserverlib.RespStateIDs // event_id to response
-	getEvent         map[string]gomatrixserverlib.Transaction  // event_id to response
-	getMissingEvents func(gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error)
-}
-
-func (c *txnFedClient) LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (
-	res gomatrixserverlib.RespState, err error,
-) {
-	fmt.Println("testFederationClient.LookupState", eventID)
-	r, ok := c.state[eventID]
-	if !ok {
-		err = fmt.Errorf("txnFedClient: no /state for event %s", eventID)
-		return
-	}
-	res = r
-	return
-}
-func (c *txnFedClient) LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) {
-	fmt.Println("testFederationClient.LookupStateIDs", eventID)
-	r, ok := c.stateIDs[eventID]
-	if !ok {
-		err = fmt.Errorf("txnFedClient: no /state_ids for event %s", eventID)
-		return
-	}
-	res = r
-	return
-}
-func (c *txnFedClient) GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) {
-	fmt.Println("testFederationClient.GetEvent", eventID)
-	r, ok := c.getEvent[eventID]
-	if !ok {
-		err = fmt.Errorf("txnFedClient: no /event for event ID %s", eventID)
-		return
-	}
-	res = r
-	return
-}
-func (c *txnFedClient) LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents,
-	roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) {
-	return c.getMissingEvents(missing)
-}
-
-func mustCreateTransaction(rsAPI api.FederationRoomserverAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq {
-	t := &txnReq{
-		rsAPI:      rsAPI,
-		keys:       &test.NopJSONVerifier{},
-		federation: fedClient,
-		roomsMu:    internal.NewMutexByRoom(),
-	}
-	t.PDUs = pdus
-	t.Origin = testOrigin
-	t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
-	t.Destination = testDestination
-	return t
-}
-
-func mustProcessTransaction(t *testing.T, txn *txnReq, pdusWithErrors []string) {
-	res, err := txn.processTransaction(context.Background())
-	if err != nil {
-		t.Errorf("txn.processTransaction returned an error: %v", err)
-		return
-	}
-	if len(res.PDUs) != len(txn.PDUs) {
-		t.Errorf("txn.processTransaction did not return results for all PDUs, got %d want %d", len(res.PDUs), len(txn.PDUs))
-		return
-	}
-NextPDU:
-	for eventID, result := range res.PDUs {
-		if result.Error == "" {
-			continue
-		}
-		for _, eventIDWantError := range pdusWithErrors {
-			if eventID == eventIDWantError {
-				break NextPDU
-			}
-		}
-		t.Errorf("txn.processTransaction PDU %s returned an error %s", eventID, result.Error)
-	}
-}
-
-/*
-func fromStateTuples(tuples []gomatrixserverlib.StateKeyTuple, omitTuples []gomatrixserverlib.StateKeyTuple) (result []*gomatrixserverlib.HeaderedEvent) {
-NextTuple:
-	for _, t := range tuples {
-		for _, o := range omitTuples {
-			if t == o {
-				break NextTuple
-			}
-		}
-		h, ok := testStateEvents[t]
-		if ok {
-			result = append(result, h)
-		}
-	}
-	return
-}
-*/
-
-func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []*gomatrixserverlib.HeaderedEvent) {
-	for _, g := range got {
-		fmt.Println("GOT ", g.Event.EventID())
-	}
-	if len(got) != len(want) {
-		t.Errorf("wrong number of InputRoomEvents: got %d want %d", len(got), len(want))
-		return
-	}
-	for i := range got {
-		if got[i].Event.EventID() != want[i].EventID() {
-			t.Errorf("InputRoomEvents[%d] got %s want %s", i, got[i].Event.EventID(), want[i].EventID())
-		}
-	}
-}
-
-// The purpose of this test is to check that receiving an event over federation for which we have the prev_events works correctly, and passes it on
-// to the roomserver. It's the most basic test possible.
-func TestBasicTransaction(t *testing.T) {
-	rsAPI := &testRoomserverAPI{}
-	pdus := []json.RawMessage{
-		testData[len(testData)-1], // a message event
-	}
-	txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus)
-	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
-}
-
-// The purpose of this test is to check that if the event received fails auth checks the event is still sent to the roomserver
-// as it does the auth check.
-func TestTransactionFailAuthChecks(t *testing.T) {
-	rsAPI := &testRoomserverAPI{}
-	pdus := []json.RawMessage{
-		testData[len(testData)-1], // a message event
-	}
-	txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus)
-	mustProcessTransaction(t, txn, []string{})
-	// expect message to be sent to the roomserver
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
-}
-
-// The purpose of this test is to make sure that when an event is received for which we do not know the prev_events,
-// we request them from /get_missing_events. It works by setting PrevEventsExist=false in the roomserver query response,
-// resulting in a call to /get_missing_events which returns the missing prev event. Both events should be processed in
-// topological order and sent to the roomserver.
-/*
-func TestTransactionFetchMissingPrevEvents(t *testing.T) {
-	haveEvent := testEvents[len(testEvents)-3]
-	prevEvent := testEvents[len(testEvents)-2]
-	inputEvent := testEvents[len(testEvents)-1]
-
-	var rsAPI *testRoomserverAPI // ref here so we can refer to inputRoomEvents inside these functions
-	rsAPI = &testRoomserverAPI{
-		queryEventsByID: func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse {
-			res := api.QueryEventsByIDResponse{}
-			for _, ev := range testEvents {
-				for _, id := range req.EventIDs {
-					if ev.EventID() == id {
-						res.Events = append(res.Events, ev)
-					}
-				}
-			}
-			return res
-		},
-		queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse {
-			return api.QueryStateAfterEventsResponse{
-				PrevEventsExist: true,
-				StateEvents:     testEvents[:5],
-			}
-		},
-		queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse {
-			missingPrevEvent := []string{"missing_prev_event"}
-			if len(req.PrevEventIDs) == 1 {
-				switch req.PrevEventIDs[0] {
-				case haveEvent.EventID():
-					missingPrevEvent = []string{}
-				case prevEvent.EventID():
-					// we only have this event if we've been send prevEvent
-					if len(rsAPI.inputRoomEvents) == 1 && rsAPI.inputRoomEvents[0].Event.EventID() == prevEvent.EventID() {
-						missingPrevEvent = []string{}
-					}
-				}
-			}
-
-			return api.QueryMissingAuthPrevEventsResponse{
-				RoomExists:          true,
-				MissingAuthEventIDs: []string{},
-				MissingPrevEventIDs: missingPrevEvent,
-			}
-		},
-		queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse {
-			return api.QueryLatestEventsAndStateResponse{
-				RoomExists: true,
-				Depth:      haveEvent.Depth(),
-				LatestEvents: []gomatrixserverlib.EventReference{
-					haveEvent.EventReference(),
-				},
-				StateEvents: fromStateTuples(req.StateToFetch, nil),
-			}
-		},
-	}
-
-	cli := &txnFedClient{
-		getMissingEvents: func(missing gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error) {
-			if !reflect.DeepEqual(missing.EarliestEvents, []string{haveEvent.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong earliest events: got %v want %v", missing.EarliestEvents, haveEvent.EventID())
-			}
-			if !reflect.DeepEqual(missing.LatestEvents, []string{inputEvent.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong latest events: got %v want %v", missing.LatestEvents, inputEvent.EventID())
-			}
-			return gomatrixserverlib.RespMissingEvents{
-				Events: []*gomatrixserverlib.Event{
-					prevEvent.Unwrap(),
-				},
-			}, nil
-		},
-	}
-
-	pdus := []json.RawMessage{
-		inputEvent.JSON(),
-	}
-	txn := mustCreateTransaction(rsAPI, cli, pdus)
-	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{prevEvent, inputEvent})
-}
-
-// The purpose of this test is to check that when there are missing prev_events and we still haven't been able to fill
-// in the hole with /get_missing_events that the state BEFORE the events we want to persist is fetched via /state_ids
-// and /event. It works by setting PrevEventsExist=false in the roomserver query response, resulting in
-// a call to /get_missing_events which returns 1 out of the 2 events it needs to fill in the gap. Synapse and Dendrite
-// both give up after 1x /get_missing_events call, relying on requesting the state AFTER the missing event in order to
-// continue. The DAG looks something like:
-// FE           GME    TXN
-//  A ---> B ---> C ---> D
-// TXN=event in the txn, GME=response to /get_missing_events, FE=roomserver's forward extremity. Should result in:
-// - /state_ids?event=B is requested, then /event/B to get the state AFTER B. B is a state event.
-// - state resolution is done to check C is allowed.
-// This results in B being sent as an outlier FIRST, then C,D.
-func TestTransactionFetchMissingStateByStateIDs(t *testing.T) {
-	eventA := testEvents[len(testEvents)-5]
-	// this is also len(testEvents)-4
-	eventB := testStateEvents[gomatrixserverlib.StateKeyTuple{
-		EventType: gomatrixserverlib.MRoomPowerLevels,
-		StateKey:  "",
-	}]
-	eventC := testEvents[len(testEvents)-3]
-	eventD := testEvents[len(testEvents)-2]
-	fmt.Println("a:", eventA.EventID())
-	fmt.Println("b:", eventB.EventID())
-	fmt.Println("c:", eventC.EventID())
-	fmt.Println("d:", eventD.EventID())
-	var rsAPI *testRoomserverAPI
-	rsAPI = &testRoomserverAPI{
-		queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse {
-			omitTuples := []gomatrixserverlib.StateKeyTuple{
-				{
-					EventType: gomatrixserverlib.MRoomPowerLevels,
-					StateKey:  "",
-				},
-			}
-			askingForEvent := req.PrevEventIDs[0]
-			haveEventB := false
-			haveEventC := false
-			for _, ev := range rsAPI.inputRoomEvents {
-				switch ev.Event.EventID() {
-				case eventB.EventID():
-					haveEventB = true
-					omitTuples = nil // include event B now
-				case eventC.EventID():
-					haveEventC = true
-				}
-			}
-			prevEventExists := false
-			if askingForEvent == eventC.EventID() {
-				prevEventExists = haveEventC
-			} else if askingForEvent == eventB.EventID() {
-				prevEventExists = haveEventB
-			}
-			var stateEvents []*gomatrixserverlib.HeaderedEvent
-			if prevEventExists {
-				stateEvents = fromStateTuples(req.StateToFetch, omitTuples)
-			}
-			return api.QueryStateAfterEventsResponse{
-				PrevEventsExist: prevEventExists,
-				RoomExists:      true,
-				StateEvents:     stateEvents,
-			}
-		},
-
-		queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse {
-			askingForEvent := req.PrevEventIDs[0]
-			haveEventB := false
-			haveEventC := false
-			for _, ev := range rsAPI.inputRoomEvents {
-				switch ev.Event.EventID() {
-				case eventB.EventID():
-					haveEventB = true
-				case eventC.EventID():
-					haveEventC = true
-				}
-			}
-			prevEventExists := false
-			if askingForEvent == eventC.EventID() {
-				prevEventExists = haveEventC
-			} else if askingForEvent == eventB.EventID() {
-				prevEventExists = haveEventB
-			}
-
-			var missingPrevEvent []string
-			if !prevEventExists {
-				missingPrevEvent = []string{"test"}
-			}
-
-			return api.QueryMissingAuthPrevEventsResponse{
-				RoomExists:          true,
-				MissingAuthEventIDs: []string{},
-				MissingPrevEventIDs: missingPrevEvent,
-			}
-		},
-
-		queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse {
-			omitTuples := []gomatrixserverlib.StateKeyTuple{
-				{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""},
-			}
-			return api.QueryLatestEventsAndStateResponse{
-				RoomExists: true,
-				Depth:      eventA.Depth(),
-				LatestEvents: []gomatrixserverlib.EventReference{
-					eventA.EventReference(),
-				},
-				StateEvents: fromStateTuples(req.StateToFetch, omitTuples),
-			}
-		},
-		queryEventsByID: func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse {
-			var res api.QueryEventsByIDResponse
-			fmt.Println("queryEventsByID ", req.EventIDs)
-			for _, wantEventID := range req.EventIDs {
-				for _, ev := range testStateEvents {
-					// roomserver is missing the power levels event unless it's been sent to us recently as an outlier
-					if wantEventID == eventB.EventID() {
-						fmt.Println("Asked for pl event")
-						for _, inEv := range rsAPI.inputRoomEvents {
-							fmt.Println("recv ", inEv.Event.EventID())
-							if inEv.Event.EventID() == wantEventID {
-								res.Events = append(res.Events, inEv.Event)
-								break
-							}
-						}
-						continue
-					}
-					if ev.EventID() == wantEventID {
-						res.Events = append(res.Events, ev)
-					}
-				}
-			}
-			return res
-		},
-	}
-	// /state_ids for event B returns every state event but B (it's the state before)
-	var authEventIDs []string
-	var stateEventIDs []string
-	for _, ev := range testStateEvents {
-		if ev.EventID() == eventB.EventID() {
-			continue
-		}
-		// state res checks what auth events you give it, and this isn't a valid auth event
-		if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility {
-			authEventIDs = append(authEventIDs, ev.EventID())
-		}
-		stateEventIDs = append(stateEventIDs, ev.EventID())
-	}
-	cli := &txnFedClient{
-		stateIDs: map[string]gomatrixserverlib.RespStateIDs{
-			eventB.EventID(): {
-				StateEventIDs: stateEventIDs,
-				AuthEventIDs:  authEventIDs,
-			},
-		},
-		// /event for event B returns it
-		getEvent: map[string]gomatrixserverlib.Transaction{
-			eventB.EventID(): {
-				PDUs: []json.RawMessage{
-					eventB.JSON(),
-				},
-			},
-		},
-		// /get_missing_events should be done exactly once
-		getMissingEvents: func(missing gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error) {
-			if !reflect.DeepEqual(missing.EarliestEvents, []string{eventA.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong earliest events: got %v want %v", missing.EarliestEvents, eventA.EventID())
-			}
-			if !reflect.DeepEqual(missing.LatestEvents, []string{eventD.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong latest events: got %v want %v", missing.LatestEvents, eventD.EventID())
-			}
-			// just return event C, not event B so /state_ids logic kicks in as there will STILL be missing prev_events
-			return gomatrixserverlib.RespMissingEvents{
-				Events: []*gomatrixserverlib.Event{
-					eventC.Unwrap(),
-				},
-			}, nil
-		},
-	}
-
-	pdus := []json.RawMessage{
-		eventD.JSON(),
-	}
-	txn := mustCreateTransaction(rsAPI, cli, pdus)
-	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{eventB, eventC, eventD})
-}
-*/

federationapi/statistics/statistics.go

@@ -1,6 +1,7 @@
 package statistics
 
 import (
+	"context"
 	"math"
 	"math/rand"
 	"sync"
@@ -28,25 +29,30 @@ type Statistics struct {
 	// just blacklist the host altogether? The backoff is exponential,
 	// so the max time here to attempt is 2**failures seconds.
 	FailuresUntilBlacklist uint32
+
+	// How many times should we tolerate consecutive failures before we
+	// mark the destination as offline. At this point we should attempt
+	// to send messages to the user's async relay servers if we know them.
+	FailuresUntilAssumedOffline uint32
 }
 
-func NewStatistics(db storage.Database, failuresUntilBlacklist uint32) Statistics {
+func NewStatistics(
+	db storage.Database,
+	failuresUntilBlacklist uint32,
+	failuresUntilAssumedOffline uint32,
+) Statistics {
 	return Statistics{
-		DB:                     db,
-		FailuresUntilBlacklist: failuresUntilBlacklist,
-		backoffTimers:          make(map[gomatrixserverlib.ServerName]*time.Timer),
+		DB:                          db,
+		FailuresUntilBlacklist:      failuresUntilBlacklist,
+		FailuresUntilAssumedOffline: failuresUntilAssumedOffline,
+		backoffTimers:               make(map[gomatrixserverlib.ServerName]*time.Timer),
+		servers:                     make(map[gomatrixserverlib.ServerName]*ServerStatistics),
 	}
 }
 
 // ForServer returns server statistics for the given server name. If it
 // does not exist, it will create empty statistics and return those.
 func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerStatistics {
-	// If the map hasn't been initialised yet then do that.
-	if s.servers == nil {
-		s.mutex.Lock()
-		s.servers = make(map[gomatrixserverlib.ServerName]*ServerStatistics)
-		s.mutex.Unlock()
-	}
 	// Look up if we have statistics for this server already.
 	s.mutex.RLock()
 	server, found := s.servers[serverName]
@@ -55,8 +61,9 @@ func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerS
 	if !found {
 		s.mutex.Lock()
 		server = &ServerStatistics{
-			statistics: s,
-			serverName: serverName,
+			statistics:        s,
+			serverName:        serverName,
+			knownRelayServers: []gomatrixserverlib.ServerName{},
 		}
 		s.servers[serverName] = server
 		s.mutex.Unlock()
@@ -66,24 +73,49 @@ func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerS
 		} else {
 			server.blacklisted.Store(blacklisted)
 		}
+		assumedOffline, err := s.DB.IsServerAssumedOffline(context.Background(), serverName)
+		if err != nil {
+			logrus.WithError(err).Errorf("Failed to get assumed offline entry %q", serverName)
+		} else {
+			server.assumedOffline.Store(assumedOffline)
+		}
+
+		knownRelayServers, err := s.DB.P2PGetRelayServersForServer(context.Background(), serverName)
+		if err != nil {
+			logrus.WithError(err).Errorf("Failed to get relay server list for %q", serverName)
+		} else {
+			server.relayMutex.Lock()
+			server.knownRelayServers = knownRelayServers
+			server.relayMutex.Unlock()
+		}
 	}
 	return server
 }
 
+type SendMethod uint8
+
+const (
+	SendDirect SendMethod = iota
+	SendViaRelay
+)
+
 // ServerStatistics contains information about our interactions with a
 // remote federated host, e.g. how many times we were successful, how
 // many times we failed etc. It also manages the backoff time and black-
 // listing a remote host if it remains uncooperative.
 type ServerStatistics struct {
-	statistics      *Statistics                  //
-	serverName      gomatrixserverlib.ServerName //
-	blacklisted     atomic.Bool                  // is the node blacklisted
-	backoffStarted  atomic.Bool                  // is the backoff started
-	backoffUntil    atomic.Value                 // time.Time until this backoff interval ends
-	backoffCount    atomic.Uint32                // number of times BackoffDuration has been called
-	successCounter  atomic.Uint32                // how many times have we succeeded?
-	backoffNotifier func()                       // notifies destination queue when backoff completes
-	notifierMutex   sync.Mutex
+	statistics        *Statistics                  //
+	serverName        gomatrixserverlib.ServerName //
+	blacklisted       atomic.Bool                  // is the node blacklisted
+	assumedOffline    atomic.Bool                  // is the node assumed to be offline
+	backoffStarted    atomic.Bool                  // is the backoff started
+	backoffUntil      atomic.Value                 // time.Time until this backoff interval ends
+	backoffCount      atomic.Uint32                // number of times BackoffDuration has been called
+	successCounter    atomic.Uint32                // how many times have we succeeded?
+	backoffNotifier   func()                       // notifies destination queue when backoff completes
+	notifierMutex     sync.Mutex
+	knownRelayServers []gomatrixserverlib.ServerName
+	relayMutex        sync.Mutex
 }
 
 const maxJitterMultiplier = 1.4
@@ -118,13 +150,19 @@ func (s *ServerStatistics) AssignBackoffNotifier(notifier func()) {
 // attempt, which increases the sent counter and resets the idle and
 // failure counters. If a host was blacklisted at this point then
 // we will unblacklist it.
-func (s *ServerStatistics) Success() {
+// `relay` specifies whether the success was to the actual destination
+// or one of their relay servers.
+func (s *ServerStatistics) Success(method SendMethod) {
 	s.cancel()
 	s.backoffCount.Store(0)
-	s.successCounter.Inc()
-	if s.statistics.DB != nil {
-		if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil {
-			logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName)
+	// NOTE : Sending to the final destination vs. a relay server has
+	// slightly different semantics.
+	if method == SendDirect {
+		s.successCounter.Inc()
+		if s.blacklisted.Load() && s.statistics.DB != nil {
+			if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil {
+				logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName)
+			}
 		}
 	}
 }
@@ -144,7 +182,18 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 	// start a goroutine which will wait out the backoff and
 	// unset the backoffStarted flag when done.
 	if s.backoffStarted.CompareAndSwap(false, true) {
-		if s.backoffCount.Inc() >= s.statistics.FailuresUntilBlacklist {
+		backoffCount := s.backoffCount.Inc()
+
+		if backoffCount >= s.statistics.FailuresUntilAssumedOffline {
+			s.assumedOffline.CompareAndSwap(false, true)
+			if s.statistics.DB != nil {
+				if err := s.statistics.DB.SetServerAssumedOffline(context.Background(), s.serverName); err != nil {
+					logrus.WithError(err).Errorf("Failed to set %q as assumed offline", s.serverName)
+				}
+			}
+		}
+
+		if backoffCount >= s.statistics.FailuresUntilBlacklist {
 			s.blacklisted.Store(true)
 			if s.statistics.DB != nil {
 				if err := s.statistics.DB.AddServerToBlacklist(s.serverName); err != nil {
@@ -162,13 +211,21 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 		s.backoffUntil.Store(until)
 
 		s.statistics.backoffMutex.Lock()
-		defer s.statistics.backoffMutex.Unlock()
 		s.statistics.backoffTimers[s.serverName] = time.AfterFunc(time.Until(until), s.backoffFinished)
+		s.statistics.backoffMutex.Unlock()
 	}
 
 	return s.backoffUntil.Load().(time.Time), false
 }
 
+// MarkServerAlive removes the assumed offline and blacklisted statuses from this server.
+// Returns whether the server was blacklisted before this point.
+func (s *ServerStatistics) MarkServerAlive() bool {
+	s.removeAssumedOffline()
+	wasBlacklisted := s.removeBlacklist()
+	return wasBlacklisted
+}
+
 // ClearBackoff stops the backoff timer for this destination if it is running
 // and removes the timer from the backoffTimers map.
 func (s *ServerStatistics) ClearBackoff() {
@@ -196,13 +253,13 @@ func (s *ServerStatistics) backoffFinished() {
 }
 
 // BackoffInfo returns information about the current or previous backoff.
-// Returns the last backoffUntil time and whether the server is currently blacklisted or not.
-func (s *ServerStatistics) BackoffInfo() (*time.Time, bool) {
+// Returns the last backoffUntil time.
+func (s *ServerStatistics) BackoffInfo() *time.Time {
 	until, ok := s.backoffUntil.Load().(time.Time)
 	if ok {
-		return &until, s.blacklisted.Load()
+		return &until
 	}
-	return nil, s.blacklisted.Load()
+	return nil
 }
 
 // Blacklisted returns true if the server is blacklisted and false
@@ -211,10 +268,33 @@ func (s *ServerStatistics) Blacklisted() bool {
 	return s.blacklisted.Load()
 }
 
-// RemoveBlacklist removes the blacklisted status from the server.
-func (s *ServerStatistics) RemoveBlacklist() {
+// AssumedOffline returns true if the server is assumed offline and false
+// otherwise.
+func (s *ServerStatistics) AssumedOffline() bool {
+	return s.assumedOffline.Load()
+}
+
+// removeBlacklist removes the blacklisted status from the server.
+// Returns whether the server was blacklisted.
+func (s *ServerStatistics) removeBlacklist() bool {
+	var wasBlacklisted bool
+
+	if s.Blacklisted() {
+		wasBlacklisted = true
+		_ = s.statistics.DB.RemoveServerFromBlacklist(s.serverName)
+	}
 	s.cancel()
 	s.backoffCount.Store(0)
+
+	return wasBlacklisted
+}
+
+// removeAssumedOffline removes the assumed offline status from the server.
+func (s *ServerStatistics) removeAssumedOffline() {
+	if s.AssumedOffline() {
+		_ = s.statistics.DB.RemoveServerAssumedOffline(context.Background(), s.serverName)
+	}
+	s.assumedOffline.Store(false)
 }
 
 // SuccessCount returns the number of successful requests. This is
@@ -222,3 +302,46 @@ func (s *ServerStatistics) RemoveBlacklist() {
 func (s *ServerStatistics) SuccessCount() uint32 {
 	return s.successCounter.Load()
 }
+
+// KnownRelayServers returns the list of relay servers associated with this
+// server.
+func (s *ServerStatistics) KnownRelayServers() []gomatrixserverlib.ServerName {
+	s.relayMutex.Lock()
+	defer s.relayMutex.Unlock()
+	return s.knownRelayServers
+}
+
+func (s *ServerStatistics) AddRelayServers(relayServers []gomatrixserverlib.ServerName) {
+	seenSet := make(map[gomatrixserverlib.ServerName]bool)
+	uniqueList := []gomatrixserverlib.ServerName{}
+	for _, srv := range relayServers {
+		if seenSet[srv] {
+			continue
+		}
+		seenSet[srv] = true
+		uniqueList = append(uniqueList, srv)
+	}
+
+	err := s.statistics.DB.P2PAddRelayServersForServer(context.Background(), s.serverName, uniqueList)
+	if err != nil {
+		logrus.WithError(err).Errorf("Failed to add relay servers for %q. Servers: %v", s.serverName, uniqueList)
+		return
+	}
+
+	for _, newServer := range uniqueList {
+		alreadyKnown := false
+		knownRelayServers := s.KnownRelayServers()
+		for _, srv := range knownRelayServers {
+			if srv == newServer {
+				alreadyKnown = true
+			}
+		}
+		if !alreadyKnown {
+			{
+				s.relayMutex.Lock()
+				s.knownRelayServers = append(s.knownRelayServers, newServer)
+				s.relayMutex.Unlock()
+			}
+		}
+	}
+}

federationapi/statistics/statistics_test.go

@@ -4,17 +4,26 @@ import (
 	"math"
 	"testing"
 	"time"
+
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	FailuresUntilAssumedOffline = 3
+	FailuresUntilBlacklist      = 8
 )
 
 func TestBackoff(t *testing.T) {
-	stats := NewStatistics(nil, 7)
+	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
 	server := ServerStatistics{
 		statistics: &stats,
 		serverName: "test.com",
 	}
 
 	// Start by checking that counting successes works.
-	server.Success()
+	server.Success(SendDirect)
 	if successes := server.SuccessCount(); successes != 1 {
 		t.Fatalf("Expected success count 1, got %d", successes)
 	}
@@ -31,9 +40,8 @@ func TestBackoff(t *testing.T) {
 		// side effects since a backoff is already in progress. If it does
 		// then we'll fail.
 		until, blacklisted := server.Failure()
-
-		// Get the duration.
-		_, blacklist := server.BackoffInfo()
+		blacklist := server.Blacklisted()
+		assumedOffline := server.AssumedOffline()
 		duration := time.Until(until)
 
 		// Unset the backoff, or otherwise our next call will think that
@@ -41,16 +49,43 @@ func TestBackoff(t *testing.T) {
 		server.cancel()
 		server.backoffStarted.Store(false)
 
+		if i >= stats.FailuresUntilAssumedOffline {
+			if !assumedOffline {
+				t.Fatalf("Backoff %d should have resulted in assuming the destination was offline but didn't", i)
+			}
+		}
+
+		// Check if we should be assumed offline by now.
+		if i >= stats.FailuresUntilAssumedOffline {
+			if !assumedOffline {
+				t.Fatalf("Backoff %d should have resulted in assumed offline but didn't", i)
+			} else {
+				t.Logf("Backoff %d is assumed offline as expected", i)
+			}
+		} else {
+			if assumedOffline {
+				t.Fatalf("Backoff %d should not have resulted in assumed offline but did", i)
+			} else {
+				t.Logf("Backoff %d is not assumed offline as expected", i)
+			}
+		}
+
 		// Check if we should be blacklisted by now.
 		if i >= stats.FailuresUntilBlacklist {
 			if !blacklist {
 				t.Fatalf("Backoff %d should have resulted in blacklist but didn't", i)
 			} else if blacklist != blacklisted {
-				t.Fatalf("BackoffInfo and Failure returned different blacklist values")
+				t.Fatalf("Blacklisted and Failure returned different blacklist values")
 			} else {
 				t.Logf("Backoff %d is blacklisted as expected", i)
 				continue
 			}
+		} else {
+			if blacklist {
+				t.Fatalf("Backoff %d should not have resulted in blacklist but did", i)
+			} else {
+				t.Logf("Backoff %d is not blacklisted as expected", i)
+			}
 		}
 
 		// Check if the duration is what we expect.
@@ -69,3 +104,14 @@ func TestBackoff(t *testing.T) {
 		}
 	}
 }
+
+func TestRelayServersListing(t *testing.T) {
+	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	server := ServerStatistics{statistics: &stats}
+	server.AddRelayServers([]gomatrixserverlib.ServerName{"relay1", "relay1", "relay2"})
+	relayServers := server.KnownRelayServers()
+	assert.Equal(t, []gomatrixserverlib.ServerName{"relay1", "relay2"}, relayServers)
+	server.AddRelayServers([]gomatrixserverlib.ServerName{"relay1", "relay1", "relay2"})
+	relayServers = server.KnownRelayServers()
+	assert.Equal(t, []gomatrixserverlib.ServerName{"relay1", "relay2"}, relayServers)
+}

federationapi/storage/interface.go

@@ -20,11 +20,12 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/dendrite/federationapi/types"
 )
 
 type Database interface {
+	P2PDatabase
 	gomatrixserverlib.KeyDatabase
 
 	UpdateRoom(ctx context.Context, roomID string, addHosts []types.JoinedHost, removeHosts []string, purgeRoomFirst bool) (joinedHosts []types.JoinedHost, err error)
@@ -34,19 +35,16 @@ type Database interface {
 	// GetJoinedHostsForRooms returns the complete set of servers in the rooms given.
 	GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, excludeSelf, excludeBlacklisted bool) ([]gomatrixserverlib.ServerName, error)
 
-	StoreJSON(ctx context.Context, js string) (*shared.Receipt, error)
+	StoreJSON(ctx context.Context, js string) (*receipt.Receipt, error)
 
-	GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error)
-	GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error)
+	GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*receipt.Receipt]*gomatrixserverlib.HeaderedEvent, err error)
+	GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*receipt.Receipt]*gomatrixserverlib.EDU, err error)
 
-	AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt) error
-	AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error
+	AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, dbReceipt *receipt.Receipt) error
+	AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, dbReceipt *receipt.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error
 
-	CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
-	CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
-
-	GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error)
-	GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error)
+	CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*receipt.Receipt) error
+	CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*receipt.Receipt) error
 
 	GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
 	GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
@@ -57,6 +55,18 @@ type Database interface {
 	RemoveAllServersFromBlacklist() error
 	IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error)
 
+	// Adds the server to the list of assumed offline servers.
+	// If the server already exists in the table, nothing happens and returns success.
+	SetServerAssumedOffline(ctx context.Context, serverName gomatrixserverlib.ServerName) error
+	// Removes the server from the list of assumed offline servers.
+	// If the server doesn't exist in the table, nothing happens and returns success.
+	RemoveServerAssumedOffline(ctx context.Context, serverName gomatrixserverlib.ServerName) error
+	// Purges all entries from the assumed offline table.
+	RemoveAllServersAssumedOffline(ctx context.Context) error
+	// Gets whether the provided server is present in the table.
+	// If it is present, returns true. If not, returns false.
+	IsServerAssumedOffline(ctx context.Context, serverName gomatrixserverlib.ServerName) (bool, error)
+
 	AddOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error
 	RenewOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error
 	GetOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string) (*types.OutboundPeek, error)
@@ -74,4 +84,24 @@ type Database interface {
 	GetNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, optKeyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error)
 	// DeleteExpiredEDUs cleans up expired EDUs
 	DeleteExpiredEDUs(ctx context.Context) error
+
+	PurgeRoom(ctx context.Context, roomID string) error
+}
+
+type P2PDatabase interface {
+	// Stores the given list of servers as relay servers for the provided destination server.
+	// Providing duplicates will only lead to a single entry and won't lead to an error.
+	P2PAddRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName, relayServers []gomatrixserverlib.ServerName) error
+
+	// Get the list of relay servers associated with the provided destination server.
+	// If no entry exists in the table, an empty list is returned and does not result in an error.
+	P2PGetRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName) ([]gomatrixserverlib.ServerName, error)
+
+	// Deletes any entries for the provided destination server that match the provided relayServers list.
+	// If any of the provided servers don't match an entry, nothing happens and no error is returned.
+	P2PRemoveRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName, relayServers []gomatrixserverlib.ServerName) error
+
+	// Deletes all entries for the provided destination server.
+	// If the destination server doesn't exist in the table, nothing happens and no error is returned.
+	P2PRemoveAllRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName) error
 }

federationapi/storage/postgres/assumed_offline_table.go

@@ -0,0 +1,107 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const assumedOfflineSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_assumed_offline(
+    -- The assumed offline server name
+	server_name TEXT PRIMARY KEY NOT NULL
+);
+`
+
+const insertAssumedOfflineSQL = "" +
+	"INSERT INTO federationsender_assumed_offline (server_name) VALUES ($1)" +
+	" ON CONFLICT DO NOTHING"
+
+const selectAssumedOfflineSQL = "" +
+	"SELECT server_name FROM federationsender_assumed_offline WHERE server_name = $1"
+
+const deleteAssumedOfflineSQL = "" +
+	"DELETE FROM federationsender_assumed_offline WHERE server_name = $1"
+
+const deleteAllAssumedOfflineSQL = "" +
+	"TRUNCATE federationsender_assumed_offline"
+
+type assumedOfflineStatements struct {
+	db                          *sql.DB
+	insertAssumedOfflineStmt    *sql.Stmt
+	selectAssumedOfflineStmt    *sql.Stmt
+	deleteAssumedOfflineStmt    *sql.Stmt
+	deleteAllAssumedOfflineStmt *sql.Stmt
+}
+
+func NewPostgresAssumedOfflineTable(db *sql.DB) (s *assumedOfflineStatements, err error) {
+	s = &assumedOfflineStatements{
+		db: db,
+	}
+	_, err = db.Exec(assumedOfflineSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertAssumedOfflineStmt, insertAssumedOfflineSQL},
+		{&s.selectAssumedOfflineStmt, selectAssumedOfflineSQL},
+		{&s.deleteAssumedOfflineStmt, deleteAssumedOfflineSQL},
+		{&s.deleteAllAssumedOfflineStmt, deleteAllAssumedOfflineSQL},
+	}.Prepare(db)
+}
+
+func (s *assumedOfflineStatements) InsertAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.insertAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx, serverName)
+	return err
+}
+
+func (s *assumedOfflineStatements) SelectAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) (bool, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectAssumedOfflineStmt)
+	res, err := stmt.QueryContext(ctx, serverName)
+	if err != nil {
+		return false, err
+	}
+	defer res.Close() // nolint:errcheck
+	// The query will return the server name if the server is assume offline, and
+	// will return no rows if not. By calling Next, we find out if a row was
+	// returned or not - we don't care about the value itself.
+	return res.Next(), nil
+}
+
+func (s *assumedOfflineStatements) DeleteAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx, serverName)
+	return err
+}
+
+func (s *assumedOfflineStatements) DeleteAllAssumedOffline(
+	ctx context.Context, txn *sql.Tx,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteAllAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx)
+	return err
+}

federationapi/storage/postgres/inbound_peeks_table.go

@@ -44,13 +44,13 @@ const selectInboundPeekSQL = "" +
 	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_inbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const selectInboundPeeksSQL = "" +
-	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_inbound_peeks WHERE room_id = $1"
+	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_inbound_peeks WHERE room_id = $1 ORDER by creation_ts"
 
 const renewInboundPeekSQL = "" +
 	"UPDATE federationsender_inbound_peeks SET renewed_ts=$1, renewal_interval=$2 WHERE room_id = $3 and server_name = $4 and peek_id = $5"
 
 const deleteInboundPeekSQL = "" +
-	"DELETE FROM federationsender_inbound_peeks WHERE room_id = $1 and server_name = $2"
+	"DELETE FROM federationsender_inbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const deleteInboundPeeksSQL = "" +
 	"DELETE FROM federationsender_inbound_peeks WHERE room_id = $1"
@@ -74,25 +74,15 @@ func NewPostgresInboundPeeksTable(db *sql.DB) (s *inboundPeeksStatements, err er
 		return
 	}
 
-	if s.insertInboundPeekStmt, err = db.Prepare(insertInboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectInboundPeekStmt, err = db.Prepare(selectInboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectInboundPeeksStmt, err = db.Prepare(selectInboundPeeksSQL); err != nil {
-		return
-	}
-	if s.renewInboundPeekStmt, err = db.Prepare(renewInboundPeekSQL); err != nil {
-		return
-	}
-	if s.deleteInboundPeeksStmt, err = db.Prepare(deleteInboundPeeksSQL); err != nil {
-		return
-	}
-	if s.deleteInboundPeekStmt, err = db.Prepare(deleteInboundPeekSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertInboundPeekStmt, insertInboundPeekSQL},
+		{&s.selectInboundPeekStmt, selectInboundPeekSQL},
+		{&s.selectInboundPeekStmt, selectInboundPeekSQL},
+		{&s.selectInboundPeeksStmt, selectInboundPeeksSQL},
+		{&s.renewInboundPeekStmt, renewInboundPeekSQL},
+		{&s.deleteInboundPeeksStmt, deleteInboundPeeksSQL},
+		{&s.deleteInboundPeekStmt, deleteInboundPeekSQL},
+	}.Prepare(db)
 }
 
 func (s *inboundPeeksStatements) InsertInboundPeek(

federationapi/storage/postgres/outbound_peeks_table.go

@@ -44,13 +44,13 @@ const selectOutboundPeekSQL = "" +
 	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_outbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const selectOutboundPeeksSQL = "" +
-	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_outbound_peeks WHERE room_id = $1"
+	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_outbound_peeks WHERE room_id = $1 ORDER BY creation_ts"
 
 const renewOutboundPeekSQL = "" +
 	"UPDATE federationsender_outbound_peeks SET renewed_ts=$1, renewal_interval=$2 WHERE room_id = $3 and server_name = $4 and peek_id = $5"
 
 const deleteOutboundPeekSQL = "" +
-	"DELETE FROM federationsender_outbound_peeks WHERE room_id = $1 and server_name = $2"
+	"DELETE FROM federationsender_outbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const deleteOutboundPeeksSQL = "" +
 	"DELETE FROM federationsender_outbound_peeks WHERE room_id = $1"
@@ -74,25 +74,14 @@ func NewPostgresOutboundPeeksTable(db *sql.DB) (s *outboundPeeksStatements, err
 		return
 	}
 
-	if s.insertOutboundPeekStmt, err = db.Prepare(insertOutboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectOutboundPeekStmt, err = db.Prepare(selectOutboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectOutboundPeeksStmt, err = db.Prepare(selectOutboundPeeksSQL); err != nil {
-		return
-	}
-	if s.renewOutboundPeekStmt, err = db.Prepare(renewOutboundPeekSQL); err != nil {
-		return
-	}
-	if s.deleteOutboundPeeksStmt, err = db.Prepare(deleteOutboundPeeksSQL); err != nil {
-		return
-	}
-	if s.deleteOutboundPeekStmt, err = db.Prepare(deleteOutboundPeekSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertOutboundPeekStmt, insertOutboundPeekSQL},
+		{&s.selectOutboundPeekStmt, selectOutboundPeekSQL},
+		{&s.selectOutboundPeeksStmt, selectOutboundPeeksSQL},
+		{&s.renewOutboundPeekStmt, renewOutboundPeekSQL},
+		{&s.deleteOutboundPeeksStmt, deleteOutboundPeeksSQL},
+		{&s.deleteOutboundPeekStmt, deleteOutboundPeekSQL},
+	}.Prepare(db)
 }
 
 func (s *outboundPeeksStatements) InsertOutboundPeek(

federationapi/storage/postgres/queue_edus_table.go

@@ -62,10 +62,6 @@ const selectQueueEDUReferenceJSONCountSQL = "" +
 	"SELECT COUNT(*) FROM federationsender_queue_edus" +
 	" WHERE json_nid = $1"
 
-const selectQueueEDUCountSQL = "" +
-	"SELECT COUNT(*) FROM federationsender_queue_edus" +
-	" WHERE server_name = $1"
-
 const selectQueueServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_edus"
 
@@ -81,7 +77,6 @@ type queueEDUsStatements struct {
 	deleteQueueEDUStmt                   *sql.Stmt
 	selectQueueEDUStmt                   *sql.Stmt
 	selectQueueEDUReferenceJSONCountStmt *sql.Stmt
-	selectQueueEDUCountStmt              *sql.Stmt
 	selectQueueEDUServerNamesStmt        *sql.Stmt
 	selectExpiredEDUsStmt                *sql.Stmt
 	deleteExpiredEDUsStmt                *sql.Stmt
@@ -116,7 +111,6 @@ func (s *queueEDUsStatements) Prepare() error {
 		{&s.deleteQueueEDUStmt, deleteQueueEDUSQL},
 		{&s.selectQueueEDUStmt, selectQueueEDUSQL},
 		{&s.selectQueueEDUReferenceJSONCountStmt, selectQueueEDUReferenceJSONCountSQL},
-		{&s.selectQueueEDUCountStmt, selectQueueEDUCountSQL},
 		{&s.selectQueueEDUServerNamesStmt, selectQueueServerNamesSQL},
 		{&s.selectExpiredEDUsStmt, selectExpiredEDUsSQL},
 		{&s.deleteExpiredEDUsStmt, deleteExpiredEDUsSQL},
@@ -186,21 +180,6 @@ func (s *queueEDUsStatements) SelectQueueEDUReferenceJSONCount(
 	return count, err
 }
 
-func (s *queueEDUsStatements) SelectQueueEDUCount(
-	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
-) (int64, error) {
-	var count int64
-	stmt := sqlutil.TxStmt(txn, s.selectQueueEDUCountStmt)
-	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
-	if err == sql.ErrNoRows {
-		// It's acceptable for there to be no rows referencing a given
-		// JSON NID but it's not an error condition. Just return as if
-		// there's a zero count.
-		return 0, nil
-	}
-	return count, err
-}
-
 func (s *queueEDUsStatements) SelectQueueEDUServerNames(
 	ctx context.Context, txn *sql.Tx,
 ) ([]gomatrixserverlib.ServerName, error) {

federationapi/storage/postgres/queue_pdus_table.go

@@ -58,10 +58,6 @@ const selectQueuePDUReferenceJSONCountSQL = "" +
 	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
 	" WHERE json_nid = $1"
 
-const selectQueuePDUsCountSQL = "" +
-	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
-	" WHERE server_name = $1"
-
 const selectQueuePDUServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_pdus"
 
@@ -71,7 +67,6 @@ type queuePDUsStatements struct {
 	deleteQueuePDUsStmt                  *sql.Stmt
 	selectQueuePDUsStmt                  *sql.Stmt
 	selectQueuePDUReferenceJSONCountStmt *sql.Stmt
-	selectQueuePDUsCountStmt             *sql.Stmt
 	selectQueuePDUServerNamesStmt        *sql.Stmt
 }
 
@@ -95,9 +90,6 @@ func NewPostgresQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) {
 	if s.selectQueuePDUReferenceJSONCountStmt, err = s.db.Prepare(selectQueuePDUReferenceJSONCountSQL); err != nil {
 		return
 	}
-	if s.selectQueuePDUsCountStmt, err = s.db.Prepare(selectQueuePDUsCountSQL); err != nil {
-		return
-	}
 	if s.selectQueuePDUServerNamesStmt, err = s.db.Prepare(selectQueuePDUServerNamesSQL); err != nil {
 		return
 	}
@@ -146,21 +138,6 @@ func (s *queuePDUsStatements) SelectQueuePDUReferenceJSONCount(
 	return count, err
 }
 
-func (s *queuePDUsStatements) SelectQueuePDUCount(
-	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
-) (int64, error) {
-	var count int64
-	stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt)
-	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
-	if err == sql.ErrNoRows {
-		// It's acceptable for there to be no rows referencing a given
-		// JSON NID but it's not an error condition. Just return as if
-		// there's a zero count.
-		return 0, nil
-	}
-	return count, err
-}
-
 func (s *queuePDUsStatements) SelectQueuePDUs(
 	ctx context.Context, txn *sql.Tx,
 	serverName gomatrixserverlib.ServerName,

federationapi/storage/postgres/relay_servers_table.go

@@ -0,0 +1,137 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const relayServersSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_relay_servers (
+	-- The destination server name
+	server_name TEXT NOT NULL,
+	-- The relay server name for a given destination
+	relay_server_name TEXT NOT NULL,
+	UNIQUE (server_name, relay_server_name)
+);
+
+CREATE INDEX IF NOT EXISTS federationsender_relay_servers_server_name_idx
+	ON federationsender_relay_servers (server_name);
+`
+
+const insertRelayServersSQL = "" +
+	"INSERT INTO federationsender_relay_servers (server_name, relay_server_name) VALUES ($1, $2)" +
+	" ON CONFLICT DO NOTHING"
+
+const selectRelayServersSQL = "" +
+	"SELECT relay_server_name FROM federationsender_relay_servers WHERE server_name = $1"
+
+const deleteRelayServersSQL = "" +
+	"DELETE FROM federationsender_relay_servers WHERE server_name = $1 AND relay_server_name = ANY($2)"
+
+const deleteAllRelayServersSQL = "" +
+	"DELETE FROM federationsender_relay_servers WHERE server_name = $1"
+
+type relayServersStatements struct {
+	db                        *sql.DB
+	insertRelayServersStmt    *sql.Stmt
+	selectRelayServersStmt    *sql.Stmt
+	deleteRelayServersStmt    *sql.Stmt
+	deleteAllRelayServersStmt *sql.Stmt
+}
+
+func NewPostgresRelayServersTable(db *sql.DB) (s *relayServersStatements, err error) {
+	s = &relayServersStatements{
+		db: db,
+	}
+	_, err = db.Exec(relayServersSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertRelayServersStmt, insertRelayServersSQL},
+		{&s.selectRelayServersStmt, selectRelayServersSQL},
+		{&s.deleteRelayServersStmt, deleteRelayServersSQL},
+		{&s.deleteAllRelayServersStmt, deleteAllRelayServersSQL},
+	}.Prepare(db)
+}
+
+func (s *relayServersStatements) InsertRelayServers(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	relayServers []gomatrixserverlib.ServerName,
+) error {
+	for _, relayServer := range relayServers {
+		stmt := sqlutil.TxStmt(txn, s.insertRelayServersStmt)
+		if _, err := stmt.ExecContext(ctx, serverName, relayServer); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *relayServersStatements) SelectRelayServers(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+) ([]gomatrixserverlib.ServerName, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectRelayServersStmt)
+	rows, err := stmt.QueryContext(ctx, serverName)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectRelayServers: rows.close() failed")
+
+	var result []gomatrixserverlib.ServerName
+	for rows.Next() {
+		var relayServer string
+		if err = rows.Scan(&relayServer); err != nil {
+			return nil, err
+		}
+		result = append(result, gomatrixserverlib.ServerName(relayServer))
+	}
+	return result, nil
+}
+
+func (s *relayServersStatements) DeleteRelayServers(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	relayServers []gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteRelayServersStmt)
+	_, err := stmt.ExecContext(ctx, serverName, pq.Array(relayServers))
+	return err
+}
+
+func (s *relayServersStatements) DeleteAllRelayServers(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteAllRelayServersStmt)
+	if _, err := stmt.ExecContext(ctx, serverName); err != nil {
+		return err
+	}
+	return nil
+}

federationapi/storage/postgres/storage.go

@@ -62,6 +62,14 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	if err != nil {
 		return nil, err
 	}
+	assumedOffline, err := NewPostgresAssumedOfflineTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	relayServers, err := NewPostgresRelayServersTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	inboundPeeks, err := NewPostgresInboundPeeksTable(d.db)
 	if err != nil {
 		return nil, err
@@ -104,6 +112,8 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		FederationQueueEDUs:      queueEDUs,
 		FederationQueueJSON:      queueJSON,
 		FederationBlacklist:      blacklist,
+		FederationAssumedOffline: assumedOffline,
+		FederationRelayServers:   relayServers,
 		FederationInboundPeeks:   inboundPeeks,
 		FederationOutboundPeeks:  outboundPeeks,
 		NotaryServerKeysJSON:     notaryJSON,

federationapi/storage/shared/receipt/receipt.go

@@ -0,0 +1,42 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// A Receipt contains the NIDs of a call to GetNextTransactionPDUs/EDUs.
+// We don't actually export the NIDs but we need the caller to be able
+// to pass them back so that we can clean up if the transaction sends
+// successfully.
+
+package receipt
+
+import "fmt"
+
+// Receipt is a wrapper type used to represent a nid that corresponds to a unique row entry
+// in some database table.
+// The internal nid value cannot be modified after a Receipt has been created.
+// This guarantees a receipt will always refer to the same table entry that it was created
+// to represent.
+type Receipt struct {
+	nid int64
+}
+
+func NewReceipt(nid int64) Receipt {
+	return Receipt{nid: nid}
+}
+
+func (r *Receipt) GetNID() int64 {
+	return r.nid
+}
+
+func (r *Receipt) String() string {
+	return fmt.Sprintf("%d", r.nid)
+}

federationapi/storage/shared/storage.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/dendrite/federationapi/storage/tables"
 	"github.com/matrix-org/dendrite/federationapi/types"
 	"github.com/matrix-org/dendrite/internal/caching"
@@ -37,6 +38,8 @@ type Database struct {
 	FederationQueueJSON      tables.FederationQueueJSON
 	FederationJoinedHosts    tables.FederationJoinedHosts
 	FederationBlacklist      tables.FederationBlacklist
+	FederationAssumedOffline tables.FederationAssumedOffline
+	FederationRelayServers   tables.FederationRelayServers
 	FederationOutboundPeeks  tables.FederationOutboundPeeks
 	FederationInboundPeeks   tables.FederationInboundPeeks
 	NotaryServerKeysJSON     tables.FederationNotaryServerKeysJSON
@@ -44,22 +47,6 @@ type Database struct {
 	ServerSigningKeys        tables.FederationServerSigningKeys
 }
 
-// An Receipt contains the NIDs of a call to GetNextTransactionPDUs/EDUs.
-// We don't actually export the NIDs but we need the caller to be able
-// to pass them back so that we can clean up if the transaction sends
-// successfully.
-type Receipt struct {
-	nid int64
-}
-
-func NewReceipt(nid int64) Receipt {
-	return Receipt{nid: nid}
-}
-
-func (r *Receipt) String() string {
-	return fmt.Sprintf("%d", r.nid)
-}
-
 // UpdateRoom updates the joined hosts for a room and returns what the joined
 // hosts were before the update, or nil if this was a duplicate message.
 // This is called when we receive a message from kafka, so we pass in
@@ -113,11 +100,18 @@ func (d *Database) GetJoinedHosts(
 // GetAllJoinedHosts returns the currently joined hosts for
 // all rooms known to the federation sender.
 // Returns an error if something goes wrong.
-func (d *Database) GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error) {
+func (d *Database) GetAllJoinedHosts(
+	ctx context.Context,
+) ([]gomatrixserverlib.ServerName, error) {
 	return d.FederationJoinedHosts.SelectAllJoinedHosts(ctx)
 }
 
-func (d *Database) GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, excludeSelf, excludeBlacklisted bool) ([]gomatrixserverlib.ServerName, error) {
+func (d *Database) GetJoinedHostsForRooms(
+	ctx context.Context,
+	roomIDs []string,
+	excludeSelf,
+	excludeBlacklisted bool,
+) ([]gomatrixserverlib.ServerName, error) {
 	servers, err := d.FederationJoinedHosts.SelectJoinedHostsForRooms(ctx, roomIDs, excludeBlacklisted)
 	if err != nil {
 		return nil, err
@@ -139,7 +133,7 @@ func (d *Database) GetJoinedHostsForRooms(ctx context.Context, roomIDs []string,
 // metadata entries.
 func (d *Database) StoreJSON(
 	ctx context.Context, js string,
-) (*Receipt, error) {
+) (*receipt.Receipt, error) {
 	var nid int64
 	var err error
 	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
@@ -149,18 +143,21 @@ func (d *Database) StoreJSON(
 	if err != nil {
 		return nil, fmt.Errorf("d.insertQueueJSON: %w", err)
 	}
-	return &Receipt{
-		nid: nid,
-	}, nil
+	newReceipt := receipt.NewReceipt(nid)
+	return &newReceipt, nil
 }
 
-func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error {
+func (d *Database) AddServerToBlacklist(
+	serverName gomatrixserverlib.ServerName,
+) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationBlacklist.InsertBlacklist(context.TODO(), txn, serverName)
 	})
 }
 
-func (d *Database) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error {
+func (d *Database) RemoveServerFromBlacklist(
+	serverName gomatrixserverlib.ServerName,
+) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationBlacklist.DeleteBlacklist(context.TODO(), txn, serverName)
 	})
@@ -172,51 +169,166 @@ func (d *Database) RemoveAllServersFromBlacklist() error {
 	})
 }
 
-func (d *Database) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) {
+func (d *Database) IsServerBlacklisted(
+	serverName gomatrixserverlib.ServerName,
+) (bool, error) {
 	return d.FederationBlacklist.SelectBlacklist(context.TODO(), nil, serverName)
 }
 
-func (d *Database) AddOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error {
+func (d *Database) SetServerAssumedOffline(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.FederationAssumedOffline.InsertAssumedOffline(ctx, txn, serverName)
+	})
+}
+
+func (d *Database) RemoveServerAssumedOffline(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.FederationAssumedOffline.DeleteAssumedOffline(ctx, txn, serverName)
+	})
+}
+
+func (d *Database) RemoveAllServersAssumedOffline(
+	ctx context.Context,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.FederationAssumedOffline.DeleteAllAssumedOffline(ctx, txn)
+	})
+}
+
+func (d *Database) IsServerAssumedOffline(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+) (bool, error) {
+	return d.FederationAssumedOffline.SelectAssumedOffline(ctx, nil, serverName)
+}
+
+func (d *Database) P2PAddRelayServersForServer(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	relayServers []gomatrixserverlib.ServerName,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.FederationRelayServers.InsertRelayServers(ctx, txn, serverName, relayServers)
+	})
+}
+
+func (d *Database) P2PGetRelayServersForServer(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+) ([]gomatrixserverlib.ServerName, error) {
+	return d.FederationRelayServers.SelectRelayServers(ctx, nil, serverName)
+}
+
+func (d *Database) P2PRemoveRelayServersForServer(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	relayServers []gomatrixserverlib.ServerName,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.FederationRelayServers.DeleteRelayServers(ctx, txn, serverName, relayServers)
+	})
+}
+
+func (d *Database) P2PRemoveAllRelayServersForServer(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		return d.FederationRelayServers.DeleteAllRelayServers(ctx, txn, serverName)
+	})
+}
+
+func (d *Database) AddOutboundPeek(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	roomID string,
+	peekID string,
+	renewalInterval int64,
+) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationOutboundPeeks.InsertOutboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval)
 	})
 }
 
-func (d *Database) RenewOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error {
+func (d *Database) RenewOutboundPeek(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	roomID string,
+	peekID string,
+	renewalInterval int64,
+) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationOutboundPeeks.RenewOutboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval)
 	})
 }
 
-func (d *Database) GetOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string) (*types.OutboundPeek, error) {
+func (d *Database) GetOutboundPeek(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	roomID,
+	peekID string,
+) (*types.OutboundPeek, error) {
 	return d.FederationOutboundPeeks.SelectOutboundPeek(ctx, nil, serverName, roomID, peekID)
 }
 
-func (d *Database) GetOutboundPeeks(ctx context.Context, roomID string) ([]types.OutboundPeek, error) {
+func (d *Database) GetOutboundPeeks(
+	ctx context.Context,
+	roomID string,
+) ([]types.OutboundPeek, error) {
 	return d.FederationOutboundPeeks.SelectOutboundPeeks(ctx, nil, roomID)
 }
 
-func (d *Database) AddInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error {
+func (d *Database) AddInboundPeek(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	roomID string,
+	peekID string,
+	renewalInterval int64,
+) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationInboundPeeks.InsertInboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval)
 	})
 }
 
-func (d *Database) RenewInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error {
+func (d *Database) RenewInboundPeek(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	roomID string,
+	peekID string,
+	renewalInterval int64,
+) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.FederationInboundPeeks.RenewInboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval)
 	})
 }
 
-func (d *Database) GetInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string) (*types.InboundPeek, error) {
+func (d *Database) GetInboundPeek(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	roomID string,
+	peekID string,
+) (*types.InboundPeek, error) {
 	return d.FederationInboundPeeks.SelectInboundPeek(ctx, nil, serverName, roomID, peekID)
 }
 
-func (d *Database) GetInboundPeeks(ctx context.Context, roomID string) ([]types.InboundPeek, error) {
+func (d *Database) GetInboundPeeks(
+	ctx context.Context,
+	roomID string,
+) ([]types.InboundPeek, error) {
 	return d.FederationInboundPeeks.SelectInboundPeeks(ctx, nil, roomID)
 }
 
-func (d *Database) UpdateNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, serverKeys gomatrixserverlib.ServerKeys) error {
+func (d *Database) UpdateNotaryKeys(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	serverKeys gomatrixserverlib.ServerKeys,
+) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		validUntil := serverKeys.ValidUntilTS
 		// Servers MUST use the lesser of this field and 7 days into the future when determining if a key is valid.
@@ -251,7 +363,9 @@ func (d *Database) UpdateNotaryKeys(ctx context.Context, serverName gomatrixserv
 }
 
 func (d *Database) GetNotaryKeys(
-	ctx context.Context, serverName gomatrixserverlib.ServerName, optKeyIDs []gomatrixserverlib.KeyID,
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	optKeyIDs []gomatrixserverlib.KeyID,
 ) (sks []gomatrixserverlib.ServerKeys, err error) {
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		sks, err = d.NotaryServerKeysMetadata.SelectKeys(ctx, txn, serverName, optKeyIDs)
@@ -259,3 +373,18 @@ func (d *Database) GetNotaryKeys(
 	})
 	return sks, err
 }
+
+func (d *Database) PurgeRoom(ctx context.Context, roomID string) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		if err := d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil {
+			return fmt.Errorf("failed to purge joined hosts: %w", err)
+		}
+		if err := d.FederationInboundPeeks.DeleteInboundPeeks(ctx, txn, roomID); err != nil {
+			return fmt.Errorf("failed to purge inbound peeks: %w", err)
+		}
+		if err := d.FederationOutboundPeeks.DeleteOutboundPeeks(ctx, txn, roomID); err != nil {
+			return fmt.Errorf("failed to purge outbound peeks: %w", err)
+		}
+		return nil
+	})
+}

federationapi/storage/shared/storage_edus.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -41,7 +42,7 @@ var defaultExpireEDUTypes = map[string]time.Duration{
 func (d *Database) AssociateEDUWithDestinations(
 	ctx context.Context,
 	destinations map[gomatrixserverlib.ServerName]struct{},
-	receipt *Receipt,
+	dbReceipt *receipt.Receipt,
 	eduType string,
 	expireEDUTypes map[string]time.Duration,
 ) error {
@@ -62,12 +63,12 @@ func (d *Database) AssociateEDUWithDestinations(
 		var err error
 		for destination := range destinations {
 			err = d.FederationQueueEDUs.InsertQueueEDU(
-				ctx,         // context
-				txn,         // SQL transaction
-				eduType,     // EDU type for coalescing
-				destination, // destination server name
-				receipt.nid, // NID from the federationapi_queue_json table
-				expiresAt,   // The timestamp this EDU will expire
+				ctx,                // context
+				txn,                // SQL transaction
+				eduType,            // EDU type for coalescing
+				destination,        // destination server name
+				dbReceipt.GetNID(), // NID from the federationapi_queue_json table
+				expiresAt,          // The timestamp this EDU will expire
 			)
 		}
 		return err
@@ -81,10 +82,10 @@ func (d *Database) GetPendingEDUs(
 	serverName gomatrixserverlib.ServerName,
 	limit int,
 ) (
-	edus map[*Receipt]*gomatrixserverlib.EDU,
+	edus map[*receipt.Receipt]*gomatrixserverlib.EDU,
 	err error,
 ) {
-	edus = make(map[*Receipt]*gomatrixserverlib.EDU)
+	edus = make(map[*receipt.Receipt]*gomatrixserverlib.EDU)
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		nids, err := d.FederationQueueEDUs.SelectQueueEDUs(ctx, txn, serverName, limit)
 		if err != nil {
@@ -94,7 +95,8 @@ func (d *Database) GetPendingEDUs(
 		retrieve := make([]int64, 0, len(nids))
 		for _, nid := range nids {
 			if edu, ok := d.Cache.GetFederationQueuedEDU(nid); ok {
-				edus[&Receipt{nid}] = edu
+				newReceipt := receipt.NewReceipt(nid)
+				edus[&newReceipt] = edu
 			} else {
 				retrieve = append(retrieve, nid)
 			}
@@ -110,7 +112,8 @@ func (d *Database) GetPendingEDUs(
 			if err := json.Unmarshal(blob, &event); err != nil {
 				return fmt.Errorf("json.Unmarshal: %w", err)
 			}
-			edus[&Receipt{nid}] = &event
+			newReceipt := receipt.NewReceipt(nid)
+			edus[&newReceipt] = &event
 			d.Cache.StoreFederationQueuedEDU(nid, &event)
 		}
 
@@ -124,7 +127,7 @@ func (d *Database) GetPendingEDUs(
 func (d *Database) CleanEDUs(
 	ctx context.Context,
 	serverName gomatrixserverlib.ServerName,
-	receipts []*Receipt,
+	receipts []*receipt.Receipt,
 ) error {
 	if len(receipts) == 0 {
 		return errors.New("expected receipt")
@@ -132,7 +135,7 @@ func (d *Database) CleanEDUs(
 
 	nids := make([]int64, len(receipts))
 	for i := range receipts {
-		nids[i] = receipts[i].nid
+		nids[i] = receipts[i].GetNID()
 	}
 
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
@@ -162,15 +165,6 @@ func (d *Database) CleanEDUs(
 	})
 }
 
-// GetPendingEDUCount returns the number of EDUs waiting to be
-// sent for a given servername.
-func (d *Database) GetPendingEDUCount(
-	ctx context.Context,
-	serverName gomatrixserverlib.ServerName,
-) (int64, error) {
-	return d.FederationQueueEDUs.SelectQueueEDUCount(ctx, nil, serverName)
-}
-
 // GetPendingServerNames returns the server names that have EDUs
 // waiting to be sent.
 func (d *Database) GetPendingEDUServerNames(

federationapi/storage/shared/storage_pdus.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -30,17 +31,17 @@ import (
 func (d *Database) AssociatePDUWithDestinations(
 	ctx context.Context,
 	destinations map[gomatrixserverlib.ServerName]struct{},
-	receipt *Receipt,
+	dbReceipt *receipt.Receipt,
 ) error {
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		var err error
 		for destination := range destinations {
 			err = d.FederationQueuePDUs.InsertQueuePDU(
-				ctx,         // context
-				txn,         // SQL transaction
-				"",          // transaction ID
-				destination, // destination server name
-				receipt.nid, // NID from the federationapi_queue_json table
+				ctx,                // context
+				txn,                // SQL transaction
+				"",                 // transaction ID
+				destination,        // destination server name
+				dbReceipt.GetNID(), // NID from the federationapi_queue_json table
 			)
 		}
 		return err
@@ -54,7 +55,7 @@ func (d *Database) GetPendingPDUs(
 	serverName gomatrixserverlib.ServerName,
 	limit int,
 ) (
-	events map[*Receipt]*gomatrixserverlib.HeaderedEvent,
+	events map[*receipt.Receipt]*gomatrixserverlib.HeaderedEvent,
 	err error,
 ) {
 	// Strictly speaking this doesn't need to be using the writer
@@ -62,7 +63,7 @@ func (d *Database) GetPendingPDUs(
 	// a guarantee of transactional isolation, it's actually useful
 	// to know in SQLite mode that nothing else is trying to modify
 	// the database.
-	events = make(map[*Receipt]*gomatrixserverlib.HeaderedEvent)
+	events = make(map[*receipt.Receipt]*gomatrixserverlib.HeaderedEvent)
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		nids, err := d.FederationQueuePDUs.SelectQueuePDUs(ctx, txn, serverName, limit)
 		if err != nil {
@@ -72,7 +73,8 @@ func (d *Database) GetPendingPDUs(
 		retrieve := make([]int64, 0, len(nids))
 		for _, nid := range nids {
 			if event, ok := d.Cache.GetFederationQueuedPDU(nid); ok {
-				events[&Receipt{nid}] = event
+				newReceipt := receipt.NewReceipt(nid)
+				events[&newReceipt] = event
 			} else {
 				retrieve = append(retrieve, nid)
 			}
@@ -88,7 +90,8 @@ func (d *Database) GetPendingPDUs(
 			if err := json.Unmarshal(blob, &event); err != nil {
 				return fmt.Errorf("json.Unmarshal: %w", err)
 			}
-			events[&Receipt{nid}] = &event
+			newReceipt := receipt.NewReceipt(nid)
+			events[&newReceipt] = &event
 			d.Cache.StoreFederationQueuedPDU(nid, &event)
 		}
 
@@ -103,7 +106,7 @@ func (d *Database) GetPendingPDUs(
 func (d *Database) CleanPDUs(
 	ctx context.Context,
 	serverName gomatrixserverlib.ServerName,
-	receipts []*Receipt,
+	receipts []*receipt.Receipt,
 ) error {
 	if len(receipts) == 0 {
 		return errors.New("expected receipt")
@@ -111,7 +114,7 @@ func (d *Database) CleanPDUs(
 
 	nids := make([]int64, len(receipts))
 	for i := range receipts {
-		nids[i] = receipts[i].nid
+		nids[i] = receipts[i].GetNID()
 	}
 
 	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
@@ -141,15 +144,6 @@ func (d *Database) CleanPDUs(
 	})
 }
 
-// GetPendingPDUCount returns the number of PDUs waiting to be
-// sent for a given servername.
-func (d *Database) GetPendingPDUCount(
-	ctx context.Context,
-	serverName gomatrixserverlib.ServerName,
-) (int64, error) {
-	return d.FederationQueuePDUs.SelectQueuePDUCount(ctx, nil, serverName)
-}
-
 // GetPendingServerNames returns the server names that have PDUs
 // waiting to be sent.
 func (d *Database) GetPendingPDUServerNames(

federationapi/storage/sqlite3/assumed_offline_table.go

@@ -0,0 +1,107 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const assumedOfflineSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_assumed_offline(
+    -- The assumed offline server name
+	server_name TEXT PRIMARY KEY NOT NULL
+);
+`
+
+const insertAssumedOfflineSQL = "" +
+	"INSERT INTO federationsender_assumed_offline (server_name) VALUES ($1)" +
+	" ON CONFLICT DO NOTHING"
+
+const selectAssumedOfflineSQL = "" +
+	"SELECT server_name FROM federationsender_assumed_offline WHERE server_name = $1"
+
+const deleteAssumedOfflineSQL = "" +
+	"DELETE FROM federationsender_assumed_offline WHERE server_name = $1"
+
+const deleteAllAssumedOfflineSQL = "" +
+	"DELETE FROM federationsender_assumed_offline"
+
+type assumedOfflineStatements struct {
+	db                          *sql.DB
+	insertAssumedOfflineStmt    *sql.Stmt
+	selectAssumedOfflineStmt    *sql.Stmt
+	deleteAssumedOfflineStmt    *sql.Stmt
+	deleteAllAssumedOfflineStmt *sql.Stmt
+}
+
+func NewSQLiteAssumedOfflineTable(db *sql.DB) (s *assumedOfflineStatements, err error) {
+	s = &assumedOfflineStatements{
+		db: db,
+	}
+	_, err = db.Exec(assumedOfflineSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertAssumedOfflineStmt, insertAssumedOfflineSQL},
+		{&s.selectAssumedOfflineStmt, selectAssumedOfflineSQL},
+		{&s.deleteAssumedOfflineStmt, deleteAssumedOfflineSQL},
+		{&s.deleteAllAssumedOfflineStmt, deleteAllAssumedOfflineSQL},
+	}.Prepare(db)
+}
+
+func (s *assumedOfflineStatements) InsertAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.insertAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx, serverName)
+	return err
+}
+
+func (s *assumedOfflineStatements) SelectAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) (bool, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectAssumedOfflineStmt)
+	res, err := stmt.QueryContext(ctx, serverName)
+	if err != nil {
+		return false, err
+	}
+	defer res.Close() // nolint:errcheck
+	// The query will return the server name if the server is assume offline, and
+	// will return no rows if not. By calling Next, we find out if a row was
+	// returned or not - we don't care about the value itself.
+	return res.Next(), nil
+}
+
+func (s *assumedOfflineStatements) DeleteAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx, serverName)
+	return err
+}
+
+func (s *assumedOfflineStatements) DeleteAllAssumedOffline(
+	ctx context.Context, txn *sql.Tx,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteAllAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx)
+	return err
+}

federationapi/storage/sqlite3/inbound_peeks_table.go

@@ -44,13 +44,13 @@ const selectInboundPeekSQL = "" +
 	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_inbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const selectInboundPeeksSQL = "" +
-	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_inbound_peeks WHERE room_id = $1"
+	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_inbound_peeks WHERE room_id = $1 ORDER BY creation_ts"
 
 const renewInboundPeekSQL = "" +
 	"UPDATE federationsender_inbound_peeks SET renewed_ts=$1, renewal_interval=$2 WHERE room_id = $3 and server_name = $4 and peek_id = $5"
 
 const deleteInboundPeekSQL = "" +
-	"DELETE FROM federationsender_inbound_peeks WHERE room_id = $1 and server_name = $2"
+	"DELETE FROM federationsender_inbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const deleteInboundPeeksSQL = "" +
 	"DELETE FROM federationsender_inbound_peeks WHERE room_id = $1"
@@ -74,25 +74,15 @@ func NewSQLiteInboundPeeksTable(db *sql.DB) (s *inboundPeeksStatements, err erro
 		return
 	}
 
-	if s.insertInboundPeekStmt, err = db.Prepare(insertInboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectInboundPeekStmt, err = db.Prepare(selectInboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectInboundPeeksStmt, err = db.Prepare(selectInboundPeeksSQL); err != nil {
-		return
-	}
-	if s.renewInboundPeekStmt, err = db.Prepare(renewInboundPeekSQL); err != nil {
-		return
-	}
-	if s.deleteInboundPeeksStmt, err = db.Prepare(deleteInboundPeeksSQL); err != nil {
-		return
-	}
-	if s.deleteInboundPeekStmt, err = db.Prepare(deleteInboundPeekSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertInboundPeekStmt, insertInboundPeekSQL},
+		{&s.selectInboundPeekStmt, selectInboundPeekSQL},
+		{&s.selectInboundPeekStmt, selectInboundPeekSQL},
+		{&s.selectInboundPeeksStmt, selectInboundPeeksSQL},
+		{&s.renewInboundPeekStmt, renewInboundPeekSQL},
+		{&s.deleteInboundPeeksStmt, deleteInboundPeeksSQL},
+		{&s.deleteInboundPeekStmt, deleteInboundPeekSQL},
+	}.Prepare(db)
 }
 
 func (s *inboundPeeksStatements) InsertInboundPeek(

federationapi/storage/sqlite3/outbound_peeks_table.go

@@ -44,13 +44,13 @@ const selectOutboundPeekSQL = "" +
 	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_outbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const selectOutboundPeeksSQL = "" +
-	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_outbound_peeks WHERE room_id = $1"
+	"SELECT room_id, server_name, peek_id, creation_ts, renewed_ts, renewal_interval FROM federationsender_outbound_peeks WHERE room_id = $1 ORDER BY creation_ts"
 
 const renewOutboundPeekSQL = "" +
 	"UPDATE federationsender_outbound_peeks SET renewed_ts=$1, renewal_interval=$2 WHERE room_id = $3 and server_name = $4 and peek_id = $5"
 
 const deleteOutboundPeekSQL = "" +
-	"DELETE FROM federationsender_outbound_peeks WHERE room_id = $1 and server_name = $2"
+	"DELETE FROM federationsender_outbound_peeks WHERE room_id = $1 and server_name = $2 and peek_id = $3"
 
 const deleteOutboundPeeksSQL = "" +
 	"DELETE FROM federationsender_outbound_peeks WHERE room_id = $1"
@@ -74,25 +74,14 @@ func NewSQLiteOutboundPeeksTable(db *sql.DB) (s *outboundPeeksStatements, err er
 		return
 	}
 
-	if s.insertOutboundPeekStmt, err = db.Prepare(insertOutboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectOutboundPeekStmt, err = db.Prepare(selectOutboundPeekSQL); err != nil {
-		return
-	}
-	if s.selectOutboundPeeksStmt, err = db.Prepare(selectOutboundPeeksSQL); err != nil {
-		return
-	}
-	if s.renewOutboundPeekStmt, err = db.Prepare(renewOutboundPeekSQL); err != nil {
-		return
-	}
-	if s.deleteOutboundPeeksStmt, err = db.Prepare(deleteOutboundPeeksSQL); err != nil {
-		return
-	}
-	if s.deleteOutboundPeekStmt, err = db.Prepare(deleteOutboundPeekSQL); err != nil {
-		return
-	}
-	return
+	return s, sqlutil.StatementList{
+		{&s.insertOutboundPeekStmt, insertOutboundPeekSQL},
+		{&s.selectOutboundPeekStmt, selectOutboundPeekSQL},
+		{&s.selectOutboundPeeksStmt, selectOutboundPeeksSQL},
+		{&s.renewOutboundPeekStmt, renewOutboundPeekSQL},
+		{&s.deleteOutboundPeeksStmt, deleteOutboundPeeksSQL},
+		{&s.deleteOutboundPeekStmt, deleteOutboundPeekSQL},
+	}.Prepare(db)
 }
 
 func (s *outboundPeeksStatements) InsertOutboundPeek(

federationapi/storage/sqlite3/queue_edus_table.go

@@ -63,10 +63,6 @@ const selectQueueEDUReferenceJSONCountSQL = "" +
 	"SELECT COUNT(*) FROM federationsender_queue_edus" +
 	" WHERE json_nid = $1"
 
-const selectQueueEDUCountSQL = "" +
-	"SELECT COUNT(*) FROM federationsender_queue_edus" +
-	" WHERE server_name = $1"
-
 const selectQueueServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_edus"
 
@@ -82,7 +78,6 @@ type queueEDUsStatements struct {
 	// deleteQueueEDUStmt                *sql.Stmt - prepared at runtime due to variadic
 	selectQueueEDUStmt                   *sql.Stmt
 	selectQueueEDUReferenceJSONCountStmt *sql.Stmt
-	selectQueueEDUCountStmt              *sql.Stmt
 	selectQueueEDUServerNamesStmt        *sql.Stmt
 	selectExpiredEDUsStmt                *sql.Stmt
 	deleteExpiredEDUsStmt                *sql.Stmt
@@ -116,7 +111,6 @@ func (s *queueEDUsStatements) Prepare() error {
 		{&s.insertQueueEDUStmt, insertQueueEDUSQL},
 		{&s.selectQueueEDUStmt, selectQueueEDUSQL},
 		{&s.selectQueueEDUReferenceJSONCountStmt, selectQueueEDUReferenceJSONCountSQL},
-		{&s.selectQueueEDUCountStmt, selectQueueEDUCountSQL},
 		{&s.selectQueueEDUServerNamesStmt, selectQueueServerNamesSQL},
 		{&s.selectExpiredEDUsStmt, selectExpiredEDUsSQL},
 		{&s.deleteExpiredEDUsStmt, deleteExpiredEDUsSQL},
@@ -198,21 +192,6 @@ func (s *queueEDUsStatements) SelectQueueEDUReferenceJSONCount(
 	return count, err
 }
 
-func (s *queueEDUsStatements) SelectQueueEDUCount(
-	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
-) (int64, error) {
-	var count int64
-	stmt := sqlutil.TxStmt(txn, s.selectQueueEDUCountStmt)
-	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
-	if err == sql.ErrNoRows {
-		// It's acceptable for there to be no rows referencing a given
-		// JSON NID but it's not an error condition. Just return as if
-		// there's a zero count.
-		return 0, nil
-	}
-	return count, err
-}
-
 func (s *queueEDUsStatements) SelectQueueEDUServerNames(
 	ctx context.Context, txn *sql.Tx,
 ) ([]gomatrixserverlib.ServerName, error) {

federationapi/storage/sqlite3/queue_pdus_table.go

@@ -66,10 +66,6 @@ const selectQueuePDUsReferenceJSONCountSQL = "" +
 	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
 	" WHERE json_nid = $1"
 
-const selectQueuePDUsCountSQL = "" +
-	"SELECT COUNT(*) FROM federationsender_queue_pdus" +
-	" WHERE server_name = $1"
-
 const selectQueuePDUsServerNamesSQL = "" +
 	"SELECT DISTINCT server_name FROM federationsender_queue_pdus"
 
@@ -79,7 +75,6 @@ type queuePDUsStatements struct {
 	selectQueueNextTransactionIDStmt  *sql.Stmt
 	selectQueuePDUsStmt               *sql.Stmt
 	selectQueueReferenceJSONCountStmt *sql.Stmt
-	selectQueuePDUsCountStmt          *sql.Stmt
 	selectQueueServerNamesStmt        *sql.Stmt
 	// deleteQueuePDUsStmt *sql.Stmt - prepared at runtime due to variadic
 }
@@ -107,9 +102,6 @@ func NewSQLiteQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) {
 	if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueuePDUsReferenceJSONCountSQL); err != nil {
 		return
 	}
-	if s.selectQueuePDUsCountStmt, err = db.Prepare(selectQueuePDUsCountSQL); err != nil {
-		return
-	}
 	if s.selectQueueServerNamesStmt, err = db.Prepare(selectQueuePDUsServerNamesSQL); err != nil {
 		return
 	}
@@ -179,21 +171,6 @@ func (s *queuePDUsStatements) SelectQueuePDUReferenceJSONCount(
 	return count, err
 }
 
-func (s *queuePDUsStatements) SelectQueuePDUCount(
-	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
-) (int64, error) {
-	var count int64
-	stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt)
-	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
-	if err == sql.ErrNoRows {
-		// It's acceptable for there to be no rows referencing a given
-		// JSON NID but it's not an error condition. Just return as if
-		// there's a zero count.
-		return 0, nil
-	}
-	return count, err
-}
-
 func (s *queuePDUsStatements) SelectQueuePDUs(
 	ctx context.Context, txn *sql.Tx,
 	serverName gomatrixserverlib.ServerName,