Merge branch 'main' of github.com:matrix-org/dendrite into gh-pages

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -17,7 +17,6 @@ see: https://www.matrix.org/security-disclosure-policy/
 ### Background information
 <!-- Please include versions of all software when known e.g database versions, docker versions, client versions -->
 - **Dendrite version or git SHA**:
-- **Monolith or Polylith?**:
 - **SQLite3 or Postgres?**:
 - **Running in Docker?**:
 - **`go version`**:

.github/workflows/dendrite.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: "stable"
           cache: true
 
       - name: Install Node
@@ -62,14 +62,14 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: "stable"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
 
   # run go test with different go versions
   test:
     timeout-minutes: 10
-    name: Unit tests (Go ${{ matrix.go }})
+    name: Unit tests
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
     services:
@@ -91,25 +91,21 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-    strategy:
-      fail-fast: false
-      matrix:
-        go: ["1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: "stable"
       - uses: actions/cache@v3
         # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go${{ matrix.go }}-unit-
+            ${{ runner.os }}-go-stable-unit-
       - name: Set up gotestfmt
         uses: gotesttools/gotestfmt-action@v2
         with:
@@ -130,7 +126,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18", "1.19"]
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
@@ -138,15 +133,15 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: "stable"
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
+            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
       - name: Install dependencies x86
         if: ${{ matrix.goarch == '386' }}
         run: sudo apt update && sudo apt-get install -y gcc-multilib
@@ -164,23 +159,22 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.18", "1.19"]
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
       - uses: actions/checkout@v3
-      - name: Setup Go ${{ matrix.go }}
+      - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: "stable"
       - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
+            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
       - env:
@@ -206,7 +200,7 @@ jobs:
   integration:
     timeout-minutes: 20
     needs: initial-tests-done
-    name: Integration tests (Go ${{ matrix.go }})
+    name: Integration tests
     runs-on: ubuntu-latest
     # Service containers to run with `container-job`
     services:
@@ -228,16 +222,12 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-    strategy:
-      fail-fast: false
-      matrix:
-        go: ["1.19"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ matrix.go }}
+          go-version: "stable"
       - name: Set up gotestfmt
         uses: gotesttools/gotestfmt-action@v2
         with:
@@ -248,9 +238,9 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go${{ matrix.go }}-test-race-
+            ${{ runner.os }}-go-stable-test-race-
       - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
         env:
           POSTGRES_HOST: localhost
@@ -261,6 +251,7 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           flags: unittests
+          fail_ci_if_error: true
 
   # run database upgrade tests
   upgrade_test:
@@ -273,7 +264,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "stable"
           cache: true
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
@@ -293,7 +284,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "stable"
           cache: true
       - name: Build upgrade-tests
         run: go build ./cmd/dendrite-upgrade-tests
@@ -317,19 +308,9 @@ jobs:
           - label: SQLite Cgo
             cgo: 1
 
-          - label: SQLite native, full HTTP APIs
-            api: full-http
-
-          - label: SQLite Cgo, full HTTP APIs
-            api: full-http
-            cgo: 1
-
           - label: PostgreSQL
             postgres: postgres
 
-          - label: PostgreSQL, full HTTP APIs
-            postgres: postgres
-            api: full-http
     container:
       image: matrixdotorg/sytest-dendrite
       volumes:
@@ -338,7 +319,6 @@ jobs:
         - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
-        API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         CGO_ENABLED: ${{ matrix.cgo && 1 }}
     steps:
@@ -390,22 +370,9 @@ jobs:
           - label: SQLite Cgo
             cgo: 1
 
-          - label: SQLite native, full HTTP APIs
-            api: full-http
-            cgo: 0
-
-          - label: SQLite Cgo, full HTTP APIs
-            api: full-http
-            cgo: 1
-
           - label: PostgreSQL
             postgres: Postgres
             cgo: 0
-
-          - label: PostgreSQL, full HTTP APIs
-            postgres: Postgres
-            api: full-http
-            cgo: 0
     steps:
       # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
       # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@@ -447,7 +414,7 @@ jobs:
             (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
       # Build initial Dendrite image
-      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
+      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
         working-directory: dendrite
         env:
           DOCKER_BUILDKIT: 1
@@ -459,8 +426,8 @@ jobs:
         shell: bash
         name: Run Complement Tests
         env:
-          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
-          API: ${{ matrix.api && 1 }}
+          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
+          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
         working-directory: complement
 
   integration-tests-done:

.github/workflows/docker.yml

@@ -61,7 +61,6 @@ jobs:
           cache-to: type=gha,mode=max
           context: .
           build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -77,7 +76,6 @@ jobs:
           cache-to: type=gha,mode=max
           context: .
           build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: monolith
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -98,86 +96,6 @@ jobs:
         with:
           sarif_file: "trivy-results.sarif"
 
-  polylith:
-    name: Polylith image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-      security-events: write # To upload Trivy sarif files
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Login to GitHub Containers
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build main polylith image
-        if: github.ref_name == 'main'
-        id: docker_build_polylith
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: polylith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-
-      - name: Build release polylith image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_polylith_release
-        uses: docker/build-push-action@v3
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
-          target: polylith
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
-            ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
-            ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
-          format: "sarif"
-          output: "trivy-results.sarif"
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-
   demo-pinecone:
     name: Pinecone demo image
     runs-on: ubuntu-latest

.github/workflows/gh-pages.yml

@@ -4,7 +4,7 @@ name: Deploy GitHub Pages dependencies preinstalled
 on:
   # Runs on pushes targeting the default branch
   push:
-    branches: ["main", "gh-pages"]
+    branches: ["gh-pages"]
     paths:
       - 'docs/**' # only execute if we have docs changes
 

.github/workflows/schedules.yaml

@@ -19,17 +19,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - label: SQLite
+          - label: SQLite native
 
-          - label: SQLite, full HTTP APIs
-            api: full-http
+          - label: SQLite Cgo
+            cgo: 1
 
           - label: PostgreSQL
             postgres: postgres
-
-          - label: PostgreSQL, full HTTP APIs
-            postgres: postgres
-            api: full-http
     container:
       image: matrixdotorg/sytest-dendrite:latest
       volumes:
@@ -38,9 +34,9 @@ jobs:
         - /root/.cache/go-mod:/gopath/pkg/mod
       env:
         POSTGRES: ${{ matrix.postgres && 1}}
-        API: ${{ matrix.api && 1 }}
         SYTEST_BRANCH: ${{ github.head_ref }}
         RACE_DETECTION: 1
+        COVER: 1
     steps:
       - uses: actions/checkout@v3
       - uses: actions/cache@v3
@@ -73,3 +69,192 @@ jobs:
           path: |
             /logs/results.tap
             /logs/**/*.log*
+
+  sytest-coverage:
+    timeout-minutes: 5
+    name: "Sytest Coverage"
+    runs-on: ubuntu-latest
+    needs: sytest # only run once Sytest is done
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+          cache: true
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+      - name: Install gocovmerge
+        run: go install github.com/wadey/gocovmerge@latest
+      - name: Run gocovmerge
+        run: |
+          find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
+          go tool cover -func=sytest.cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./sytest.cov
+          flags: sytest
+          fail_ci_if_error: true
+
+  # run Complement
+  complement:
+    name: "Complement (${{ matrix.label }})"
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite native
+            cgo: 0
+
+          - label: SQLite Cgo
+            cgo: 1
+
+          - label: PostgreSQL
+            postgres: Postgres
+            cgo: 0
+    steps:
+      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
+      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+      - name: "Set Go Version"
+        run: |
+          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+          echo "~/go/bin" >> $GITHUB_PATH
+      - name: "Install Complement Dependencies"
+        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
+        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
+        run: |
+          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+          go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+      - name: Run actions/checkout@v3 for dendrite
+        uses: actions/checkout@v3
+        with:
+          path: dendrite
+
+      # Attempt to check out the same branch of Complement as the PR. If it
+      # doesn't exist, fallback to main.
+      - name: Checkout complement
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
+      # Build initial Dendrite image
+      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
+        working-directory: dendrite
+        env:
+          DOCKER_BUILDKIT: 1
+
+      - name: Create post test script
+        run: |
+          cat <<EOF > /tmp/posttest.sh
+          #!/bin/bash
+          mkdir -p /tmp/Complement/logs/\$2/\$1/
+          docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
+          EOF
+          
+          chmod +x /tmp/posttest.sh
+      # Run Complement
+      - run: |
+          set -o pipefail &&
+          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+        shell: bash
+        name: Run Complement Tests
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
+          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
+          COMPLEMENT_DENDRITE_COVER: 1
+          COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh
+        working-directory: complement
+
+      - name: Upload Complement logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /tmp/Complement/**/complementcover.log
+
+  complement-coverage:
+    timeout-minutes: 5
+    name: "Complement Coverage"
+    runs-on: ubuntu-latest
+    needs: complement # only run once Complement is done
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 'stable'
+          cache: true
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+      - name: Install gocovmerge
+        run: go install github.com/wadey/gocovmerge@latest
+      - name: Run gocovmerge
+        run: |
+          find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
+          go tool cover -func=complement.cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./complement.cov
+          flags: complement
+          fail_ci_if_error: true
+
+  element_web:
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    steps:
+      - uses: tecolicom/actions-use-apt-tools@v1
+        with:
+          # Our test suite includes some screenshot tests with unusual diacritics, which are
+          # supposed to be covered by STIXGeneral.
+          tools: fonts-stix
+      - uses: actions/checkout@v2
+        with:
+          repository: matrix-org/matrix-react-sdk
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+      - name: Fetch layered build
+        run: scripts/ci/layered.sh
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+      - name: Build
+        env:
+          CI_PACKAGE: true
+          NODE_OPTIONS: "--openssl-legacy-provider"
+        run: yarn build
+        working-directory: ./element-web
+      - name: Edit Test Config
+        run: |
+          sed -i '/HOMESERVER/c\        HOMESERVER: "dendrite",' cypress.config.ts
+      - name: "Run cypress tests"
+        uses: cypress-io/github-action@v4.1.1
+        with:
+          browser: chrome
+          start: npx serve -p 8080 ./element-web/webapp
+          wait-on: 'http://localhost:8080'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          TMPDIR: ${{ runner.temp }}

.gitignore

@@ -56,6 +56,7 @@ dendrite.yaml
 
 # Database files
 *.db
+*.db-journal
 
 # Log files
 *.log*

CHANGES.md

@@ -1,5 +1,55 @@
 # Changelog
 
+## Dendrite 0.11.1 (2023-02-10)
+
+**⚠️ DEPRECATION WARNING: This is the last release to have polylith and HTTP API mode. Future releases are monolith only.**
+
+### Features
+
+* Dendrite can now be compiled against Go 1.20
+* Initial store and forward support has been added
+* A landing page showing that Dendrite is running has been added (contributed by [LukasLJL](https://github.com/LukasLJL))
+
+### Fixes
+
+- `/sync` is now using significantly less database round trips when using Postgres, resulting in faster initial syncs, allowing larger accounts to login again
+- Many under the hood pinecone improvements
+- Publishing rooms is now possible again
+
+## Dendrite 0.11.0 (2023-01-20)
+
+The last three missing federation API Sytests have been fixed - bringing us to 100% server-server Synapse parity, with client-server parity at 93% 🎉
+
+### Features
+
+* Added `/_dendrite/admin/purgeRoom/{roomID}` to clean up the database
+* The default room version was updated to 10 (contributed by [FSG-Cat](https://github.com/FSG-Cat))
+
+### Fixes
+
+* An oversight in the `create-config` binary, which now correctly sets the media path if specified (contributed by [BieHDC](https://github.com/BieHDC))
+* The Helm chart now uses the `$.Chart.AppVersion` as the default image version to pull, with the possibility to override it (contributed by [genofire](https://github.com/genofire))
+
+## Dendrite 0.10.9 (2023-01-17)
+
+### Features
+
+* Stale device lists are now cleaned up on startup, removing entries for users the server doesn't share a room with anymore
+* Dendrite now has its own Helm chart
+* Guest access is now handled correctly (disallow joins, kick guests on revocation of guest access, as well as over federation)
+
+### Fixes
+
+* Push rules have seen several tweaks and fixes, which should, for example, fix notifications for `m.read_receipts`
+* Outgoing presence will now correctly be sent to newly joined hosts
+* Fixes the `/_dendrite/admin/resetPassword/{userID}` admin endpoint to use the correct variable
+* Federated backfilling for medium/large rooms has been fixed
+* `/login` causing wrong device list updates has been resolved
+* `/sync` should now return the correct room summary heroes
+* The default config options for `recaptcha_sitekey_class` and `recaptcha_form_field` are now set correctly 
+* `/messages` now omits empty `state` to be more spec compliant (contributed by [handlerug](https://github.com/handlerug))
+* `/sync` has been optimised to only query state events for history visibility if they are really needed
+
 ## Dendrite 0.10.8 (2022-11-29)
 
 ### Features

Dockerfile

@@ -3,7 +3,7 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base
 RUN apk --update --no-cache add bash build-base curl
 
 #
@@ -23,44 +23,27 @@ RUN --mount=target=. \
     CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
     go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
 
+
 #
-# The dendrite base image
+# Builds the Dendrite image containing all required binaries
 #
-FROM alpine:latest AS dendrite-base
+FROM alpine:latest
 RUN apk --update --no-cache add curl
+LABEL org.opencontainers.image.title="Dendrite"
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
 LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
 
-#
-# Builds the polylith image and only contains the polylith binary
-#
-FROM dendrite-base AS polylith
-LABEL org.opencontainers.image.title="Dendrite (Polylith)"
-
-COPY --from=build /out/dendrite-polylith-multi /usr/bin/
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
-
-#
-# Builds the monolith image and contains all required binaries
-#
-FROM dendrite-base AS monolith
-LABEL org.opencontainers.image.title="Dendrite (Monolith)"
-
 COPY --from=build /out/create-account /usr/bin/create-account
 COPY --from=build /out/generate-config /usr/bin/generate-config
 COPY --from=build /out/generate-keys /usr/bin/generate-keys
-COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
+COPY --from=build /out/dendrite /usr/bin/dendrite
 
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
+ENTRYPOINT ["/usr/bin/dendrite"]
 EXPOSE 8008 8448
 

README.md

@@ -24,7 +24,6 @@ This does not mean:
 - Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support.
 
 Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
-In the future, we will be able to scale up to gigantic servers (equivalent to `matrix.org`) via polylith mode.
 
 If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in:
 
@@ -88,7 +87,7 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
 
 We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
 test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
-updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
+updates with CI. As of January 2023, we have 100% server-server parity with Synapse, and the client-server parity is at 93% , though check
 CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
 servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs).
 

appservice/appservice.go

@@ -21,14 +21,12 @@ import (
 	"sync"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/gomatrixserverlib"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/appservice/consumers"
-	"github.com/matrix-org/dendrite/appservice/inthttp"
 	"github.com/matrix-org/dendrite/appservice/query"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -36,16 +34,11 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
-// AddInternalRoutes registers HTTP handlers for internal API calls
-func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI, enableMetrics bool) {
-	inthttp.AddRoutes(queryAPI, router, enableMetrics)
-}
-
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.AppserviceUserAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
 	client := &http.Client{

appservice/appservice_test.go

@@ -10,12 +10,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/gorilla/mux"
-
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/appservice/inthttp"
-	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
@@ -129,26 +125,10 @@ func TestAppserviceInternalAPI(t *testing.T) {
 
 		// Create required internal APIs
 		rsAPI := roomserver.NewInternalAPI(base)
-		usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil)
+		usrAPI := userapi.NewInternalAPI(base, rsAPI, nil)
 		asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
 
-		// Finally execute the tests
-		t.Run("HTTP API", func(t *testing.T) {
-			router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
-			appservice.AddInternalRoutes(router, asAPI, base.EnableMetrics)
-			apiURL, cancel := test.ListenAndServe(t, router, false)
-			defer cancel()
-
-			asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{})
-			if err != nil {
-				t.Fatalf("failed to create HTTP client: %s", err)
-			}
-			runCases(t, asHTTPApi)
-		})
-
-		t.Run("Monolith", func(t *testing.T) {
-			runCases(t, asAPI)
-		})
+		runCases(t, asAPI)
 	})
 }
 

appservice/inthttp/client.go

@@ -1,84 +0,0 @@
-package inthttp
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-)
-
-// HTTP paths for the internal HTTP APIs
-const (
-	AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
-	AppServiceUserIDExistsPath    = "/appservice/UserIDExists"
-	AppServiceLocationsPath       = "/appservice/locations"
-	AppServiceUserPath            = "/appservice/users"
-	AppServiceProtocolsPath       = "/appservice/protocols"
-)
-
-// httpAppServiceQueryAPI contains the URL to an appservice query API and a
-// reference to a httpClient used to reach it
-type httpAppServiceQueryAPI struct {
-	appserviceURL string
-	httpClient    *http.Client
-}
-
-// NewAppserviceClient creates a AppServiceQueryAPI implemented by talking
-// to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewAppserviceClient(
-	appserviceURL string,
-	httpClient *http.Client,
-) (api.AppServiceInternalAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
-	}
-	return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil
-}
-
-// RoomAliasExists implements AppServiceQueryAPI
-func (h *httpAppServiceQueryAPI) RoomAliasExists(
-	ctx context.Context,
-	request *api.RoomAliasExistsRequest,
-	response *api.RoomAliasExistsResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"RoomAliasExists", h.appserviceURL+AppServiceRoomAliasExistsPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// UserIDExists implements AppServiceQueryAPI
-func (h *httpAppServiceQueryAPI) UserIDExists(
-	ctx context.Context,
-	request *api.UserIDExistsRequest,
-	response *api.UserIDExistsResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"UserIDExists", h.appserviceURL+AppServiceUserIDExistsPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error {
-	return httputil.CallInternalRPCAPI(
-		"ASLocation", h.appserviceURL+AppServiceLocationsPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error {
-	return httputil.CallInternalRPCAPI(
-		"ASUser", h.appserviceURL+AppServiceUserPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error {
-	return httputil.CallInternalRPCAPI(
-		"ASProtocols", h.appserviceURL+AppServiceProtocolsPath,
-		h.httpClient, ctx, request, response,
-	)
-}

appservice/inthttp/server.go

@@ -1,36 +0,0 @@
-package inthttp
-
-import (
-	"github.com/gorilla/mux"
-
-	"github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-)
-
-// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
-func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
-	internalAPIMux.Handle(
-		AppServiceRoomAliasExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", enableMetrics, a.RoomAliasExists),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceUserIDExistsPath,
-		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", enableMetrics, a.UserIDExists),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceProtocolsPath,
-		httputil.MakeInternalRPCAPI("AppserviceProtocols", enableMetrics, a.Protocols),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceLocationsPath,
-		httputil.MakeInternalRPCAPI("AppserviceLocations", enableMetrics, a.Locations),
-	)
-
-	internalAPIMux.Handle(
-		AppServiceUserPath,
-		httputil.MakeInternalRPCAPI("AppserviceUser", enableMetrics, a.User),
-	)
-}

are-we-synapse-yet.list

@@ -937,3 +937,11 @@ fst Room state after a rejected state event is the same as before
 fpb Federation publicRoom Name/topic keys are correct
 fed New federated private chats get full presence information (SYN-115) (10 subtests)
 dvk Rejects invalid device keys
+rmv User can create and send/receive messages in a room with version 10
+rmv local user can join room with version 10
+rmv User can invite local user to room with version 10
+rmv remote user can join room with version 10
+rmv User can invite remote user to room with version 10
+rmv Remote user can backfill in a room with version 10
+rmv Can reject invites over federation for rooms with version 10
+rmv Can receive redactions from regular users over federation in room version 10

build/dendritejs-pinecone/main.go

@@ -30,7 +30,6 @@ import (
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -177,19 +176,17 @@ func startup() {
 	if err := cfg.Derive(); err != nil {
 		logrus.Fatalf("Failed to derive values from config: %s", err)
 	}
-	base := base.NewBaseDendrite(cfg, "Monolith")
+	base := base.NewBaseDendrite(cfg)
 	defer base.Close() // nolint: errcheck
 
 	rsAPI := roomserver.NewInternalAPI(base)
 
 	federation := conn.CreateFederationClient(base, pSessions)
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI)
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
+	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
 
 	asQuery := appservice.NewInternalAPI(
 		base, userAPI, rsAPI,
@@ -208,14 +205,12 @@ func startup() {
 		FederationAPI: fedSenderAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
 		//ServerKeyAPI:        serverKeyAPI,
 		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
 	}
 	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 

build/docker/README.md

@@ -5,7 +5,6 @@ These are Docker images for Dendrite!
 They can be found on Docker Hub:
 
 - [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments
-- [matrixdotorg/dendrite-polylith](https://hub.docker.com/r/matrixdotorg/dendrite-polylith) for polylith deployments
 
 ## Dockerfiles
 
@@ -15,7 +14,6 @@ repository, run:
 
 ```
 docker build . --target monolith -t matrixdotorg/dendrite-monolith
-docker build . --target polylith -t matrixdotorg/dendrite-monolith
 docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone
 docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
 ```
@@ -25,7 +23,6 @@ docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
 There are two sample `docker-compose` files:
 
 - `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
-- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment
 
 ## Configuration
 
@@ -51,9 +48,9 @@ docker run --rm --entrypoint="" \
 
 The key files will now exist in your current working directory, and can be mounted into place.
 
-## Starting Dendrite as a monolith deployment
+## Starting Dendrite
 
-Create your config based on the [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) sample configuration file.
+Create your config based on the [`dendrite-sample.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.yaml) sample configuration file.
 
 Then start the deployment:
 
@@ -61,16 +58,6 @@ Then start the deployment:
 docker-compose -f docker-compose.monolith.yml up
 ```
 
-## Starting Dendrite as a polylith deployment
-
-Create your config based on the [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) sample configuration file.
-
-Then start the deployment:
-
-```
-docker-compose -f docker-compose.polylith.yml up
-```
-
 ## Building the images
 
 The `build/docker/images-build.sh` script will build the base image, followed by

build/docker/docker-compose.polylith.yml

@@ -1,143 +0,0 @@
-version: "3.4"
-services:
-  postgres:
-    hostname: postgres
-    image: postgres:14
-    restart: always
-    volumes:
-      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
-      # To persist your PostgreSQL databases outside of the Docker image, 
-      # to prevent data loss, modify the following ./path_to path:
-      - ./path_to/postgresql:/var/lib/postgresql/data
-    environment:
-      POSTGRES_PASSWORD: itsasecret
-      POSTGRES_USER: dendrite
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U dendrite"]
-      interval: 5s
-      timeout: 5s
-      retries: 5
-    networks:
-      - internal
-
-  jetstream:
-    hostname: jetstream
-    image: nats:latest
-    command: |
-      --jetstream
-      --store_dir /var/lib/nats
-      --cluster_name Dendrite
-    volumes:
-      # To persist your NATS JetStream streams outside of the Docker image,
-      # prevent data loss, modify the following ./path_to path:
-      - ./path_to/nats:/var/lib/nats
-    networks:
-      - internal
-
-  client_api:
-    hostname: client_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: clientapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  media_api:
-    hostname: media_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: mediaapi
-    volumes:
-      - ./config:/etc/dendrite
-      - ./media:/var/dendrite/media
-    networks:
-      - internal
-    restart: unless-stopped
-
-  sync_api:
-    hostname: sync_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: syncapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  room_server:
-    hostname: room_server
-    image: matrixdotorg/dendrite-polylith:latest
-    command: roomserver
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  federation_api:
-    hostname: federation_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: federationapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-      
-  key_server:
-    hostname: key_server
-    image: matrixdotorg/dendrite-polylith:latest
-    command: keyserver
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-      
-  user_api:
-    hostname: user_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: userapi
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - jetstream
-      - postgres
-    networks:
-      - internal
-    restart: unless-stopped
-
-  appservice_api:
-    hostname: appservice_api
-    image: matrixdotorg/dendrite-polylith:latest
-    command: appservice
-    volumes:
-      - ./config:/etc/dendrite
-    networks:
-      - internal
-    depends_on:
-      - jetstream
-      - postgres
-      - room_server
-      - user_api
-    restart: unless-stopped
-
-networks:
-  internal:
-    attachable: true

build/docker/images-build.sh

@@ -7,6 +7,5 @@ TAG=${1:-latest}
 echo "Building tag '${TAG}'"
 
 docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
-docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
 docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
 docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG}

build/docker/images-pull.sh

@@ -5,4 +5,3 @@ TAG=${1:-latest}
 echo "Pulling tag '${TAG}'"
 
 docker pull matrixdotorg/dendrite-monolith:${TAG}
-docker pull matrixdotorg/dendrite-polylith:${TAG}

build/docker/images-push.sh

@@ -5,4 +5,3 @@ TAG=${1:-latest}
 echo "Pushing tag '${TAG}'"
 
 docker push matrixdotorg/dendrite-monolith:${TAG}
-docker push matrixdotorg/dendrite-polylith:${TAG}

build/gobind-pinecone/build.sh

@@ -7,7 +7,7 @@ do
     case "$option"
     in
     a) gomobile bind -v -target android -trimpath -ldflags="-s -w" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
-    i) gomobile bind -v -target ios -trimpath -ldflags="" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
+    i) gomobile bind -v -target ios -trimpath -ldflags="" -o ~/DendriteBindings/Gobind.xcframework . ;;
     *) echo "No target specified, specify -a or -i"; exit 1 ;;
     esac
 done

build/gobind-pinecone/monolith.go

@@ -18,50 +18,25 @@ import (
 	"context"
 	"crypto/ed25519"
 	"crypto/rand"
-	"crypto/tls"
 	"encoding/hex"
 	"fmt"
-	"io"
 	"net"
-	"net/http"
-	"os"
 	"path/filepath"
 	"strings"
-	"sync"
-	"time"
 
-	"go.uber.org/atomic"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conduit"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/monolith"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/relay"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/process"
-	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/userapi"
 	userapiAPI "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/pinecone/types"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/net/http2"
-	"golang.org/x/net/http2/h2c"
 
-	pineconeConnections "github.com/matrix-org/pinecone/connections"
 	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
 	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeEvents "github.com/matrix-org/pinecone/router/events"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-	"github.com/matrix-org/pinecone/types"
 
 	_ "golang.org/x/mobile/bind"
 )
@@ -71,36 +46,37 @@ const (
 	PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
 	PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
 	PeerTypeBonjour   = pineconeRouter.PeerTypeBonjour
+
+	MaxFrameSize = types.MaxFrameSize
 )
 
+// Re-export Conduit in this package for bindings.
+type Conduit struct {
+	conduit.Conduit
+}
+
 type DendriteMonolith struct {
-	logger            logrus.Logger
-	PineconeRouter    *pineconeRouter.Router
-	PineconeMulticast *pineconeMulticast.Multicast
-	PineconeQUIC      *pineconeSessions.Sessions
-	PineconeManager   *pineconeConnections.ConnectionManager
-	StorageDirectory  string
-	CacheDirectory    string
-	listener          net.Listener
-	httpServer        *http.Server
-	processContext    *process.ProcessContext
-	userAPI           userapiAPI.UserInternalAPI
+	logger           logrus.Logger
+	p2pMonolith      monolith.P2PMonolith
+	StorageDirectory string
+	CacheDirectory   string
+	listener         net.Listener
 }
 
 func (m *DendriteMonolith) PublicKey() string {
-	return m.PineconeRouter.PublicKey().String()
+	return m.p2pMonolith.Router.PublicKey().String()
 }
 
 func (m *DendriteMonolith) BaseURL() string {
-	return fmt.Sprintf("http://%s", m.listener.Addr().String())
+	return fmt.Sprintf("http://%s", m.p2pMonolith.Addr())
 }
 
 func (m *DendriteMonolith) PeerCount(peertype int) int {
-	return m.PineconeRouter.PeerCount(peertype)
+	return m.p2pMonolith.Router.PeerCount(peertype)
 }
 
 func (m *DendriteMonolith) SessionCount() int {
-	return len(m.PineconeQUIC.Protocol("matrix").Sessions())
+	return len(m.p2pMonolith.Sessions.Protocol(monolith.SessionProtocol).Sessions())
 }
 
 type InterfaceInfo struct {
@@ -142,55 +118,156 @@ func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriev
 		}
 		return intfs
 	}
-	m.PineconeMulticast.RegisterNetworkCallback(callback)
+	m.p2pMonolith.Multicast.RegisterNetworkCallback(callback)
 }
 
 func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 	if enabled {
-		m.PineconeMulticast.Start()
+		m.p2pMonolith.Multicast.Start()
 	} else {
-		m.PineconeMulticast.Stop()
+		m.p2pMonolith.Multicast.Stop()
 		m.DisconnectType(int(pineconeRouter.PeerTypeMulticast))
 	}
 }
 
 func (m *DendriteMonolith) SetStaticPeer(uri string) {
-	m.PineconeManager.RemovePeers()
+	m.p2pMonolith.ConnManager.RemovePeers()
 	for _, uri := range strings.Split(uri, ",") {
-		m.PineconeManager.AddPeer(strings.TrimSpace(uri))
+		m.p2pMonolith.ConnManager.AddPeer(strings.TrimSpace(uri))
 	}
 }
 
+func getServerKeyFromString(nodeID string) (gomatrixserverlib.ServerName, error) {
+	var nodeKey gomatrixserverlib.ServerName
+	if userID, err := gomatrixserverlib.NewUserID(nodeID, false); err == nil {
+		hexKey, decodeErr := hex.DecodeString(string(userID.Domain()))
+		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
+			return "", fmt.Errorf("UserID domain is not a valid ed25519 public key: %v", userID.Domain())
+		} else {
+			nodeKey = userID.Domain()
+		}
+	} else {
+		hexKey, decodeErr := hex.DecodeString(nodeID)
+		if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
+			return "", fmt.Errorf("Relay server uri is not a valid ed25519 public key: %v", nodeID)
+		} else {
+			nodeKey = gomatrixserverlib.ServerName(nodeID)
+		}
+	}
+
+	return nodeKey, nil
+}
+
+func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) {
+	relays := []gomatrixserverlib.ServerName{}
+	for _, uri := range strings.Split(uris, ",") {
+		uri = strings.TrimSpace(uri)
+		if len(uri) == 0 {
+			continue
+		}
+
+		nodeKey, err := getServerKeyFromString(uri)
+		if err != nil {
+			logrus.Errorf(err.Error())
+			continue
+		}
+		relays = append(relays, nodeKey)
+	}
+
+	nodeKey, err := getServerKeyFromString(nodeID)
+	if err != nil {
+		logrus.Errorf(err.Error())
+		return
+	}
+
+	if string(nodeKey) == m.PublicKey() {
+		logrus.Infof("Setting own relay servers to: %v", relays)
+		m.p2pMonolith.RelayRetriever.SetRelayServers(relays)
+	} else {
+		relay.UpdateNodeRelayServers(
+			gomatrixserverlib.ServerName(nodeKey),
+			relays,
+			m.p2pMonolith.BaseDendrite.Context(),
+			m.p2pMonolith.GetFederationAPI(),
+		)
+	}
+}
+
+func (m *DendriteMonolith) GetRelayServers(nodeID string) string {
+	nodeKey, err := getServerKeyFromString(nodeID)
+	if err != nil {
+		logrus.Errorf(err.Error())
+		return ""
+	}
+
+	relaysString := ""
+	if string(nodeKey) == m.PublicKey() {
+		relays := m.p2pMonolith.RelayRetriever.GetRelayServers()
+
+		for i, relay := range relays {
+			if i != 0 {
+				// Append a comma to the previous entry if there is one.
+				relaysString += ","
+			}
+			relaysString += string(relay)
+		}
+	} else {
+		request := api.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(nodeKey)}
+		response := api.P2PQueryRelayServersResponse{}
+		err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.BaseDendrite.Context(), &request, &response)
+		if err != nil {
+			logrus.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
+			return ""
+		}
+
+		for i, relay := range response.RelayServers {
+			if i != 0 {
+				// Append a comma to the previous entry if there is one.
+				relaysString += ","
+			}
+			relaysString += string(relay)
+		}
+	}
+
+	return relaysString
+}
+
+func (m *DendriteMonolith) RelayingEnabled() bool {
+	return m.p2pMonolith.GetRelayAPI().RelayingEnabled()
+}
+
+func (m *DendriteMonolith) SetRelayingEnabled(enabled bool) {
+	m.p2pMonolith.GetRelayAPI().SetRelayingEnabled(enabled)
+}
+
 func (m *DendriteMonolith) DisconnectType(peertype int) {
-	for _, p := range m.PineconeRouter.Peers() {
+	for _, p := range m.p2pMonolith.Router.Peers() {
 		if int(peertype) == p.PeerType {
-			m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
+			m.p2pMonolith.Router.Disconnect(types.SwitchPortID(p.Port), nil)
 		}
 	}
 }
 
 func (m *DendriteMonolith) DisconnectZone(zone string) {
-	for _, p := range m.PineconeRouter.Peers() {
+	for _, p := range m.p2pMonolith.Router.Peers() {
 		if zone == p.Zone {
-			m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
+			m.p2pMonolith.Router.Disconnect(types.SwitchPortID(p.Port), nil)
 		}
 	}
 }
 
 func (m *DendriteMonolith) DisconnectPort(port int) {
-	m.PineconeRouter.Disconnect(types.SwitchPortID(port), nil)
+	m.p2pMonolith.Router.Disconnect(types.SwitchPortID(port), nil)
 }
 
 func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) {
 	l, r := net.Pipe()
-	conduit := &Conduit{conn: r, port: 0}
+	newConduit := Conduit{conduit.NewConduit(r, 0)}
 	go func() {
-		conduit.portMutex.Lock()
-		defer conduit.portMutex.Unlock()
-
 		logrus.Errorf("Attempting authenticated connect")
+		var port types.SwitchPortID
 		var err error
-		if conduit.port, err = m.PineconeRouter.Connect(
+		if port, err = m.p2pMonolith.Router.Connect(
 			l,
 			pineconeRouter.ConnectionZone(zone),
 			pineconeRouter.ConnectionPeerType(peertype),
@@ -198,16 +275,17 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error)
 			logrus.Errorf("Authenticated connect failed: %s", err)
 			_ = l.Close()
 			_ = r.Close()
-			_ = conduit.Close()
+			_ = newConduit.Close()
 			return
 		}
-		logrus.Infof("Authenticated connect succeeded (port %d)", conduit.port)
+		newConduit.SetPort(port)
+		logrus.Infof("Authenticated connect succeeded (port %d)", newConduit.Port())
 	}()
-	return conduit, nil
+	return &newConduit, nil
 }
 
 func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, error) {
-	pubkey := m.PineconeRouter.PublicKey()
+	pubkey := m.p2pMonolith.Router.PublicKey()
 	userID := userutil.MakeUserID(
 		localpart,
 		gomatrixserverlib.ServerName(hex.EncodeToString(pubkey[:])),
@@ -218,7 +296,7 @@ func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, err
 		Password:    password,
 	}
 	userRes := &userapiAPI.PerformAccountCreationResponse{}
-	if err := m.userAPI.PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
+	if err := m.p2pMonolith.GetUserAPI().PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
 		return userID, fmt.Errorf("userAPI.PerformAccountCreation: %w", err)
 	}
 	return userID, nil
@@ -236,7 +314,7 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
 		AccessToken: hex.EncodeToString(accessTokenBytes[:n]),
 	}
 	loginRes := &userapiAPI.PerformDeviceCreationResponse{}
-	if err := m.userAPI.PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
+	if err := m.p2pMonolith.GetUserAPI().PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
 		return "", fmt.Errorf("userAPI.PerformDeviceCreation: %w", err)
 	}
 	if !loginRes.DeviceCreated {
@@ -245,51 +323,10 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
 	return loginRes.Device.AccessToken, nil
 }
 
-// nolint:gocyclo
 func (m *DendriteMonolith) Start() {
-	var sk ed25519.PrivateKey
-	var pk ed25519.PublicKey
-
 	keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
-	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
-		oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
-		if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
-			if err = test.NewMatrixKey(keyfile); err != nil {
-				panic("failed to generate a new PEM key: " + err.Error())
-			}
-			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-				panic("failed to load PEM key: " + err.Error())
-			}
-			if len(sk) != ed25519.PrivateKeySize {
-				panic("the private key is not long enough")
-			}
-		} else {
-			if sk, err = os.ReadFile(oldkeyfile); err != nil {
-				panic("failed to read the old private key: " + err.Error())
-			}
-			if len(sk) != ed25519.PrivateKeySize {
-				panic("the private key is not long enough")
-			}
-			if err = test.SaveMatrixKey(keyfile, sk); err != nil {
-				panic("failed to convert the private key to PEM format: " + err.Error())
-			}
-		}
-	} else {
-		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-			panic("failed to load PEM key: " + err.Error())
-		}
-		if len(sk) != ed25519.PrivateKeySize {
-			panic("the private key is not long enough")
-		}
-	}
-
-	pk = sk.Public().(ed25519.PublicKey)
-
-	var err error
-	m.listener, err = net.Listen("tcp", "localhost:65432")
-	if err != nil {
-		panic(err)
-	}
+	oldKeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
+	sk, pk := monolith.GetOrCreateKey(keyfile, oldKeyfile)
 
 	m.logger = logrus.Logger{
 		Out: BindLogger{},
@@ -297,226 +334,25 @@ func (m *DendriteMonolith) Start() {
 	m.logger.SetOutput(BindLogger{})
 	logrus.SetOutput(BindLogger{})
 
-	pineconeEventChannel := make(chan pineconeEvents.Event)
-	m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
-	m.PineconeRouter.EnableHopLimiting()
-	m.PineconeRouter.EnableWakeupBroadcasts()
-	m.PineconeRouter.Subscribe(pineconeEventChannel)
-
-	m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
-	m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
-	m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
+	m.p2pMonolith = monolith.P2PMonolith{}
+	m.p2pMonolith.SetupPinecone(sk)
 
 	prefix := hex.EncodeToString(pk)
-	cfg := &config.Dendrite{}
-	cfg.Defaults(config.DefaultOpts{
-		Generate:   true,
-		Monolithic: true,
-	})
+	cfg := monolith.GenerateDefaultConfig(sk, m.StorageDirectory, m.CacheDirectory, prefix)
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
-	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.InMemory = false
-	cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix)))
-	cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
-	cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
-	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
-	cfg.ClientAPI.RegistrationDisabled = false
-	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-	cfg.SyncAPI.Fulltext.Enabled = true
-	cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search"))
-	if err = cfg.Derive(); err != nil {
-		panic(err)
-	}
+	// NOTE : disabled for now since there is a 64 bit alignment panic on 32 bit systems
+	// This isn't actually fixed: https://github.com/blevesearch/zapx/pull/147
+	cfg.SyncAPI.Fulltext.Enabled = false
 
-	base := base.NewBaseDendrite(cfg, "Monolith")
-	base.ConfigureAdminEndpoints()
-	defer base.Close() // nolint: errcheck
-
-	federation := conn.CreateFederationClient(base, m.PineconeQUIC)
-
-	serverKeyAPI := &signing.YggdrasilKeys{}
-	keyRing := serverKeyAPI.KeyRing()
-
-	rsAPI := roomserver.NewInternalAPI(base)
-
-	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
-	)
-
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
-	m.userAPI = userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(m.userAPI)
-
-	asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
-
-	// The underlying roomserver implementation needs to be able to call the fedsender.
-	// This is different to rsAPI which can be the http client which doesn't need this dependency
-	rsAPI.SetFederationAPI(fsAPI, keyRing)
-
-	userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation)
-	roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    conn.CreateClient(base, m.PineconeQUIC),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI:            asAPI,
-		FederationAPI:            fsAPI,
-		RoomserverAPI:            rsAPI,
-		UserAPI:                  m.userAPI,
-		KeyAPI:                   keyAPI,
-		ExtPublicRoomsProvider:   roomProvider,
-		ExtUserDirectoryProvider: userProvider,
-	}
-	monolith.AddAllPublicRoutes(base)
-
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
-	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
-	httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
-
-	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
-	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP()
-	pHTTP.Mux().Handle(users.PublicURL, pMux)
-	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
-	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
-
-	// Build both ends of a HTTP multiplex.
-	h2s := &http2.Server{}
-	m.httpServer = &http.Server{
-		Addr:         ":0",
-		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 10 * time.Second,
-		IdleTimeout:  30 * time.Second,
-		BaseContext: func(_ net.Listener) context.Context {
-			return context.Background()
-		},
-		Handler: h2c.NewHandler(pMux, h2s),
-	}
-
-	m.processContext = base.ProcessContext
-
-	go func() {
-		m.logger.Info("Listening on ", cfg.Global.ServerName)
-
-		switch m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")) {
-		case net.ErrClosed, http.ErrServerClosed:
-			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
-		default:
-			m.logger.Fatal(err)
-		}
-	}()
-	go func() {
-		logrus.Info("Listening on ", m.listener.Addr())
-
-		switch http.Serve(m.listener, httpRouter) {
-		case net.ErrClosed, http.ErrServerClosed:
-			m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
-		default:
-			m.logger.Fatal(err)
-		}
-	}()
-
-	go func(ch <-chan pineconeEvents.Event) {
-		eLog := logrus.WithField("pinecone", "events")
-
-		for event := range ch {
-			switch e := event.(type) {
-			case pineconeEvents.PeerAdded:
-			case pineconeEvents.PeerRemoved:
-			case pineconeEvents.TreeParentUpdate:
-			case pineconeEvents.SnakeDescUpdate:
-			case pineconeEvents.TreeRootAnnUpdate:
-			case pineconeEvents.SnakeEntryAdded:
-			case pineconeEvents.SnakeEntryRemoved:
-			case pineconeEvents.BroadcastReceived:
-				eLog.Info("Broadcast received from: ", e.PeerID)
-
-				req := &api.PerformWakeupServersRequest{
-					ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
-				}
-				res := &api.PerformWakeupServersResponse{}
-				if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
-					logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
-				}
-			case pineconeEvents.BandwidthReport:
-			default:
-			}
-		}
-	}(pineconeEventChannel)
+	enableRelaying := false
+	enableMetrics := false
+	enableWebsockets := false
+	m.p2pMonolith.SetupDendrite(cfg, 65432, enableRelaying, enableMetrics, enableWebsockets)
+	m.p2pMonolith.StartMonolith()
 }
 
 func (m *DendriteMonolith) Stop() {
-	m.processContext.ShutdownDendrite()
-	_ = m.listener.Close()
-	m.PineconeMulticast.Stop()
-	_ = m.PineconeQUIC.Close()
-	_ = m.PineconeRouter.Close()
-	m.processContext.WaitForComponentsToFinish()
-}
-
-const MaxFrameSize = types.MaxFrameSize
-
-type Conduit struct {
-	closed    atomic.Bool
-	conn      net.Conn
-	port      types.SwitchPortID
-	portMutex sync.Mutex
-}
-
-func (c *Conduit) Port() int {
-	c.portMutex.Lock()
-	defer c.portMutex.Unlock()
-	return int(c.port)
-}
-
-func (c *Conduit) Read(b []byte) (int, error) {
-	if c.closed.Load() {
-		return 0, io.EOF
-	}
-	return c.conn.Read(b)
-}
-
-func (c *Conduit) ReadCopy() ([]byte, error) {
-	if c.closed.Load() {
-		return nil, io.EOF
-	}
-	var buf [65535 * 2]byte
-	n, err := c.conn.Read(buf[:])
-	if err != nil {
-		return nil, err
-	}
-	return buf[:n], nil
-}
-
-func (c *Conduit) Write(b []byte) (int, error) {
-	if c.closed.Load() {
-		return 0, io.EOF
-	}
-	return c.conn.Write(b)
-}
-
-func (c *Conduit) Close() error {
-	if c.closed.Load() {
-		return io.ErrClosedPipe
-	}
-	c.closed.Store(true)
-	return c.conn.Close()
+	m.p2pMonolith.Stop()
 }

build/gobind-pinecone/monolith_test.go

@@ -0,0 +1,152 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gobind
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func TestMonolithStarts(t *testing.T) {
+	monolith := DendriteMonolith{}
+	monolith.Start()
+	monolith.PublicKey()
+	monolith.Stop()
+}
+
+func TestMonolithSetRelayServers(t *testing.T) {
+	testCases := []struct {
+		name           string
+		nodeID         string
+		relays         string
+		expectedRelays string
+		expectSelf     bool
+	}{
+		{
+			name:           "assorted valid, invalid, empty & self keys",
+			nodeID:         "@valid:abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
+			relays:         "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
+			expectedRelays: "123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
+			expectSelf:     true,
+		},
+		{
+			name:           "invalid node key",
+			nodeID:         "@invalid:notakey",
+			relays:         "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
+			expectedRelays: "",
+			expectSelf:     false,
+		},
+		{
+			name:           "node is self",
+			nodeID:         "self",
+			relays:         "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
+			expectedRelays: "123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
+			expectSelf:     false,
+		},
+	}
+
+	for _, tc := range testCases {
+		monolith := DendriteMonolith{}
+		monolith.Start()
+
+		inputRelays := tc.relays
+		expectedRelays := tc.expectedRelays
+		if tc.expectSelf {
+			inputRelays += "," + monolith.PublicKey()
+			expectedRelays += "," + monolith.PublicKey()
+		}
+		nodeID := tc.nodeID
+		if nodeID == "self" {
+			nodeID = monolith.PublicKey()
+		}
+
+		monolith.SetRelayServers(nodeID, inputRelays)
+		relays := monolith.GetRelayServers(nodeID)
+		monolith.Stop()
+
+		if !containSameKeys(strings.Split(relays, ","), strings.Split(expectedRelays, ",")) {
+			t.Fatalf("%s: expected %s got %s", tc.name, expectedRelays, relays)
+		}
+	}
+}
+
+func containSameKeys(expected []string, actual []string) bool {
+	if len(expected) != len(actual) {
+		return false
+	}
+
+	for _, expectedKey := range expected {
+		hasMatch := false
+		for _, actualKey := range actual {
+			if actualKey == expectedKey {
+				hasMatch = true
+			}
+		}
+
+		if !hasMatch {
+			return false
+		}
+	}
+
+	return true
+}
+
+func TestParseServerKey(t *testing.T) {
+	testCases := []struct {
+		name        string
+		serverKey   string
+		expectedErr bool
+		expectedKey gomatrixserverlib.ServerName
+	}{
+		{
+			name:        "valid userid as key",
+			serverKey:   "@valid:abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
+			expectedErr: false,
+			expectedKey: "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
+		},
+		{
+			name:        "valid key",
+			serverKey:   "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
+			expectedErr: false,
+			expectedKey: "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
+		},
+		{
+			name:        "invalid userid key",
+			serverKey:   "@invalid:notakey",
+			expectedErr: true,
+			expectedKey: "",
+		},
+		{
+			name:        "invalid key",
+			serverKey:   "@invalid:notakey",
+			expectedErr: true,
+			expectedKey: "",
+		},
+	}
+
+	for _, tc := range testCases {
+		key, err := getServerKeyFromString(tc.serverKey)
+		if tc.expectedErr && err == nil {
+			t.Fatalf("%s: expected an error", tc.name)
+		} else if !tc.expectedErr && err != nil {
+			t.Fatalf("%s: didn't expect an error: %s", tc.name, err.Error())
+		}
+		if tc.expectedKey != key {
+			t.Fatalf("%s: keys not equal. expected: %s got: %s", tc.name, tc.expectedKey, key)
+		}
+	}
+}

build/gobind-yggdrasil/monolith.go

@@ -20,7 +20,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -127,8 +126,8 @@ func (m *DendriteMonolith) Start() {
 
 	cfg := &config.Dendrite{}
 	cfg.Defaults(config.DefaultOpts{
-		Generate:   true,
-		Monolithic: true,
+		Generate:       true,
+		SingleDatabase: true,
 	})
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
@@ -149,7 +148,7 @@ func (m *DendriteMonolith) Start() {
 		panic(err)
 	}
 
-	base := base.NewBaseDendrite(cfg, "Monolith")
+	base := base.NewBaseDendrite(cfg)
 	base.ConfigureAdminEndpoints()
 	m.processContext = base.ProcessContext
 	defer base.Close() // nolint: errcheck
@@ -165,9 +164,7 @@ func (m *DendriteMonolith) Start() {
 		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI)
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
+	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
@@ -186,7 +183,6 @@ func (m *DendriteMonolith) Start() {
 		FederationAPI: fsAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
@@ -194,7 +190,6 @@ func (m *DendriteMonolith) Start() {
 	monolith.AddAllPublicRoutes(base)
 
 	httpRouter := mux.NewRouter()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)

build/scripts/Complement.Dockerfile

@@ -16,8 +16,8 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \
+    CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/ComplementLocal.Dockerfile

@@ -19,13 +19,13 @@ WORKDIR /runtime
 # This script compiles Dendrite for us.
 RUN echo '\
     #!/bin/bash -eux \n\
-    if test -f "/runtime/dendrite-monolith-server" && test -f "/runtime/dendrite-monolith-server-cover"; then \n\
+    if test -f "/runtime/dendrite" && test -f "/runtime/dendrite-cover"; then \n\
     echo "Skipping compilation; binaries exist" \n\
     exit 0 \n\
     fi \n\
     cd /dendrite \n\
-    go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
-    go test -c -cover -covermode=atomic -o /runtime/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite-monolith-server \n\
+    go build -v -o /runtime /dendrite/cmd/dendrite \n\
+    go test -c -cover -covermode=atomic -o /runtime/dendrite-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite \n\
     ' > compile.sh && chmod +x compile.sh
 
 # This script runs Dendrite for us. Must be run in the /runtime directory.
@@ -35,8 +35,8 @@ RUN echo '\
     ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
     ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
-    [ ${COVER} -eq 1 ] && exec ./dendrite-monolith-server-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
-    exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    [ ${COVER} -eq 1 ] && exec ./dendrite-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
+    exec ./dendrite --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
     ' > run.sh && chmod +x run.sh
 
 

build/scripts/ComplementPostgres.Dockerfile

@@ -34,8 +34,8 @@ RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
     CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
-    CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server && \
-    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \
+    CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
+    CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
     cp build/scripts/complement-cmd.sh /complement-cmd.sh
 
 WORKDIR /dendrite

build/scripts/complement-cmd.sh

@@ -4,19 +4,17 @@
 
 if [[ "${COVER}" -eq 1 ]]; then
   echo "Running with coverage"
-  exec /dendrite/dendrite-monolith-server-cover \
+  exec /dendrite/dendrite-cover \
     --really-enable-open-registration \
     --tls-cert server.crt \
     --tls-key server.key \
     --config dendrite.yaml \
-    -api=${API:-0} \
-    --test.coverprofile=integrationcover.log
+    --test.coverprofile=complementcover.log
 else
   echo "Not running with coverage"
-  exec /dendrite/dendrite-monolith-server \
+  exec /dendrite/dendrite \
     --really-enable-open-registration \
     --tls-cert server.crt \
     --tls-key server.key \
-    --config dendrite.yaml \
-    -api=${API:-0}
+    --config dendrite.yaml
 fi

clientapi/admin_test.go

@@ -7,9 +7,11 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/syncapi"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
@@ -37,11 +39,9 @@ func TestAdminResetPassword(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(base)
 		// Needed for changing the password/login
-		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
-		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
-		keyAPI.SetUserAPI(userAPI)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
-		AddPublicRoutes(base, nil, nil, nil, nil, nil, userAPI, nil, nil, nil)
+		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
 
 		// Create the users in the userapi and login
 		accessTokens := map[*test.User]string{
@@ -112,6 +112,7 @@ func TestAdminResetPassword(t *testing.T) {
 		}
 
 		for _, tc := range testCases {
+			tc := tc // ensure we don't accidentally only test the last test case
 			t.Run(tc.name, func(t *testing.T) {
 				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/resetPassword/"+tc.userID)
 				if tc.requestOpt != nil {
@@ -132,3 +133,98 @@ func TestAdminResetPassword(t *testing.T) {
 		}
 	})
 }
+
+func TestPurgeRoom(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, aliceAdmin, test.RoomPreset(test.PresetTrustedPrivateChat))
+
+	// Invite Bob
+	room.CreateAndInsert(t, aliceAdmin, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "invite",
+	}, test.WithStateKey(bob.ID))
+
+	ctx := context.Background()
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+
+		fedClient := base.CreateFederationClient()
+		rsAPI := roomserver.NewInternalAPI(base)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
+
+		// this starts the JetStream consumers
+		syncapi.AddPublicRoutes(base, userAPI, rsAPI)
+		federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		// Create the room
+		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil)
+
+		// Create the users in the userapi and login
+		accessTokens := map[*test.User]string{
+			aliceAdmin: "",
+		}
+		for u := range accessTokens {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+			password := util.RandomString(8)
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+
+			req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+				"type": authtypes.LoginTypePassword,
+				"identifier": map[string]interface{}{
+					"type": "m.id.user",
+					"user": u.ID,
+				},
+				"password": password,
+			}))
+			rec := httptest.NewRecorder()
+			base.PublicClientAPIMux.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to login: %s", rec.Body.String())
+			}
+			accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
+		}
+
+		testCases := []struct {
+			name   string
+			roomID string
+			wantOK bool
+		}{
+			{name: "Can purge existing room", wantOK: true, roomID: room.ID},
+			{name: "Can not purge non-existent room", wantOK: false, roomID: "!doesnotexist:localhost"},
+			{name: "rejects invalid room ID", wantOK: false, roomID: "@doesnotexist:localhost"},
+		}
+
+		for _, tc := range testCases {
+			tc := tc // ensure we don't accidentally only test the last test case
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
+
+				req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
+
+				rec := httptest.NewRecorder()
+				base.DendriteAdminMux.ServeHTTP(rec, req)
+				t.Logf("%s", rec.Body.String())
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
+				}
+			})
+		}
+
+	})
+}

clientapi/clientapi.go

@@ -15,6 +15,7 @@
 package clientapi
 
 import (
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
@@ -23,11 +24,9 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/transactions"
-	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
@@ -40,7 +39,6 @@ func AddPublicRoutes(
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	keyAPI keyserverAPI.ClientKeyAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 ) {
 	cfg := &base.Cfg.ClientAPI
@@ -61,7 +59,7 @@ func AddPublicRoutes(
 		base,
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
-		syncProducer, transactionsCache, fsAPI, keyAPI,
+		syncProducer, transactionsCache, fsAPI,
 		extRoomsProvider, mscCfg, natsClient,
 	)
 }

clientapi/routing/admin.go

@@ -1,6 +1,7 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -15,14 +16,13 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 )
 
-func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -55,7 +55,7 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi
 	}
 }
 
-func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -98,7 +98,38 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi
 	}
 }
 
-func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
+func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+	if err != nil {
+		return util.ErrorResponse(err)
+	}
+	roomID, ok := vars["roomID"]
+	if !ok {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument("Expecting room ID."),
+		}
+	}
+	res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
+	if err := rsAPI.PerformAdminPurgeRoom(
+		context.Background(),
+		&roomserverAPI.PerformAdminPurgeRoomRequest{
+			RoomID: roomID,
+		},
+		res,
+	); err != nil {
+		return util.ErrorResponse(err)
+	}
+	if err := res.Error; err != nil {
+		return err.JSONResponse()
+	}
+	return util.JSONResponse{
+		Code: 200,
+		JSON: res,
+	}
+}
+
+func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI api.ClientUserAPI) util.JSONResponse {
 	if req.Body == nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -118,8 +149,8 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 			JSON: jsonerror.InvalidArgumentValue(err.Error()),
 		}
 	}
-	accAvailableResp := &userapi.QueryAccountAvailabilityResponse{}
-	if err = userAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{
+	accAvailableResp := &api.QueryAccountAvailabilityResponse{}
+	if err = userAPI.QueryAccountAvailability(req.Context(), &api.QueryAccountAvailabilityRequest{
 		Localpart:  localpart,
 		ServerName: serverName,
 	}, accAvailableResp); err != nil {
@@ -154,13 +185,13 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 		return *internal.PasswordResponse(err)
 	}
 
-	updateReq := &userapi.PerformPasswordUpdateRequest{
+	updateReq := &api.PerformPasswordUpdateRequest{
 		Localpart:     localpart,
 		ServerName:    serverName,
 		Password:      request.Password,
 		LogoutDevices: true,
 	}
-	updateRes := &userapi.PerformPasswordUpdateResponse{}
+	updateRes := &api.PerformPasswordUpdateResponse{}
 	if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -177,7 +208,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
 	}
 }
 
-func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse {
+func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *api.Device, natsClient *nats.Conn) util.JSONResponse {
 	_, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10)
 	if err != nil {
 		logrus.WithError(err).Error("failed to publish nats message")
@@ -223,7 +254,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien
 	}
 }
 
-func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
+func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
 	vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 	if err != nil {
 		return util.ErrorResponse(err)

clientapi/routing/auth_fallback_test.go

@@ -22,7 +22,7 @@ func Test_AuthFallback(t *testing.T) {
 			for _, wantErr := range []bool{false, true} {
 				t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) {
 					// Set the defaults for each test
-					base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, Monolithic: true})
+					base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true})
 					base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
 					base.Cfg.ClientAPI.RecaptchaPublicKey = "pub"
 					base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv"
@@ -33,7 +33,7 @@ func Test_AuthFallback(t *testing.T) {
 						base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
 					}
 					cfgErrs := &config.ConfigErrors{}
-					base.Cfg.ClientAPI.Verify(cfgErrs, true)
+					base.Cfg.ClientAPI.Verify(cfgErrs)
 					if len(*cfgErrs) > 0 {
 						t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error())
 					}

clientapi/routing/joinroom_test.go

@@ -10,7 +10,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
@@ -29,8 +28,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 		defer baseClose()
 
 		rsAPI := roomserver.NewInternalAPI(base)
-		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
-		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
 		asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
 

clientapi/routing/key_crosssigning.go

@@ -21,9 +21,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 )
 
@@ -34,8 +33,8 @@ type crossSigningRequest struct {
 
 func UploadCrossSigningDeviceKeys(
 	req *http.Request, userInteractiveAuth *auth.UserInteractive,
-	keyserverAPI api.ClientKeyAPI, device *userapi.Device,
-	accountAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
+	keyserverAPI api.ClientKeyAPI, device *api.Device,
+	accountAPI api.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
 	uploadReq := &crossSigningRequest{}
 	uploadRes := &api.PerformUploadDeviceKeysResponse{}
@@ -107,7 +106,7 @@ func UploadCrossSigningDeviceKeys(
 	}
 }
 
-func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
+func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse {
 	uploadReq := &api.PerformUploadDeviceSignaturesRequest{}
 	uploadRes := &api.PerformUploadDeviceSignaturesResponse{}
 

clientapi/routing/keys.go

@@ -23,8 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/keyserver/api"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 )
 
 type uploadKeysRequest struct {
@@ -32,7 +31,7 @@ type uploadKeysRequest struct {
 	OneTimeKeys map[string]json.RawMessage `json:"one_time_keys"`
 }
 
-func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
+func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse {
 	var r uploadKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
@@ -106,7 +105,7 @@ func (r *queryKeysRequest) GetTimeout() time.Duration {
 	return timeout
 }
 
-func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
+func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse {
 	var r queryKeysRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {

clientapi/routing/login_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -39,12 +38,10 @@ func TestLogin(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(base)
 		// Needed for /login
-		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
-		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
-		keyAPI.SetUserAPI(userAPI)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
-		Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, keyAPI, nil, &base.Cfg.MSCs, nil)
+		Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, &base.Cfg.MSCs, nil)
 
 		// Create password
 		password := util.RandomString(8)

clientapi/routing/register_test.go

@@ -30,7 +30,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
@@ -201,8 +200,8 @@ func TestValidationOfApplicationServices(t *testing.T) {
 	// Set up a config
 	fakeConfig := &config.Dendrite{}
 	fakeConfig.Defaults(config.DefaultOpts{
-		Generate:   true,
-		Monolithic: true,
+		Generate:       true,
+		SingleDatabase: true,
 	})
 	fakeConfig.Global.ServerName = "localhost"
 	fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
@@ -409,9 +408,7 @@ func Test_register(t *testing.T) {
 		defer baseClose()
 
 		rsAPI := roomserver.NewInternalAPI(base)
-		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
-		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
-		keyAPI.SetUserAPI(userAPI)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -582,9 +579,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 		base.Cfg.Global.ServerName = "server"
 
 		rsAPI := roomserver.NewInternalAPI(base)
-		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
-		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
-		keyAPI.SetUserAPI(userAPI)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
@@ -623,9 +618,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
 
 		rsAPI := roomserver.NewInternalAPI(base)
-		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
-		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
-		keyAPI.SetUserAPI(userAPI)
+		userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)

clientapi/routing/routing.go

@@ -18,9 +18,11 @@ import (
 	"context"
 	"net/http"
 	"strings"
+	"sync"
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/setup/base"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
@@ -36,11 +38,9 @@ import (
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/transactions"
-	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
-	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
@@ -60,7 +60,6 @@ func Setup(
 	syncProducer *producers.SyncAPIProducer,
 	transactionsCache *transactions.Cache,
 	federationSender federationAPI.ClientFederationAPI,
-	keyAPI keyserverAPI.ClientKeyAPI,
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs, natsClient *nats.Conn,
 ) {
@@ -165,6 +164,12 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
+	dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
+		httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return AdminPurgeRoom(req, cfg, device, rsAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
 	dendriteAdminRouter.Handle("/admin/resetPassword/{userID}",
 		httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return AdminResetPassword(req, cfg, device, userAPI)
@@ -185,25 +190,31 @@ func Setup(
 
 	dendriteAdminRouter.Handle("/admin/refreshDevices/{userID}",
 		httputil.MakeAdminAPI("admin_refresh_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return AdminMarkAsStale(req, cfg, keyAPI)
+			return AdminMarkAsStale(req, cfg, userAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	// server notifications
 	if cfg.Matrix.ServerNotices.Enabled {
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
-		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
-		if err != nil {
-			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
-		}
+		var serverNotificationSender *userapi.Device
+		var err error
+		notificationSenderOnce := &sync.Once{}
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+				notificationSenderOnce.Do(func() {
+					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
+					if err != nil {
+						logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+					}
+				})
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
 				}
-				vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+				var vars map[string]string
+				vars, err = httputil.URLDecodeMapValues(mux.Vars(req))
 				if err != nil {
 					return util.ErrorResponse(err)
 				}
@@ -219,6 +230,12 @@ func Setup(
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice",
 			httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+				notificationSenderOnce.Do(func() {
+					serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
+					if err != nil {
+						logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+					}
+				})
 				// not specced, but ensure we're rate limiting requests to this endpoint
 				if r := rateLimits.Limit(req, device); r != nil {
 					return *r
@@ -1353,11 +1370,11 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, userAPI, cfg)
+		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceSignatures(req, keyAPI, device)
+		return UploadCrossSigningDeviceSignatures(req, userAPI, device)
 	}, httputil.WithAllowGuests())
 
 	v3mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
@@ -1369,22 +1386,22 @@ func Setup(
 	// Supplying a device ID is deprecated.
 	v3mux.Handle("/keys/upload/{deviceID}",
 		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return UploadKeys(req, keyAPI, device)
+			return UploadKeys(req, userAPI, device)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/keys/upload",
 		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return UploadKeys(req, keyAPI, device)
+			return UploadKeys(req, userAPI, device)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/keys/query",
 		httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return QueryKeys(req, keyAPI, device)
+			return QueryKeys(req, userAPI, device)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/keys/claim",
 		httputil.MakeAuthAPI("keys_claim", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return ClaimKeys(req, keyAPI)
+			return ClaimKeys(req, userAPI)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",

cmd/dendrite-demo-pinecone/README.md

@@ -24,3 +24,42 @@ Then point your favourite Matrix client to  the homeserver URL`http://localhost:
 If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
 
 Once logged in, you should be able to open the room directory or join a room by its ID.
+
+## Store & Forward Relays
+
+To test out the store & forward relay functionality, you need a minimum of 3 instances. 
+One instance will act as the relay, and the other two instances will be the users trying to communicate.
+Then you can send messages between the two nodes and watch as the relay is used if the receiving node is offline.
+
+### Launching the Nodes
+
+Relay Server:
+```
+go run cmd/dendrite-demo-pinecone/main.go -dir relay/ -listen "[::]:49000"
+```
+
+Node 1:
+```
+go run cmd/dendrite-demo-pinecone/main.go -dir node-1/ -peer "[::]:49000" -port 8007
+```
+
+Node 2:
+```
+go run cmd/dendrite-demo-pinecone/main.go -dir node-2/ -peer "[::]:49000" -port 8009
+```
+
+### Database Setup
+
+At the moment, the database must be manually configured.
+For both `Node 1` and `Node 2` add the following entries to their respective `relay_server` table in the federationapi database:
+```
+server_name: {node_1_public_key}, relay_server_name: {relay_public_key}
+server_name: {node_2_public_key}, relay_server_name: {relay_public_key}
+```
+
+After editing the database you will need to relaunch the nodes for the changes to be picked up by dendrite.
+
+### Testing
+
+Now you can run two separate instances of element and connect them to `Node 1` and `Node 2`.
+You can shutdown one of the nodes and continue sending messages. If you wait long enough, the message will be sent to the relay server. (you can see this in the log output of the relay server) 

cmd/dendrite-demo-pinecone/conduit/conduit.go

@@ -0,0 +1,84 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package conduit
+
+import (
+	"io"
+	"net"
+	"sync"
+
+	"github.com/matrix-org/pinecone/types"
+	"go.uber.org/atomic"
+)
+
+type Conduit struct {
+	closed    atomic.Bool
+	conn      net.Conn
+	portMutex sync.Mutex
+	port      types.SwitchPortID
+}
+
+func NewConduit(conn net.Conn, port int) Conduit {
+	return Conduit{
+		conn: conn,
+		port: types.SwitchPortID(port),
+	}
+}
+
+func (c *Conduit) Port() int {
+	c.portMutex.Lock()
+	defer c.portMutex.Unlock()
+	return int(c.port)
+}
+
+func (c *Conduit) SetPort(port types.SwitchPortID) {
+	c.portMutex.Lock()
+	defer c.portMutex.Unlock()
+	c.port = port
+}
+
+func (c *Conduit) Read(b []byte) (int, error) {
+	if c.closed.Load() {
+		return 0, io.EOF
+	}
+	return c.conn.Read(b)
+}
+
+func (c *Conduit) ReadCopy() ([]byte, error) {
+	if c.closed.Load() {
+		return nil, io.EOF
+	}
+	var buf [65535 * 2]byte
+	n, err := c.conn.Read(buf[:])
+	if err != nil {
+		return nil, err
+	}
+	return buf[:n], nil
+}
+
+func (c *Conduit) Write(b []byte) (int, error) {
+	if c.closed.Load() {
+		return 0, io.EOF
+	}
+	return c.conn.Write(b)
+}
+
+func (c *Conduit) Close() error {
+	if c.closed.Load() {
+		return io.ErrClosedPipe
+	}
+	c.closed.Store(true)
+	return c.conn.Close()
+}

cmd/dendrite-demo-pinecone/conduit/conduit_test.go

@@ -0,0 +1,121 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package conduit
+
+import (
+	"fmt"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var TestBuf = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+
+type TestNetConn struct {
+	net.Conn
+	shouldFail bool
+}
+
+func (t *TestNetConn) Read(b []byte) (int, error) {
+	if t.shouldFail {
+		return 0, fmt.Errorf("Failed")
+	} else {
+		n := copy(b, TestBuf)
+		return n, nil
+	}
+}
+
+func (t *TestNetConn) Write(b []byte) (int, error) {
+	if t.shouldFail {
+		return 0, fmt.Errorf("Failed")
+	} else {
+		return len(b), nil
+	}
+}
+
+func (t *TestNetConn) Close() error {
+	if t.shouldFail {
+		return fmt.Errorf("Failed")
+	} else {
+		return nil
+	}
+}
+
+func TestConduitStoresPort(t *testing.T) {
+	conduit := Conduit{port: 7}
+	assert.Equal(t, 7, conduit.Port())
+}
+
+func TestConduitRead(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	b := make([]byte, len(TestBuf))
+	bytes, err := conduit.Read(b)
+	assert.NoError(t, err)
+	assert.Equal(t, len(TestBuf), bytes)
+	assert.Equal(t, TestBuf, b)
+}
+
+func TestConduitReadCopy(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	result, err := conduit.ReadCopy()
+	assert.NoError(t, err)
+	assert.Equal(t, TestBuf, result)
+}
+
+func TestConduitWrite(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	bytes, err := conduit.Write(TestBuf)
+	assert.NoError(t, err)
+	assert.Equal(t, len(TestBuf), bytes)
+}
+
+func TestConduitClose(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	assert.True(t, conduit.closed.Load())
+}
+
+func TestConduitReadClosed(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	b := make([]byte, len(TestBuf))
+	_, err = conduit.Read(b)
+	assert.Error(t, err)
+}
+
+func TestConduitReadCopyClosed(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	_, err = conduit.ReadCopy()
+	assert.Error(t, err)
+}
+
+func TestConduitWriteClosed(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{}}
+	err := conduit.Close()
+	assert.NoError(t, err)
+	_, err = conduit.Write(TestBuf)
+	assert.Error(t, err)
+}
+
+func TestConduitReadCopyFails(t *testing.T) {
+	conduit := Conduit{conn: &TestNetConn{shouldFail: true}}
+	_, err := conduit.ReadCopy()
+	assert.Error(t, err)
+}

cmd/dendrite-demo-pinecone/main.go

@@ -15,58 +15,35 @@
 package main
 
 import (
-	"context"
 	"crypto/ed25519"
-	"crypto/tls"
 	"encoding/hex"
 	"flag"
 	"fmt"
 	"net"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
-	"github.com/gorilla/mux"
-	"github.com/gorilla/websocket"
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
-	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/monolith"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/userapi"
 	"github.com/matrix-org/gomatrixserverlib"
-
-	pineconeConnections "github.com/matrix-org/pinecone/connections"
-	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
-	pineconeRouter "github.com/matrix-org/pinecone/router"
-	pineconeEvents "github.com/matrix-org/pinecone/router/events"
-	pineconeSessions "github.com/matrix-org/pinecone/sessions"
-
 	"github.com/sirupsen/logrus"
+
+	pineconeRouter "github.com/matrix-org/pinecone/router"
 )
 
 var (
-	instanceName   = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance")
-	instancePort   = flag.Int("port", 8008, "the port that the client API will listen on")
-	instancePeer   = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
-	instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
-	instanceDir    = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
+	instanceName            = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance")
+	instancePort            = flag.Int("port", 8008, "the port that the client API will listen on")
+	instancePeer            = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
+	instanceListen          = flag.String("listen", ":0", "the port Pinecone peers can connect to")
+	instanceDir             = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
+	instanceRelayingEnabled = flag.Bool("relay", false, "whether to enable store & forward relaying for other nodes")
 )
 
-// nolint:gocyclo
 func main() {
 	flag.Parse()
 	internal.SetupPprof()
@@ -83,7 +60,7 @@ func main() {
 		}
 	}
 
-	cfg := &config.Dendrite{}
+	var cfg *config.Dendrite
 
 	// use custom config if config flag is set
 	if configFlagSet {
@@ -92,88 +69,30 @@ func main() {
 		pk = sk.Public().(ed25519.PublicKey)
 	} else {
 		keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
-		if _, err := os.Stat(keyfile); os.IsNotExist(err) {
-			oldkeyfile := *instanceName + ".key"
-			if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
-				if err = test.NewMatrixKey(keyfile); err != nil {
-					panic("failed to generate a new PEM key: " + err.Error())
-				}
-				if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-					panic("failed to load PEM key: " + err.Error())
-				}
-				if len(sk) != ed25519.PrivateKeySize {
-					panic("the private key is not long enough")
-				}
-			} else {
-				if sk, err = os.ReadFile(oldkeyfile); err != nil {
-					panic("failed to read the old private key: " + err.Error())
-				}
-				if len(sk) != ed25519.PrivateKeySize {
-					panic("the private key is not long enough")
-				}
-				if err := test.SaveMatrixKey(keyfile, sk); err != nil {
-					panic("failed to convert the private key to PEM format: " + err.Error())
-				}
-			}
-		} else {
-			var err error
-			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
-				panic("failed to load PEM key: " + err.Error())
-			}
-			if len(sk) != ed25519.PrivateKeySize {
-				panic("the private key is not long enough")
-			}
-		}
-
-		pk = sk.Public().(ed25519.PublicKey)
-
-		cfg.Defaults(config.DefaultOpts{
-			Generate:   true,
-			Monolithic: true,
-		})
-		cfg.Global.PrivateKey = sk
-		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(*instanceDir, *instanceName)))
-		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
-		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
-		cfg.ClientAPI.RegistrationDisabled = false
-		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-		cfg.MediaAPI.BasePath = config.Path(*instanceDir)
-		cfg.SyncAPI.Fulltext.Enabled = true
-		cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir)
-		if err := cfg.Derive(); err != nil {
-			panic(err)
-		}
+		oldKeyfile := *instanceName + ".key"
+		sk, pk = monolith.GetOrCreateKey(keyfile, oldKeyfile)
+		cfg = monolith.GenerateDefaultConfig(sk, *instanceDir, *instanceDir, *instanceName)
 	}
 
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
-	base := base.NewBaseDendrite(cfg, "Monolith")
-	base.ConfigureAdminEndpoints()
-	defer base.Close() // nolint: errcheck
+	p2pMonolith := monolith.P2PMonolith{}
+	p2pMonolith.SetupPinecone(sk)
+	p2pMonolith.Multicast.Start()
 
-	pineconeEventChannel := make(chan pineconeEvents.Event)
-	pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
-	pRouter.EnableHopLimiting()
-	pRouter.EnableWakeupBroadcasts()
-	pRouter.Subscribe(pineconeEventChannel)
-
-	pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
-	pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
-	pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
-	pMulticast.Start()
 	if instancePeer != nil && *instancePeer != "" {
 		for _, peer := range strings.Split(*instancePeer, ",") {
-			pManager.AddPeer(strings.Trim(peer, " \t\r\n"))
+			p2pMonolith.ConnManager.AddPeer(strings.Trim(peer, " \t\r\n"))
 		}
 	}
 
+	enableMetrics := true
+	enableWebsockets := true
+	p2pMonolith.SetupDendrite(cfg, *instancePort, *instanceRelayingEnabled, enableMetrics, enableWebsockets)
+	p2pMonolith.StartMonolith()
+	p2pMonolith.WaitForShutdown()
+
 	go func() {
 		listener, err := net.Listen("tcp", *instanceListen)
 		if err != nil {
@@ -189,7 +108,7 @@ func main() {
 				continue
 			}
 
-			port, err := pRouter.Connect(
+			port, err := p2pMonolith.Router.Connect(
 				conn,
 				pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
 			)
@@ -201,135 +120,4 @@ func main() {
 			fmt.Println("Inbound connection", conn.RemoteAddr(), "is connected to port", port)
 		}
 	}()
-
-	federation := conn.CreateFederationClient(base, pQUIC)
-
-	serverKeyAPI := &signing.YggdrasilKeys{}
-	keyRing := serverKeyAPI.KeyRing()
-
-	rsComponent := roomserver.NewInternalAPI(base)
-	rsAPI := rsComponent
-	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, keyRing, true,
-	)
-
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsComponent)
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
-
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-
-	rsComponent.SetFederationAPI(fsAPI, keyRing)
-
-	userProvider := users.NewPineconeUserProvider(pRouter, pQUIC, userAPI, federation)
-	roomProvider := rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    conn.CreateClient(base, pQUIC),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI:            asAPI,
-		FederationAPI:            fsAPI,
-		RoomserverAPI:            rsAPI,
-		UserAPI:                  userAPI,
-		KeyAPI:                   keyAPI,
-		ExtPublicRoomsProvider:   roomProvider,
-		ExtUserDirectoryProvider: userProvider,
-	}
-	monolith.AddAllPublicRoutes(base)
-
-	wsUpgrader := websocket.Upgrader{
-		CheckOrigin: func(_ *http.Request) bool {
-			return true
-		},
-	}
-	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
-	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
-	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
-	httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
-	httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
-		c, err := wsUpgrader.Upgrade(w, r, nil)
-		if err != nil {
-			logrus.WithError(err).Error("Failed to upgrade WebSocket connection")
-			return
-		}
-		conn := conn.WrapWebSocketConn(c)
-		if _, err = pRouter.Connect(
-			conn,
-			pineconeRouter.ConnectionZone("websocket"),
-			pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
-		); err != nil {
-			logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
-		}
-	})
-	httpRouter.HandleFunc("/pinecone", pRouter.ManholeHandler)
-	embed.Embed(httpRouter, *instancePort, "Pinecone Demo")
-
-	pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
-	pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
-	pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
-
-	pHTTP := pQUIC.Protocol("matrix").HTTP()
-	pHTTP.Mux().Handle(users.PublicURL, pMux)
-	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
-	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
-
-	// Build both ends of a HTTP multiplex.
-	httpServer := &http.Server{
-		Addr:         ":0",
-		TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 10 * time.Second,
-		IdleTimeout:  60 * time.Second,
-		BaseContext: func(_ net.Listener) context.Context {
-			return context.Background()
-		},
-		Handler: pMux,
-	}
-
-	go func() {
-		pubkey := pRouter.PublicKey()
-		logrus.Info("Listening on ", hex.EncodeToString(pubkey[:]))
-		logrus.Fatal(httpServer.Serve(pQUIC.Protocol("matrix")))
-	}()
-	go func() {
-		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
-		logrus.Info("Listening on ", httpBindAddr)
-		logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
-	}()
-
-	go func(ch <-chan pineconeEvents.Event) {
-		eLog := logrus.WithField("pinecone", "events")
-
-		for event := range ch {
-			switch e := event.(type) {
-			case pineconeEvents.PeerAdded:
-			case pineconeEvents.PeerRemoved:
-			case pineconeEvents.TreeParentUpdate:
-			case pineconeEvents.SnakeDescUpdate:
-			case pineconeEvents.TreeRootAnnUpdate:
-			case pineconeEvents.SnakeEntryAdded:
-			case pineconeEvents.SnakeEntryRemoved:
-			case pineconeEvents.BroadcastReceived:
-				eLog.Info("Broadcast received from: ", e.PeerID)
-
-				req := &api.PerformWakeupServersRequest{
-					ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
-				}
-				res := &api.PerformWakeupServersResponse{}
-				if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
-					logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
-				}
-			case pineconeEvents.BandwidthReport:
-			default:
-			}
-		}
-	}(pineconeEventChannel)
-
-	base.WaitForShutdown()
 }

cmd/dendrite-demo-pinecone/monolith/keys.go

@@ -0,0 +1,63 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package monolith
+
+import (
+	"crypto/ed25519"
+	"os"
+
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+)
+
+func GetOrCreateKey(keyfile string, oldKeyfile string) (ed25519.PrivateKey, ed25519.PublicKey) {
+	var sk ed25519.PrivateKey
+	var pk ed25519.PublicKey
+
+	if _, err := os.Stat(keyfile); os.IsNotExist(err) {
+		if _, err = os.Stat(oldKeyfile); os.IsNotExist(err) {
+			if err = test.NewMatrixKey(keyfile); err != nil {
+				panic("failed to generate a new PEM key: " + err.Error())
+			}
+			if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+				panic("failed to load PEM key: " + err.Error())
+			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
+		} else {
+			if sk, err = os.ReadFile(oldKeyfile); err != nil {
+				panic("failed to read the old private key: " + err.Error())
+			}
+			if len(sk) != ed25519.PrivateKeySize {
+				panic("the private key is not long enough")
+			}
+			if err = test.SaveMatrixKey(keyfile, sk); err != nil {
+				panic("failed to convert the private key to PEM format: " + err.Error())
+			}
+		}
+	} else {
+		if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
+			panic("failed to load PEM key: " + err.Error())
+		}
+		if len(sk) != ed25519.PrivateKeySize {
+			panic("the private key is not long enough")
+		}
+	}
+
+	pk = sk.Public().(ed25519.PublicKey)
+
+	return sk, pk
+}

cmd/dendrite-demo-pinecone/monolith/monolith.go

@@ -0,0 +1,387 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package monolith
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/tls"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/gorilla/websocket"
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/relay"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/federationapi"
+	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/producers"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/relayapi"
+	relayAPI "github.com/matrix-org/dendrite/relayapi/api"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/jetstream"
+	"github.com/matrix-org/dendrite/userapi"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+
+	pineconeConnections "github.com/matrix-org/pinecone/connections"
+	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeEvents "github.com/matrix-org/pinecone/router/events"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+)
+
+const SessionProtocol = "matrix"
+
+type P2PMonolith struct {
+	BaseDendrite   *base.BaseDendrite
+	Sessions       *pineconeSessions.Sessions
+	Multicast      *pineconeMulticast.Multicast
+	ConnManager    *pineconeConnections.ConnectionManager
+	Router         *pineconeRouter.Router
+	EventChannel   chan pineconeEvents.Event
+	RelayRetriever relay.RelayServerRetriever
+
+	dendrite           setup.Monolith
+	port               int
+	httpMux            *mux.Router
+	pineconeMux        *mux.Router
+	httpServer         *http.Server
+	listener           net.Listener
+	httpListenAddr     string
+	stopHandlingEvents chan bool
+}
+
+func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir string, dbPrefix string) *config.Dendrite {
+	cfg := config.Dendrite{}
+	cfg.Defaults(config.DefaultOpts{
+		Generate:       true,
+		SingleDatabase: true,
+	})
+	cfg.Global.PrivateKey = sk
+	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(cacheDir, dbPrefix)))
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
+	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(storageDir, dbPrefix)))
+	cfg.ClientAPI.RegistrationDisabled = false
+	cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+	cfg.MediaAPI.BasePath = config.Path(filepath.Join(cacheDir, "media"))
+	cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(cacheDir, "media"))
+	cfg.SyncAPI.Fulltext.Enabled = true
+	cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(cacheDir, "search"))
+	if err := cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	return &cfg
+}
+
+func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) {
+	p.EventChannel = make(chan pineconeEvents.Event)
+	p.Router = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
+	p.Router.EnableHopLimiting()
+	p.Router.EnableWakeupBroadcasts()
+	p.Router.Subscribe(p.EventChannel)
+
+	p.Sessions = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), p.Router, []string{SessionProtocol})
+	p.Multicast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), p.Router)
+	p.ConnManager = pineconeConnections.NewConnectionManager(p.Router, nil)
+}
+
+func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) {
+	if enableMetrics {
+		p.BaseDendrite = base.NewBaseDendrite(cfg)
+	} else {
+		p.BaseDendrite = base.NewBaseDendrite(cfg, base.DisableMetrics)
+	}
+	p.port = port
+	p.BaseDendrite.ConfigureAdminEndpoints()
+
+	federation := conn.CreateFederationClient(p.BaseDendrite, p.Sessions)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsComponent := roomserver.NewInternalAPI(p.BaseDendrite)
+	rsAPI := rsComponent
+	fsAPI := federationapi.NewInternalAPI(
+		p.BaseDendrite, federation, rsAPI, p.BaseDendrite.Caches, keyRing, true,
+	)
+
+	userAPI := userapi.NewInternalAPI(p.BaseDendrite, rsAPI, federation)
+
+	asAPI := appservice.NewInternalAPI(p.BaseDendrite, userAPI, rsAPI)
+
+	rsComponent.SetFederationAPI(fsAPI, keyRing)
+
+	userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation)
+	roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation)
+
+	js, _ := p.BaseDendrite.NATS.Prepare(p.BaseDendrite.ProcessContext, &p.BaseDendrite.Cfg.Global.JetStream)
+	producer := &producers.SyncAPIProducer{
+		JetStream:              js,
+		TopicReceiptEvent:      p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
+		TopicSendToDeviceEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
+		TopicTypingEvent:       p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
+		TopicPresenceEvent:     p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
+		TopicDeviceListUpdate:  p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
+		TopicSigningKeyUpdate:  p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
+		Config:                 &p.BaseDendrite.Cfg.FederationAPI,
+		UserAPI:                userAPI,
+	}
+	relayAPI := relayapi.NewRelayInternalAPI(p.BaseDendrite, federation, rsAPI, keyRing, producer, enableRelaying)
+	logrus.Infof("Relaying enabled: %v", relayAPI.RelayingEnabled())
+
+	p.dendrite = setup.Monolith{
+		Config:    p.BaseDendrite.Cfg,
+		Client:    conn.CreateClient(p.BaseDendrite, p.Sessions),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI:            asAPI,
+		FederationAPI:            fsAPI,
+		RoomserverAPI:            rsAPI,
+		UserAPI:                  userAPI,
+		RelayAPI:                 relayAPI,
+		ExtPublicRoomsProvider:   roomProvider,
+		ExtUserDirectoryProvider: userProvider,
+	}
+	p.dendrite.AddAllPublicRoutes(p.BaseDendrite)
+
+	p.setupHttpServers(userProvider, enableWebsockets)
+}
+
+func (p *P2PMonolith) GetFederationAPI() federationAPI.FederationInternalAPI {
+	return p.dendrite.FederationAPI
+}
+
+func (p *P2PMonolith) GetRelayAPI() relayAPI.RelayInternalAPI {
+	return p.dendrite.RelayAPI
+}
+
+func (p *P2PMonolith) GetUserAPI() userAPI.UserInternalAPI {
+	return p.dendrite.UserAPI
+}
+
+func (p *P2PMonolith) StartMonolith() {
+	p.startHTTPServers()
+	p.startEventHandler()
+}
+
+func (p *P2PMonolith) Stop() {
+	logrus.Info("Stopping monolith")
+	_ = p.BaseDendrite.Close()
+	p.WaitForShutdown()
+	logrus.Info("Stopped monolith")
+}
+
+func (p *P2PMonolith) WaitForShutdown() {
+	p.BaseDendrite.WaitForShutdown()
+	p.closeAllResources()
+}
+
+func (p *P2PMonolith) closeAllResources() {
+	logrus.Info("Closing monolith resources")
+	if p.httpServer != nil {
+		_ = p.httpServer.Shutdown(context.Background())
+	}
+
+	select {
+	case p.stopHandlingEvents <- true:
+	default:
+	}
+
+	if p.listener != nil {
+		_ = p.listener.Close()
+	}
+
+	if p.Multicast != nil {
+		p.Multicast.Stop()
+	}
+
+	if p.Sessions != nil {
+		_ = p.Sessions.Close()
+	}
+
+	if p.Router != nil {
+		_ = p.Router.Close()
+	}
+	logrus.Info("Monolith resources closed")
+}
+
+func (p *P2PMonolith) Addr() string {
+	return p.httpListenAddr
+}
+
+func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) {
+	p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath()
+	p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.PublicClientAPIMux)
+	p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux)
+	p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.DendriteAdminMux)
+	p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(p.BaseDendrite.SynapseAdminMux)
+
+	if enableWebsockets {
+		wsUpgrader := websocket.Upgrader{
+			CheckOrigin: func(_ *http.Request) bool {
+				return true
+			},
+		}
+		p.httpMux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
+			c, err := wsUpgrader.Upgrade(w, r, nil)
+			if err != nil {
+				logrus.WithError(err).Error("Failed to upgrade WebSocket connection")
+				return
+			}
+			conn := conn.WrapWebSocketConn(c)
+			if _, err = p.Router.Connect(
+				conn,
+				pineconeRouter.ConnectionZone("websocket"),
+				pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
+			); err != nil {
+				logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
+			}
+		})
+	}
+
+	p.httpMux.HandleFunc("/pinecone", p.Router.ManholeHandler)
+
+	if enableWebsockets {
+		embed.Embed(p.httpMux, p.port, "Pinecone Demo")
+	}
+
+	p.pineconeMux = mux.NewRouter().SkipClean(true).UseEncodedPath()
+	p.pineconeMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
+	p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(p.BaseDendrite.PublicFederationAPIMux)
+	p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux)
+
+	pHTTP := p.Sessions.Protocol(SessionProtocol).HTTP()
+	pHTTP.Mux().Handle(users.PublicURL, p.pineconeMux)
+	pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, p.pineconeMux)
+	pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, p.pineconeMux)
+}
+
+func (p *P2PMonolith) startHTTPServers() {
+	go func() {
+		// Build both ends of a HTTP multiplex.
+		p.httpServer = &http.Server{
+			Addr:         ":0",
+			TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
+			ReadTimeout:  10 * time.Second,
+			WriteTimeout: 10 * time.Second,
+			IdleTimeout:  30 * time.Second,
+			BaseContext: func(_ net.Listener) context.Context {
+				return context.Background()
+			},
+			Handler: p.pineconeMux,
+		}
+
+		pubkey := p.Router.PublicKey()
+		pubkeyString := hex.EncodeToString(pubkey[:])
+		logrus.Info("Listening on ", pubkeyString)
+
+		switch p.httpServer.Serve(p.Sessions.Protocol(SessionProtocol)) {
+		case net.ErrClosed, http.ErrServerClosed:
+			logrus.Info("Stopped listening on ", pubkeyString)
+		default:
+			logrus.Error("Stopped listening on ", pubkeyString)
+		}
+		logrus.Info("Stopped goroutine listening on ", pubkeyString)
+	}()
+
+	p.httpListenAddr = fmt.Sprintf(":%d", p.port)
+	go func() {
+		logrus.Info("Listening on ", p.httpListenAddr)
+		switch http.ListenAndServe(p.httpListenAddr, p.httpMux) {
+		case net.ErrClosed, http.ErrServerClosed:
+			logrus.Info("Stopped listening on ", p.httpListenAddr)
+		default:
+			logrus.Error("Stopped listening on ", p.httpListenAddr)
+		}
+		logrus.Info("Stopped goroutine listening on ", p.httpListenAddr)
+	}()
+}
+
+func (p *P2PMonolith) startEventHandler() {
+	p.stopHandlingEvents = make(chan bool)
+	stopRelayServerSync := make(chan bool)
+	eLog := logrus.WithField("pinecone", "events")
+	p.RelayRetriever = relay.NewRelayServerRetriever(
+		context.Background(),
+		gomatrixserverlib.ServerName(p.Router.PublicKey().String()),
+		p.dendrite.FederationAPI,
+		p.dendrite.RelayAPI,
+		stopRelayServerSync,
+	)
+	p.RelayRetriever.InitializeRelayServers(eLog)
+
+	go func(ch <-chan pineconeEvents.Event) {
+		for {
+			select {
+			case event := <-ch:
+				switch e := event.(type) {
+				case pineconeEvents.PeerAdded:
+					p.RelayRetriever.StartSync()
+				case pineconeEvents.PeerRemoved:
+					if p.RelayRetriever.IsRunning() && p.Router.TotalPeerCount() == 0 {
+						// NOTE: Don't block on channel
+						select {
+						case stopRelayServerSync <- true:
+						default:
+						}
+					}
+				case pineconeEvents.BroadcastReceived:
+					// eLog.Info("Broadcast received from: ", e.PeerID)
+
+					req := &federationAPI.PerformWakeupServersRequest{
+						ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
+					}
+					res := &federationAPI.PerformWakeupServersResponse{}
+					if err := p.dendrite.FederationAPI.PerformWakeupServers(p.BaseDendrite.Context(), req, res); err != nil {
+						eLog.WithError(err).Error("Failed to wakeup destination", e.PeerID)
+					}
+				}
+			case <-p.stopHandlingEvents:
+				logrus.Info("Stopping processing pinecone events")
+				// NOTE: Don't block on channel
+				select {
+				case stopRelayServerSync <- true:
+				default:
+				}
+				logrus.Info("Stopped processing pinecone events")
+				return
+			}
+		}
+	}(p.EventChannel)
+}

cmd/dendrite-demo-pinecone/relay/retriever.go

@@ -0,0 +1,238 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package relay
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
+	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+	"go.uber.org/atomic"
+)
+
+const (
+	relayServerRetryInterval = time.Second * 30
+)
+
+type RelayServerRetriever struct {
+	ctx                 context.Context
+	serverName          gomatrixserverlib.ServerName
+	federationAPI       federationAPI.FederationInternalAPI
+	relayAPI            relayServerAPI.RelayInternalAPI
+	relayServersQueried map[gomatrixserverlib.ServerName]bool
+	queriedServersMutex sync.Mutex
+	running             atomic.Bool
+	quit                chan bool
+}
+
+func NewRelayServerRetriever(
+	ctx context.Context,
+	serverName gomatrixserverlib.ServerName,
+	federationAPI federationAPI.FederationInternalAPI,
+	relayAPI relayServerAPI.RelayInternalAPI,
+	quit chan bool,
+) RelayServerRetriever {
+	return RelayServerRetriever{
+		ctx:                 ctx,
+		serverName:          serverName,
+		federationAPI:       federationAPI,
+		relayAPI:            relayAPI,
+		relayServersQueried: make(map[gomatrixserverlib.ServerName]bool),
+		running:             *atomic.NewBool(false),
+		quit:                quit,
+	}
+}
+
+func (r *RelayServerRetriever) InitializeRelayServers(eLog *logrus.Entry) {
+	request := federationAPI.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(r.serverName)}
+	response := federationAPI.P2PQueryRelayServersResponse{}
+	err := r.federationAPI.P2PQueryRelayServers(r.ctx, &request, &response)
+	if err != nil {
+		eLog.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
+	}
+
+	r.queriedServersMutex.Lock()
+	defer r.queriedServersMutex.Unlock()
+	for _, server := range response.RelayServers {
+		r.relayServersQueried[server] = false
+	}
+
+	eLog.Infof("Registered relay servers: %v", response.RelayServers)
+}
+
+func (r *RelayServerRetriever) SetRelayServers(servers []gomatrixserverlib.ServerName) {
+	UpdateNodeRelayServers(r.serverName, servers, r.ctx, r.federationAPI)
+
+	// Replace list of servers to sync with and mark them all as unsynced.
+	r.queriedServersMutex.Lock()
+	defer r.queriedServersMutex.Unlock()
+	r.relayServersQueried = make(map[gomatrixserverlib.ServerName]bool)
+	for _, server := range servers {
+		r.relayServersQueried[server] = false
+	}
+
+	r.StartSync()
+}
+
+func (r *RelayServerRetriever) GetRelayServers() []gomatrixserverlib.ServerName {
+	r.queriedServersMutex.Lock()
+	defer r.queriedServersMutex.Unlock()
+	relayServers := []gomatrixserverlib.ServerName{}
+	for server := range r.relayServersQueried {
+		relayServers = append(relayServers, server)
+	}
+
+	return relayServers
+}
+
+func (r *RelayServerRetriever) GetQueriedServerStatus() map[gomatrixserverlib.ServerName]bool {
+	r.queriedServersMutex.Lock()
+	defer r.queriedServersMutex.Unlock()
+
+	result := map[gomatrixserverlib.ServerName]bool{}
+	for server, queried := range r.relayServersQueried {
+		result[server] = queried
+	}
+	return result
+}
+
+func (r *RelayServerRetriever) StartSync() {
+	if !r.running.Load() {
+		logrus.Info("Starting relay server sync")
+		go r.SyncRelayServers(r.quit)
+	}
+}
+
+func (r *RelayServerRetriever) IsRunning() bool {
+	return r.running.Load()
+}
+
+func (r *RelayServerRetriever) SyncRelayServers(stop <-chan bool) {
+	defer r.running.Store(false)
+
+	t := time.NewTimer(relayServerRetryInterval)
+	for {
+		relayServersToQuery := []gomatrixserverlib.ServerName{}
+		func() {
+			r.queriedServersMutex.Lock()
+			defer r.queriedServersMutex.Unlock()
+			for server, complete := range r.relayServersQueried {
+				if !complete {
+					relayServersToQuery = append(relayServersToQuery, server)
+				}
+			}
+		}()
+		if len(relayServersToQuery) == 0 {
+			// All relay servers have been synced.
+			logrus.Info("Finished syncing with all known relays")
+			return
+		}
+		r.queryRelayServers(relayServersToQuery)
+		t.Reset(relayServerRetryInterval)
+
+		select {
+		case <-stop:
+			if !t.Stop() {
+				<-t.C
+			}
+			logrus.Info("Stopped relay server retriever")
+			return
+		case <-t.C:
+		}
+	}
+}
+
+func (r *RelayServerRetriever) queryRelayServers(relayServers []gomatrixserverlib.ServerName) {
+	logrus.Info("Querying relay servers for any available transactions")
+	for _, server := range relayServers {
+		userID, err := gomatrixserverlib.NewUserID("@user:"+string(r.serverName), false)
+		if err != nil {
+			return
+		}
+
+		logrus.Infof("Syncing with relay: %s", string(server))
+		err = r.relayAPI.PerformRelayServerSync(context.Background(), *userID, server)
+		if err == nil {
+			func() {
+				r.queriedServersMutex.Lock()
+				defer r.queriedServersMutex.Unlock()
+				r.relayServersQueried[server] = true
+			}()
+			// TODO : What happens if your relay receives new messages after this point?
+			// Should you continue to check with them, or should they try and contact you?
+			// They could send a "new_async_events" message your way maybe?
+			// Then you could mark them as needing to be queried again.
+			// What if you miss this message?
+			// Maybe you should try querying them again after a certain period of time as a backup?
+		} else {
+			logrus.Errorf("Failed querying relay server: %s", err.Error())
+		}
+	}
+}
+
+func UpdateNodeRelayServers(
+	node gomatrixserverlib.ServerName,
+	relays []gomatrixserverlib.ServerName,
+	ctx context.Context,
+	fedAPI federationAPI.FederationInternalAPI,
+) {
+	// Get the current relay list
+	request := federationAPI.P2PQueryRelayServersRequest{Server: node}
+	response := federationAPI.P2PQueryRelayServersResponse{}
+	err := fedAPI.P2PQueryRelayServers(ctx, &request, &response)
+	if err != nil {
+		logrus.Warnf("Failed obtaining list of relay servers for %s: %s", node, err.Error())
+	}
+
+	// Remove old, non-matching relays
+	var serversToRemove []gomatrixserverlib.ServerName
+	for _, existingServer := range response.RelayServers {
+		shouldRemove := true
+		for _, newServer := range relays {
+			if newServer == existingServer {
+				shouldRemove = false
+				break
+			}
+		}
+
+		if shouldRemove {
+			serversToRemove = append(serversToRemove, existingServer)
+		}
+	}
+	removeRequest := federationAPI.P2PRemoveRelayServersRequest{
+		Server:       node,
+		RelayServers: serversToRemove,
+	}
+	removeResponse := federationAPI.P2PRemoveRelayServersResponse{}
+	err = fedAPI.P2PRemoveRelayServers(ctx, &removeRequest, &removeResponse)
+	if err != nil {
+		logrus.Warnf("Failed removing old relay servers for %s: %s", node, err.Error())
+	}
+
+	// Add new relays
+	addRequest := federationAPI.P2PAddRelayServersRequest{
+		Server:       node,
+		RelayServers: relays,
+	}
+	addResponse := federationAPI.P2PAddRelayServersResponse{}
+	err = fedAPI.P2PAddRelayServers(ctx, &addRequest, &addResponse)
+	if err != nil {
+		logrus.Warnf("Failed adding relay servers for %s: %s", node, err.Error())
+	}
+}

cmd/dendrite-demo-pinecone/relay/retriever_test.go

@@ -0,0 +1,99 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package relay
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
+	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"gotest.tools/v3/poll"
+)
+
+var testRelayServers = []gomatrixserverlib.ServerName{"relay1", "relay2"}
+
+type FakeFedAPI struct {
+	federationAPI.FederationInternalAPI
+}
+
+func (f *FakeFedAPI) P2PQueryRelayServers(
+	ctx context.Context,
+	req *federationAPI.P2PQueryRelayServersRequest,
+	res *federationAPI.P2PQueryRelayServersResponse,
+) error {
+	res.RelayServers = testRelayServers
+	return nil
+}
+
+type FakeRelayAPI struct {
+	relayServerAPI.RelayInternalAPI
+}
+
+func (r *FakeRelayAPI) PerformRelayServerSync(
+	ctx context.Context,
+	userID gomatrixserverlib.UserID,
+	relayServer gomatrixserverlib.ServerName,
+) error {
+	return nil
+}
+
+func TestRelayRetrieverInitialization(t *testing.T) {
+	retriever := NewRelayServerRetriever(
+		context.Background(),
+		"server",
+		&FakeFedAPI{},
+		&FakeRelayAPI{},
+		make(chan bool),
+	)
+
+	retriever.InitializeRelayServers(logrus.WithField("test", "relay"))
+	relayServers := retriever.GetQueriedServerStatus()
+	assert.Equal(t, 2, len(relayServers))
+}
+
+func TestRelayRetrieverSync(t *testing.T) {
+	retriever := NewRelayServerRetriever(
+		context.Background(),
+		"server",
+		&FakeFedAPI{},
+		&FakeRelayAPI{},
+		make(chan bool),
+	)
+
+	retriever.InitializeRelayServers(logrus.WithField("test", "relay"))
+	relayServers := retriever.GetQueriedServerStatus()
+	assert.Equal(t, 2, len(relayServers))
+
+	stopRelayServerSync := make(chan bool)
+	go retriever.SyncRelayServers(stopRelayServerSync)
+
+	check := func(log poll.LogT) poll.Result {
+		relayServers := retriever.GetQueriedServerStatus()
+		for _, queried := range relayServers {
+			if !queried {
+				return poll.Continue("waiting for all servers to be queried")
+			}
+		}
+
+		stopRelayServerSync <- true
+		return poll.Success()
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+}

cmd/dendrite-demo-yggdrasil/main.go

@@ -39,7 +39,6 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/dendrite/keyserver"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/base"
@@ -117,8 +116,8 @@ func main() {
 		cfg = setup.ParseFlags(true)
 	} else {
 		cfg.Defaults(config.DefaultOpts{
-			Generate:   true,
-			Monolithic: true,
+			Generate:       true,
+			SingleDatabase: true,
 		})
 		cfg.Global.PrivateKey = sk
 		cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName))
@@ -143,7 +142,7 @@ func main() {
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 
-	base := base.NewBaseDendrite(cfg, "Monolith")
+	base := base.NewBaseDendrite(cfg)
 	base.ConfigureAdminEndpoints()
 	defer base.Close() // nolint: errcheck
 
@@ -157,16 +156,11 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	rsComponent := roomserver.NewInternalAPI(
+	rsAPI := roomserver.NewInternalAPI(
 		base,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsComponent)
-
-	rsAPI := rsComponent
-
-	userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
-	keyAPI.SetUserAPI(userAPI)
+	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)
@@ -174,7 +168,7 @@ func main() {
 		base, federation, rsAPI, base.Caches, keyRing, true,
 	)
 
-	rsComponent.SetFederationAPI(fsAPI, keyRing)
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
@@ -186,7 +180,6 @@ func main() {
 		FederationAPI: fsAPI,
 		RoomserverAPI: rsAPI,
 		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
@@ -197,7 +190,6 @@ func main() {
 	}
 
 	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
 	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
 	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
 	httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)

cmd/dendrite-monolith-server/main.go

@@ -1,178 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"os"
-
-	"github.com/sirupsen/logrus"
-
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/federationapi"
-	"github.com/matrix-org/dendrite/keyserver"
-	"github.com/matrix-org/dendrite/roomserver"
-	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/setup"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/mscs"
-	"github.com/matrix-org/dendrite/userapi"
-	uapi "github.com/matrix-org/dendrite/userapi/api"
-)
-
-var (
-	httpBindAddr   = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
-	httpsBindAddr  = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
-	apiBindAddr    = flag.String("api-bind-address", "localhost:18008", "The HTTP listening port for the internal HTTP APIs (if -api is enabled)")
-	certFile       = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
-	keyFile        = flag.String("tls-key", "", "The PEM private key to use for TLS")
-	enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)")
-	traceInternal  = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1"
-)
-
-func main() {
-	cfg := setup.ParseFlags(true)
-	httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
-	httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
-	httpAPIAddr := httpAddr
-	options := []basepkg.BaseDendriteOptions{}
-	if *enableHTTPAPIs {
-		logrus.Warnf("DANGER! The -api option is enabled, exposing internal APIs on %q!", *apiBindAddr)
-		httpAPIAddr = config.HTTPAddress("http://" + *apiBindAddr)
-		// If the HTTP APIs are enabled then we need to update the Listen
-		// statements in the configuration so that we know where to find
-		// the API endpoints. They'll listen on the same port as the monolith
-		// itself.
-		cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.KeyServer.InternalAPI.Connect = httpAPIAddr
-		cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.RoomServer.InternalAPI.Connect = httpAPIAddr
-		cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr
-		cfg.UserAPI.InternalAPI.Connect = httpAPIAddr
-		options = append(options, basepkg.UseHTTPAPIs)
-	}
-
-	base := basepkg.NewBaseDendrite(cfg, "Monolith", options...)
-	defer base.Close() // nolint: errcheck
-
-	federation := base.CreateFederationClient()
-
-	rsImpl := roomserver.NewInternalAPI(base)
-	// call functions directly on the impl unless running in HTTP mode
-	rsAPI := rsImpl
-	if base.UseHTTPAPIs {
-		roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics)
-		rsAPI = base.RoomserverHTTPClient()
-	}
-	if traceInternal {
-		rsAPI = &api.RoomserverInternalAPITrace{
-			Impl: rsAPI,
-		}
-	}
-
-	fsAPI := federationapi.NewInternalAPI(
-		base, federation, rsAPI, base.Caches, nil, false,
-	)
-	fsImplAPI := fsAPI
-	if base.UseHTTPAPIs {
-		federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
-		fsAPI = base.FederationAPIHTTPClient()
-	}
-	keyRing := fsAPI.KeyRing()
-
-	keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
-	keyAPI := keyImpl
-	if base.UseHTTPAPIs {
-		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics)
-		keyAPI = base.KeyServerHTTPClient()
-	}
-
-	pgClient := base.PushGatewayHTTPClient()
-	userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
-	userAPI := userImpl
-	if base.UseHTTPAPIs {
-		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
-		userAPI = base.UserAPIClient()
-	}
-	if traceInternal {
-		userAPI = &uapi.UserInternalAPITrace{
-			Impl: userAPI,
-		}
-	}
-
-	// TODO: This should use userAPI, not userImpl, but the appservice setup races with
-	// the listeners and panics at startup if it tries to create appservice accounts
-	// before the listeners are up.
-	asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
-	if base.UseHTTPAPIs {
-		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics)
-		asAPI = base.AppserviceHTTPClient()
-	}
-
-	// The underlying roomserver implementation needs to be able to call the fedsender.
-	// This is different to rsAPI which can be the http client which doesn't need this
-	// dependency. Other components also need updating after their dependencies are up.
-	rsImpl.SetFederationAPI(fsAPI, keyRing)
-	rsImpl.SetAppserviceAPI(asAPI)
-	rsImpl.SetUserAPI(userAPI)
-	keyImpl.SetUserAPI(userAPI)
-
-	monolith := setup.Monolith{
-		Config:    base.Cfg,
-		Client:    base.CreateClient(),
-		FedClient: federation,
-		KeyRing:   keyRing,
-
-		AppserviceAPI: asAPI,
-		// always use the concrete impl here even in -http mode because adding public routes
-		// must be done on the concrete impl not an HTTP client else fedapi will call itself
-		FederationAPI: fsImplAPI,
-		RoomserverAPI: rsAPI,
-		UserAPI:       userAPI,
-		KeyAPI:        keyAPI,
-	}
-	monolith.AddAllPublicRoutes(base)
-
-	if len(base.Cfg.MSCs.MSCs) > 0 {
-		if err := mscs.Enable(base, &monolith); err != nil {
-			logrus.WithError(err).Fatalf("Failed to enable MSCs")
-		}
-	}
-
-	// Expose the matrix APIs directly rather than putting them under a /api path.
-	go func() {
-		base.SetupAndServeHTTP(
-			httpAPIAddr, // internal API
-			httpAddr,    // external API
-			nil, nil,    // TLS settings
-		)
-	}()
-	// Handle HTTPS if certificate and key are provided
-	if *certFile != "" && *keyFile != "" {
-		go func() {
-			base.SetupAndServeHTTP(
-				basepkg.NoListener, // internal API
-				httpsAddr,          // external API
-				certFile, keyFile,  // TLS settings
-			)
-		}()
-	}
-
-	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
-	base.WaitForShutdown()
-}

cmd/dendrite-polylith-multi/main.go

@@ -1,77 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"os"
-	"strings"
-
-	"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
-	"github.com/matrix-org/dendrite/setup"
-	"github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/sirupsen/logrus"
-)
-
-type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)
-
-func main() {
-	cfg := setup.ParseFlags(false)
-
-	component := ""
-	if flag.NFlag() > 0 {
-		component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
-	} else if len(os.Args) > 1 {
-		component = os.Args[1] // ./dendrite-polylith-multi clientapi
-	}
-
-	components := map[string]entrypoint{
-		"appservice":    personalities.Appservice,
-		"clientapi":     personalities.ClientAPI,
-		"federationapi": personalities.FederationAPI,
-		"keyserver":     personalities.KeyServer,
-		"mediaapi":      personalities.MediaAPI,
-		"roomserver":    personalities.RoomServer,
-		"syncapi":       personalities.SyncAPI,
-		"userapi":       personalities.UserAPI,
-	}
-
-	start, ok := components[component]
-	if !ok {
-		if component == "" {
-			logrus.Errorf("No component specified")
-			logrus.Info("The first argument on the command line must be the name of the component to run")
-		} else {
-			logrus.Errorf("Unknown component %q specified", component)
-		}
-
-		var list []string
-		for c := range components {
-			list = append(list, c)
-		}
-		logrus.Infof("Valid components: %s", strings.Join(list, ", "))
-
-		os.Exit(1)
-	}
-
-	logrus.Infof("Starting %q component", component)
-
-	base := base.NewBaseDendrite(cfg, component, base.PolylithMode) // TODO
-	defer base.Close()                                              // nolint: errcheck
-
-	go start(base, cfg)
-	base.WaitForShutdown()
-}

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -1,36 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/appservice"
-	"github.com/matrix-org/dendrite/setup/base"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := base.UserAPIClient()
-	rsAPI := base.RoomserverHTTPClient()
-
-	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                        // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -1,44 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/clientapi"
-	"github.com/matrix-org/dendrite/internal/transactions"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	federation := base.CreateFederationClient()
-
-	asQuery := base.AppserviceHTTPClient()
-	rsAPI := base.RoomserverHTTPClient()
-	fsAPI := base.FederationAPIHTTPClient()
-	userAPI := base.UserAPIClient()
-	keyAPI := base.KeyServerHTTPClient()
-
-	clientapi.AddPublicRoutes(
-		base, federation, rsAPI, asQuery,
-		transactions.New(), fsAPI, userAPI, userAPI,
-		keyAPI, nil,
-	)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.ClientAPI.InternalAPI.Listen,
-		base.Cfg.ClientAPI.ExternalAPI.Listen,
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -1,44 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/federationapi"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := base.UserAPIClient()
-	federation := base.CreateFederationClient()
-	rsAPI := base.RoomserverHTTPClient()
-	keyAPI := base.KeyServerHTTPClient()
-	fsAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, nil, true)
-	keyRing := fsAPI.KeyRing()
-
-	federationapi.AddPublicRoutes(
-		base,
-		userAPI, federation, keyRing,
-		rsAPI, fsAPI, keyAPI, nil,
-	)
-
-	federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.FederationAPI.InternalAPI.Listen,
-		base.Cfg.FederationAPI.ExternalAPI.Listen,
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -1,36 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/keyserver"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	fsAPI := base.FederationAPIHTTPClient()
-	rsAPI := base.RoomserverHTTPClient()
-	intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
-	intAPI.SetUserAPI(base.UserAPIClient())
-
-	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.KeyServer.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                    // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -1,36 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	"github.com/matrix-org/dendrite/roomserver"
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	asAPI := base.AppserviceHTTPClient()
-	fsAPI := base.FederationAPIHTTPClient()
-	rsAPI := roomserver.NewInternalAPI(base)
-	rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
-	rsAPI.SetAppserviceAPI(asAPI)
-	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.RoomServer.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                     // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/syncapi.go

@@ -1,39 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/syncapi"
-)
-
-func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := base.UserAPIClient()
-
-	rsAPI := base.RoomserverHTTPClient()
-
-	syncapi.AddPublicRoutes(
-		base,
-		userAPI, rsAPI,
-		base.KeyServerHTTPClient(),
-	)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.SyncAPI.InternalAPI.Listen,
-		base.Cfg.SyncAPI.ExternalAPI.Listen,
-		nil, nil,
-	)
-}

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -1,37 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package personalities
-
-import (
-	basepkg "github.com/matrix-org/dendrite/setup/base"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi"
-)
-
-func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
-	userAPI := userapi.NewInternalAPI(
-		base, &cfg.UserAPI, cfg.Derived.ApplicationServices,
-		base.KeyServerHTTPClient(), base.RoomserverHTTPClient(),
-		base.PushGatewayHTTPClient(),
-	)
-
-	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
-
-	base.SetupAndServeHTTP(
-		base.Cfg.UserAPI.InternalAPI.Listen, // internal listener
-		basepkg.NoListener,                  // external listener
-		nil, nil,
-	)
-}

cmd/dendrite-upgrade-tests/main.go

@@ -45,6 +45,10 @@ var (
 
 const HEAD = "HEAD"
 
+// The binary was renamed after v0.11.1, so everything after that should use the new name
+var binaryChangeVersion, _ = semver.NewVersion("v0.11.1")
+var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD"
+
 // Embed the Dockerfile to use when building dendrite versions.
 // We cannot use the dockerfile associated with the repo with each version sadly due to changes in
 // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
@@ -54,12 +58,13 @@ const HEAD = "HEAD"
 const DockerfilePostgreSQL = `FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
+ARG BINARY
 
 # Copy the build context to the repo as this is the right dendrite code. This is different to the
 # Complement Dockerfile which wgets a branch.
 COPY . .
 
-RUN go build ./cmd/dendrite-monolith-server
+RUN go build ./cmd/${BINARY}
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
 RUN go build ./cmd/create-account
@@ -88,22 +93,24 @@ done \n\
  \n\
 sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
 PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
-./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
+./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\
 ' > run_dendrite.sh && chmod +x run_dendrite.sh
 
 ENV SERVER_NAME=localhost
+ENV BINARY=dendrite
 EXPOSE 8008 8448
-CMD /build/run_dendrite.sh `
+CMD /build/run_dendrite.sh`
 
 const DockerfileSQLite = `FROM golang:1.18-stretch as build
 RUN apt-get update && apt-get install -y postgresql
 WORKDIR /build
+ARG BINARY
 
 # Copy the build context to the repo as this is the right dendrite code. This is different to the
 # Complement Dockerfile which wgets a branch.
 COPY . .
 
-RUN go build ./cmd/dendrite-monolith-server
+RUN go build ./cmd/${BINARY}
 RUN go build ./cmd/generate-keys
 RUN go build ./cmd/generate-config
 RUN go build ./cmd/create-account
@@ -118,10 +125,11 @@ RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postg
 RUN echo '\
 sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
 PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
-./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
+./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\
 ' > run_dendrite.sh && chmod +x run_dendrite.sh
 
 ENV SERVER_NAME=localhost
+ENV BINARY=dendrite
 EXPOSE 8008 8448
 CMD /build/run_dendrite.sh `
 
@@ -182,7 +190,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
 }
 
 // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
-func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
+func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir string, branchOrTagName, binary string) (string, error) {
 	var tarball *bytes.Buffer
 	var err error
 	// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
@@ -216,6 +224,9 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
 	log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
 	res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
 		Tags: []string{"dendrite-upgrade"},
+		BuildArgs: map[string]*string{
+			"BINARY": &binary,
+		},
 	})
 	if err != nil {
 		return "", fmt.Errorf("failed to start building image: %s", err)
@@ -272,7 +283,7 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve
 	return semVers, nil
 }
 
-func calculateVersions(cli *http.Client, from, to string, direct bool) []string {
+func calculateVersions(cli *http.Client, from, to string, direct bool) []*semver.Version {
 	semvers, err := getAndSortVersionsFromGithub(cli)
 	if err != nil {
 		log.Fatalf("failed to collect semvers from github: %s", err)
@@ -320,28 +331,25 @@ func calculateVersions(cli *http.Client, from, to string, direct bool) []string
 		}
 		semvers = semvers[:i+1]
 	}
-	var versions []string
-	for _, sv := range semvers {
-		versions = append(versions, sv.Original())
-	}
+
 	if to == HEAD {
-		versions = append(versions, HEAD)
+		semvers = append(semvers, latest)
 	}
 	if direct {
-		versions = []string{versions[0], versions[len(versions)-1]}
+		semvers = []*semver.Version{semvers[0], semvers[len(semvers)-1]}
 	}
-	return versions
+	return semvers
 }
 
-func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, branchOrTagNames []string) map[string]string {
+func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, versions []*semver.Version) map[string]string {
 	// concurrently build all versions, this can be done in any order. The mutex protects the map
 	branchToImageID := make(map[string]string)
 	var mu sync.Mutex
 
 	var wg sync.WaitGroup
 	wg.Add(concurrency)
-	ch := make(chan string, len(branchOrTagNames))
-	for _, branchName := range branchOrTagNames {
+	ch := make(chan *semver.Version, len(versions))
+	for _, branchName := range versions {
 		ch <- branchName
 	}
 	close(ch)
@@ -349,11 +357,13 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b
 	for i := 0; i < concurrency; i++ {
 		go func() {
 			defer wg.Done()
-			for branchName := range ch {
+			for version := range ch {
+				branchName, binary := versionToBranchAndBinary(version)
+				log.Printf("Building version %s with binary %s", branchName, binary)
 				tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "")
-				imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName)
+				imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName, binary)
 				if err != nil {
-					log.Fatalf("%s: failed to build dendrite image: %s", branchName, err)
+					log.Fatalf("%s: failed to build dendrite image: %s", version, err)
 				}
 				mu.Lock()
 				branchToImageID[branchName] = imgID
@@ -365,13 +375,14 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b
 	return branchToImageID
 }
 
-func runImage(dockerClient *client.Client, volumeName, version, imageID string) (csAPIURL, containerID string, err error) {
-	log.Printf("%s: running image %s\n", version, imageID)
+func runImage(dockerClient *client.Client, volumeName string, branchNameToImageID map[string]string, version *semver.Version) (csAPIURL, containerID string, err error) {
+	branchName, binary := versionToBranchAndBinary(version)
+	imageID := branchNameToImageID[branchName]
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancel()
 	body, err := dockerClient.ContainerCreate(ctx, &container.Config{
 		Image: imageID,
-		Env:   []string{"SERVER_NAME=hs1"},
+		Env:   []string{"SERVER_NAME=hs1", fmt.Sprintf("BINARY=%s", binary)},
 		Labels: map[string]string{
 			dendriteUpgradeTestLabel: "yes",
 		},
@@ -384,7 +395,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
 				Target: "/var/lib/postgresql/9.6/main",
 			},
 		},
-	}, nil, nil, "dendrite_upgrade_test_"+version)
+	}, nil, nil, "dendrite_upgrade_test_"+branchName)
 	if err != nil {
 		return "", "", fmt.Errorf("failed to ContainerCreate: %s", err)
 	}
@@ -451,8 +462,8 @@ func destroyContainer(dockerClient *client.Client, containerID string) {
 	}
 }
 
-func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchToImageID map[string]string) error {
-	csAPIURL, containerID, err := runImage(dockerClient, volumeName, v, branchToImageID[v])
+func loadAndRunTests(dockerClient *client.Client, volumeName string, v *semver.Version, branchToImageID map[string]string) error {
+	csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, v)
 	if err != nil {
 		return fmt.Errorf("failed to run container for branch %v: %v", v, err)
 	}
@@ -470,9 +481,10 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
 }
 
 // test that create-account is working
-func testCreateAccount(dockerClient *client.Client, v string, containerID string) error {
-	createUser := strings.ToLower("createaccountuser-" + v)
-	log.Printf("%s: Creating account %s with create-account\n", v, createUser)
+func testCreateAccount(dockerClient *client.Client, version *semver.Version, containerID string) error {
+	branchName, _ := versionToBranchAndBinary(version)
+	createUser := strings.ToLower("createaccountuser-" + branchName)
+	log.Printf("%s: Creating account %s with create-account\n", branchName, createUser)
 
 	respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
 		AttachStderr: true,
@@ -504,9 +516,21 @@ func testCreateAccount(dockerClient *client.Client, v string, containerID string
 	return nil
 }
 
-func verifyTests(dockerClient *client.Client, volumeName string, versions []string, branchToImageID map[string]string) error {
+func versionToBranchAndBinary(version *semver.Version) (branchName, binary string) {
+	binary = "dendrite-monolith-server"
+	branchName = version.Original()
+	if version.GreaterThan(binaryChangeVersion) {
+		binary = "dendrite"
+		if version.Equal(latest) {
+			branchName = HEAD
+		}
+	}
+	return
+}
+
+func verifyTests(dockerClient *client.Client, volumeName string, versions []*semver.Version, branchToImageID map[string]string) error {
 	lastVer := versions[len(versions)-1]
-	csAPIURL, containerID, err := runImage(dockerClient, volumeName, lastVer, branchToImageID[lastVer])
+	csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, lastVer)
 	if err != nil {
 		return fmt.Errorf("failed to run container for branch %v: %v", lastVer, err)
 	}

cmd/dendrite-upgrade-tests/tests.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -22,7 +23,8 @@ type user struct {
 // - register alice and bob with branch name muxed into the localpart
 // - create a DM room for the 2 users and exchange messages
 // - create/join a public #global room and exchange messages
-func runTests(baseURL, branchName string) error {
+func runTests(baseURL string, v *semver.Version) error {
+	branchName, _ := versionToBranchAndBinary(v)
 	// register 2 users
 	users := []user{
 		{
@@ -164,15 +166,16 @@ func runTests(baseURL, branchName string) error {
 }
 
 // verifyTestsRan checks that the HS has the right rooms/messages
-func verifyTestsRan(baseURL string, branchNames []string) error {
+func verifyTestsRan(baseURL string, versions []*semver.Version) error {
 	log.Println("Verifying tests....")
 	// check we can login as all users
 	var resp *gomatrix.RespLogin
-	for _, branchName := range branchNames {
+	for _, version := range versions {
 		client, err := gomatrix.NewClient(baseURL, "", "")
 		if err != nil {
 			return err
 		}
+		branchName, _ := versionToBranchAndBinary(version)
 		userLocalparts := []string{
 			"alice" + branchName,
 			"bob" + branchName,
@@ -224,7 +227,7 @@ func verifyTestsRan(baseURL string, branchNames []string) error {
 			msgCount += 1
 		}
 	}
-	wantMsgCount := len(branchNames) * 4
+	wantMsgCount := len(versions) * 4
 	if msgCount != wantMsgCount {
 		return fmt.Errorf("got %d messages in global room, want %d", msgCount, wantMsgCount)
 	}

cmd/dendrite/main.go

@@ -0,0 +1,103 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	basepkg "github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/mscs"
+	"github.com/matrix-org/dendrite/userapi"
+)
+
+var (
+	httpBindAddr  = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
+	httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
+	certFile      = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
+	keyFile       = flag.String("tls-key", "", "The PEM private key to use for TLS")
+)
+
+func main() {
+	cfg := setup.ParseFlags(true)
+	httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
+	httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
+	options := []basepkg.BaseDendriteOptions{}
+
+	base := basepkg.NewBaseDendrite(cfg, options...)
+	defer base.Close() // nolint: errcheck
+
+	federation := base.CreateFederationClient()
+
+	rsAPI := roomserver.NewInternalAPI(base)
+
+	fsAPI := federationapi.NewInternalAPI(
+		base, federation, rsAPI, base.Caches, nil, false,
+	)
+
+	keyRing := fsAPI.KeyRing()
+
+	userAPI := userapi.NewInternalAPI(base, rsAPI, federation)
+
+	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this
+	// dependency. Other components also need updating after their dependencies are up.
+	rsAPI.SetFederationAPI(fsAPI, keyRing)
+	rsAPI.SetAppserviceAPI(asAPI)
+	rsAPI.SetUserAPI(userAPI)
+
+	monolith := setup.Monolith{
+		Config:    base.Cfg,
+		Client:    base.CreateClient(),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI: asAPI,
+		// always use the concrete impl here even in -http mode because adding public routes
+		// must be done on the concrete impl not an HTTP client else fedapi will call itself
+		FederationAPI: fsAPI,
+		RoomserverAPI: rsAPI,
+		UserAPI:       userAPI,
+	}
+	monolith.AddAllPublicRoutes(base)
+
+	if len(base.Cfg.MSCs.MSCs) > 0 {
+		if err := mscs.Enable(base, &monolith); err != nil {
+			logrus.WithError(err).Fatalf("Failed to enable MSCs")
+		}
+	}
+
+	// Expose the matrix APIs directly rather than putting them under a /api path.
+	go func() {
+		base.SetupAndServeHTTP(httpAddr, nil, nil)
+	}()
+	// Handle HTTPS if certificate and key are provided
+	if *certFile != "" && *keyFile != "" {
+		go func() {
+			base.SetupAndServeHTTP(httpsAddr, certFile, keyFile)
+		}()
+	}
+
+	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
+	base.WaitForShutdown()
+}

cmd/generate-config/main.go

@@ -18,7 +18,6 @@ func main() {
 	dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)")
 	dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)")
 	normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults")
-	polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments")
 	flag.Parse()
 
 	var cfg *config.Dendrite
@@ -27,14 +26,14 @@ func main() {
 			Version: config.Version,
 		}
 		cfg.Defaults(config.DefaultOpts{
-			Generate:   true,
-			Monolithic: !*polylith,
+			Generate:       true,
+			SingleDatabase: true,
 		})
 		if *serverName != "" {
 			cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
 		}
 		uri := config.DataSource(*dbURI)
-		if *polylith || uri.IsSQLite() || uri == "" {
+		if uri.IsSQLite() || uri == "" {
 			for name, db := range map[string]*config.DatabaseOptions{
 				"federationapi": &cfg.FederationAPI.Database,
 				"keyserver":     &cfg.KeyServer.Database,
@@ -43,6 +42,7 @@ func main() {
 				"roomserver":    &cfg.RoomServer.Database,
 				"syncapi":       &cfg.SyncAPI.Database,
 				"userapi":       &cfg.UserAPI.AccountDatabase,
+				"relayapi":      &cfg.RelayAPI.Database,
 			} {
 				if uri == "" {
 					path := filepath.Join(*dirPath, fmt.Sprintf("dendrite_%s.db", name))
@@ -54,6 +54,9 @@ func main() {
 		} else {
 			cfg.Global.DatabaseOptions.ConnectionString = uri
 		}
+		cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
+		cfg.Global.JetStream.StoragePath = config.Path(*dirPath)
+		cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(*dirPath, "searchindex"))
 		cfg.Logging = []config.LogrusHook{
 			{
 				Type:  "file",
@@ -67,6 +70,7 @@ func main() {
 			cfg.AppServiceAPI.DisableTLSValidation = true
 			cfg.ClientAPI.RateLimiting.Enabled = false
 			cfg.FederationAPI.DisableTLSValidation = false
+			cfg.FederationAPI.DisableHTTPKeepalives = true
 			// don't hit matrix.org when running tests!!!
 			cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
 			cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
@@ -92,7 +96,7 @@ func main() {
 		}
 	} else {
 		var err error
-		if cfg, err = config.Load(*normalise, !*polylith); err != nil {
+		if cfg, err = config.Load(*normalise); err != nil {
 			panic(err)
 		}
 	}

cmd/resolve-state/main.go

@@ -40,7 +40,7 @@ func main() {
 		Level: "error",
 	})
 	cfg.ClientAPI.RegistrationDisabled = true
-	base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
+	base := base.NewBaseDendrite(cfg, base.DisableMetrics)
 	args := flag.Args()
 
 	fmt.Println("Room version", *roomVersion)
@@ -87,7 +87,7 @@ func main() {
 		}
 
 		var eventEntries []types.Event
-		eventEntries, err = roomserverDB.Events(ctx, eventNIDs)
+		eventEntries, err = roomserverDB.Events(ctx, 0, eventNIDs)
 		if err != nil {
 			panic(err)
 		}
@@ -145,7 +145,7 @@ func main() {
 	}
 
 	fmt.Println("Fetching", len(eventNIDMap), "state events")
-	eventEntries, err := roomserverDB.Events(ctx, eventNIDs)
+	eventEntries, err := roomserverDB.Events(ctx, 0, eventNIDs)
 	if err != nil {
 		panic(err)
 	}
@@ -165,7 +165,7 @@ func main() {
 	}
 
 	fmt.Println("Fetching", len(authEventIDs), "auth events")
-	authEventEntries, err := roomserverDB.EventsFromIDs(ctx, authEventIDs)
+	authEventEntries, err := roomserverDB.EventsFromIDs(ctx, 0, authEventIDs)
 	if err != nil {
 		panic(err)
 	}

dendrite-sample.polylith.yaml

@@ -1,417 +0,0 @@
-# This is the Dendrite configuration file.
-#
-# The configuration is split up into sections - each Dendrite component has a
-# configuration section, in addition to the "global" section which applies to
-# all components.
-
-# The version of the configuration file.
-version: 2
-
-# Global Matrix configuration. This configuration applies to all components.
-global:
-  # The domain name of this homeserver.
-  server_name: localhost
-
-  # The path to the signing private key file, used to sign requests and events.
-  # Note that this is NOT the same private key as used for TLS! To generate a
-  # signing key, use "./bin/generate-keys --private-key matrix_key.pem".
-  private_key: matrix_key.pem
-
-  # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
-  # to old signing keys that were formerly in use on this domain name. These
-  # keys will not be used for federation request or event signing, but will be
-  # provided to any other homeserver that asks when trying to verify old events.
-  old_private_keys:
-  #  If the old private key file is available:
-  #  - private_key: old_matrix_key.pem
-  #    expired_at: 1601024554498
-  #  If only the public key (in base64 format) and key ID are known:
-  #  - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
-  #    key_id: ed25519:mykeyid
-  #    expired_at: 1601024554498
-
-  # How long a remote server can cache our server signing key before requesting it
-  # again. Increasing this number will reduce the number of requests made by other
-  # servers for our key but increases the period that a compromised key will be
-  # considered valid by other homeservers.
-  key_validity_period: 168h0m0s
-
-  # Configuration for in-memory caches. Caches can often improve performance by
-  # keeping frequently accessed items (like events, identifiers etc.) in memory
-  # rather than having to read them from the database.
-  cache:
-    # The estimated maximum size for the global cache in bytes, or in terabytes,
-    # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or
-    # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a
-    # memory limit for the entire process. A cache that is too small may ultimately
-    # provide little or no benefit.
-    max_size_estimated: 1gb
-
-    # The maximum amount of time that a cache entry can live for in memory before
-    # it will be evicted and/or refreshed from the database. Lower values result in
-    # easier admission of new cache entries but may also increase database load in
-    # comparison to higher values, so adjust conservatively. Higher values may make
-    # it harder for new items to make it into the cache, e.g. if new rooms suddenly
-    # become popular.
-    max_age: 1h
-
-  # The server name to delegate server-server communications to, with optional port
-  # e.g. localhost:443
-  well_known_server_name: ""
-
-  # The server name to delegate client-server communications to, with optional port
-  # e.g. localhost:443
-  well_known_client_name: ""
-
-  # Lists of domains that the server will trust as identity servers to verify third
-  # party identifiers such as phone numbers and email addresses.
-  trusted_third_party_id_servers:
-    - matrix.org
-    - vector.im
-
-  # Disables federation. Dendrite will not be able to communicate with other servers
-  # in the Matrix federation and the federation API will not be exposed.
-  disable_federation: false
-
-  # Configures the handling of presence events. Inbound controls whether we receive
-  # presence events from other servers, outbound controls whether we send presence
-  # events for our local users to other servers.
-  presence:
-    enable_inbound: false
-    enable_outbound: false
-
-  # Configures phone-home statistics reporting. These statistics contain the server
-  # name, number of active users and some information on your deployment config.
-  # We use this information to understand how Dendrite is being used in the wild.
-  report_stats:
-    enabled: false
-    endpoint: https://matrix.org/report-usage-stats/push
-
-  # Server notices allows server admins to send messages to all users on the server.
-  server_notices:
-    enabled: false
-    # The local part, display name and avatar URL (as a mxc:// URL) for the user that
-    # will send the server notices. These are visible to all users on the deployment.
-    local_part: "_server"
-    display_name: "Server Alerts"
-    avatar_url: ""
-    # The room name to be used when sending server notices. This room name will
-    # appear in user clients.
-    room_name: "Server Alerts"
-
-  # Configuration for NATS JetStream
-  jetstream:
-    # A list of NATS Server addresses to connect to. If none are specified, an
-    # internal NATS server will be started automatically when running Dendrite in
-    # monolith mode. For polylith deployments, it is required to specify the address
-    # of at least one NATS Server node.
-    addresses:
-      - hostname:4222
-
-    # Disable the validation of TLS certificates of NATS. This is
-    # not recommended in production since it may allow NATS traffic
-    # to be sent to an insecure endpoint.
-    disable_tls_validation: false
-
-    # The prefix to use for stream names for this homeserver - really only useful
-    # if you are running more than one Dendrite server on the same NATS deployment.
-    topic_prefix: Dendrite
-
-  # Configuration for Prometheus metric collection.
-  metrics:
-    enabled: false
-    basic_auth:
-      username: metrics
-      password: metrics
-
-  # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there
-  # is no local caching resolver available for use.
-  dns_cache:
-    enabled: false
-    cache_size: 256
-    cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration
-
-# Configuration for the Appservice API.
-app_service_api:
-  internal_api:
-    listen: http://[::]:7777 # The listen address for incoming API requests
-    connect: http://app_service_api:7777 # The connect address for other components to use
-
-  # Disable the validation of TLS certificates of appservices. This is
-  # not recommended in production since it may allow appservice traffic
-  # to be sent to an insecure endpoint.
-  disable_tls_validation: false
-
-  # Appservice configuration files to load into this homeserver.
-  config_files:
-  #  - /path/to/appservice_registration.yaml
-
-# Configuration for the Client API.
-client_api:
-  internal_api:
-    listen: http://[::]:7771 # The listen address for incoming API requests
-    connect: http://client_api:7771 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8071
-
-  # Prevents new users from being able to register on this homeserver, except when
-  # using the registration shared secret below.
-  registration_disabled: true
-
-  # Prevents new guest accounts from being created. Guest registration is also
-  # disabled implicitly by setting 'registration_disabled' above.
-  guests_disabled: true
-
-  # If set, allows registration by anyone who knows the shared secret, regardless
-  # of whether registration is otherwise disabled.
-  registration_shared_secret: ""
-
-  # Whether to require reCAPTCHA for registration. If you have enabled registration
-  # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used
-  # for coordinated spam attacks.
-  enable_registration_captcha: false
-
-  # Settings for ReCAPTCHA.
-  recaptcha_public_key: ""
-  recaptcha_private_key: ""
-  recaptcha_bypass_secret: ""
-
-  # To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty.
-  # recaptcha_siteverify_api: "https://hcaptcha.com/siteverify"
-  # recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js"
-  # recaptcha_form_field: "h-captcha-response"
-  # recaptcha_sitekey_class: "h-captcha"
-
-
-  # TURN server information that this homeserver should send to clients.
-  turn:
-    turn_user_lifetime: "5m"
-    turn_uris:
-    #  - turn:turn.server.org?transport=udp
-    #  - turn:turn.server.org?transport=tcp
-    turn_shared_secret: ""
-    # If your TURN server requires static credentials, then you will need to enter
-    # them here instead of supplying a shared secret. Note that these credentials
-    # will be visible to clients!
-    # turn_username: ""
-    # turn_password: ""
-
-  # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold
-  # number of "slots" have been taken by requests from a specific host. Each "slot"
-  # will be released after the cooloff time in milliseconds. Server administrators
-  # and appservice users are exempt from rate limiting by default.
-  rate_limiting:
-    enabled: true
-    threshold: 20
-    cooloff_ms: 500
-    exempt_user_ids:
-    #  - "@user:domain.com"
-
-# Configuration for the Federation API.
-federation_api:
-  internal_api:
-    listen: http://[::]:7772 # The listen address for incoming API requests
-    connect: http://federation_api:7772 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8072
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_federationapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # How many times we will try to resend a failed transaction to a specific server. The
-  # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once
-  # the max retries are exceeded, Dendrite will no longer try to send transactions to
-  # that server until it comes back to life and connects to us again.
-  send_max_retries: 16
-
-  # Disable the validation of TLS certificates of remote federated homeservers. Do not
-  # enable this option in production as it presents a security risk!
-  disable_tls_validation: false
-
-  # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically
-  # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much
-  # more quickly than opening new connections each time. Disabling keepalives will close
-  # HTTP connections immediately after a successful request but may result in more CPU and
-  # memory being used on TLS handshakes for each new connection instead.
-  disable_http_keepalives: false
-
-  # Perspective keyservers to use as a backup when direct key fetches fail. This may
-  # be required to satisfy key requests for servers that are no longer online when
-  # joining some rooms.
-  key_perspectives:
-    - server_name: matrix.org
-      keys:
-        - key_id: ed25519:auto
-          public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
-        - key_id: ed25519:a_RXGa
-          public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
-
-  # This option will control whether Dendrite will prefer to look up keys directly
-  # or whether it should try perspective servers first, using direct fetches as a
-  # last resort.
-  prefer_direct_fetch: false
-
-# Configuration for the Key Server (for end-to-end encryption).
-key_server:
-  internal_api:
-    listen: http://[::]:7779 # The listen address for incoming API requests
-    connect: http://key_server:7779 # The connect address for other components to use
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_keyserver?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Media API.
-media_api:
-  internal_api:
-    listen: http://[::]:7774 # The listen address for incoming API requests
-    connect: http://media_api:7774 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8074
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_mediaapi?sslmode=disable
-    max_open_conns: 5
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # Storage path for uploaded media. May be relative or absolute.
-  base_path: ./media_store
-
-  # The maximum allowed file size (in bytes) for media uploads to this homeserver
-  # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least
-  #this large (e.g. the client_max_body_size setting in nginx).
-  max_file_size_bytes: 10485760
-
-  # Whether to dynamically generate thumbnails if needed.
-  dynamic_thumbnails: false
-
-  # The maximum number of simultaneous thumbnail generators to run.
-  max_thumbnail_generators: 10
-
-  # A list of thumbnail sizes to be generated for media content.
-  thumbnail_sizes:
-    - width: 32
-      height: 32
-      method: crop
-    - width: 96
-      height: 96
-      method: crop
-    - width: 640
-      height: 480
-      method: scale
-
-# Configuration for enabling experimental MSCs on this homeserver.
-mscs:
-  mscs:
-  #  - msc2836  # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
-  #  - msc2946  # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_mscs?sslmode=disable
-    max_open_conns: 5
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Room Server.
-room_server:
-  internal_api:
-    listen: http://[::]:7770 # The listen address for incoming API requests
-    connect: http://room_server:7770 # The connect address for other components to use
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_roomserver?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-# Configuration for the Sync API.
-sync_api:
-  internal_api:
-    listen: http://[::]:7773 # The listen address for incoming API requests
-    connect: http://sync_api:7773 # The connect address for other components to use
-  external_api:
-    listen: http://[::]:8073
-  database:
-    connection_string: postgresql://username:password@hostname/dendrite_syncapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # Configuration for the full-text search engine.
-  search:
-    # Whether or not search is enabled.
-    enabled: false
-
-    # The path where the search index will be created in.
-    index_path: "./searchindex"
-
-    # The language most likely to be used on the server - used when indexing, to
-    # ensure the returned results match expectations. A full list of possible languages
-    # can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang
-    language: "en"
-
-  # This option controls which HTTP header to inspect to find the real remote IP
-  # address of the client. This is likely required if Dendrite is running behind
-  # a reverse proxy server.
-  # real_ip_header: X-Real-IP
-
-# Configuration for the User API.
-user_api:
-  internal_api:
-    listen: http://[::]:7781 # The listen address for incoming API requests
-    connect: http://user_api:7781 # The connect address for other components to use
-  account_database:
-    connection_string: postgresql://username:password@hostname/dendrite_userapi?sslmode=disable
-    max_open_conns: 10
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-
-  # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31
-  # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information.
-  # Setting this lower makes registration/login consume less CPU resources at the cost
-  # of security should the database be compromised. Setting this higher makes registration/login
-  # consume more CPU resources but makes it harder to brute force password hashes. This value
-  # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds).
-  bcrypt_cost: 10
-
-  # The length of time that a token issued for a relying party from
-  # /_matrix/client/r0/user/{userId}/openid/request_token endpoint
-  # is considered to be valid in milliseconds.
-  # The default lifetime is 3600000ms (60 minutes).
-  # openid_token_lifetime_ms: 3600000
-
-  # Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option.
-  # By default, any room aliases included in this list will be created as a publicly joinable room
-  # when the first user registers for the homeserver. If the room already exists,
-  # make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'.
-  # As Spaces are just rooms under the hood, Space aliases may also be used.
-  auto_join_rooms:
-  #  - "#main:matrix.org"
-
-# Configuration for Opentracing.
-# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
-# how this works and how to set it up.
-tracing:
-  enabled: false
-  jaeger:
-    serviceName: ""
-    disabled: false
-    rpc_metrics: false
-    tags: []
-    sampler: null
-    reporter: null
-    headers: null
-    baggage_restrictions: null
-    throttler: null
-
-# Logging configuration. The "std" logging type controls the logs being sent to
-# stdout. The "file" logging type controls logs being written to a log folder on
-# the disk. Supported log levels are "debug", "info", "warn", "error".
-logging:
-  - type: std
-    level: info
-  - type: file
-    level: info
-    params:
-      path: ./logs

dendrite-sample.yaml

@@ -38,7 +38,7 @@ global:
 
   # Global database connection pool, for PostgreSQL monolith deployments only. If
   # this section is populated then you can omit the "database" blocks in all other
-  # sections. For polylith deployments, or monolith deployments using SQLite databases,
+  # sections. For monolith deployments using SQLite databases,
   # you must configure the "database" block for each component instead.
   database:
     connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable
@@ -113,8 +113,7 @@ global:
   jetstream:
     # A list of NATS Server addresses to connect to. If none are specified, an
     # internal NATS server will be started automatically when running Dendrite in
-    # monolith mode. For polylith deployments, it is required to specify the address
-    # of at least one NATS Server node.
+    # monolith mode.
     addresses:
     # - localhost:4222
 

docs/FAQ.md

@@ -35,11 +35,6 @@ possible to migrate an existing Synapse deployment to Dendrite.
 
 No, Dendrite has a very different database schema to Synapse and the two are not interchangeable.
 
-## Should I run a monolith or a polylith deployment?
-
-Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't
-recommend.
-
 ## Can I configure which port Dendrite listens on?
 
 Yes, use the cli flag `-http-bind-address`. 

docs/Gemfile.lock

@@ -1,7 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.5)
+    activesupport (6.0.6.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -14,8 +14,8 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.6)
-    concurrent-ruby (1.1.10)
+    commonmarker (0.23.7)
+    concurrent-ruby (1.2.0)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)
     em-websocket (0.5.3)
@@ -229,7 +229,7 @@ GEM
       jekyll (>= 3.5, < 5.0)
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
-    minitest (5.15.0)
+    minitest (5.17.0)
     multipart-post (2.1.1)
     nokogiri (1.13.10-arm64-darwin)
       racc (~> 1.4)
@@ -265,13 +265,13 @@ GEM
     thread_safe (0.3.6)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
-    tzinfo (1.2.10)
+    tzinfo (1.2.11)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.1)
     unicode-display_width (1.8.0)
-    zeitwerk (2.5.4)
+    zeitwerk (2.6.6)
 
 PLATFORMS
   arm64-darwin-21

docs/INSTALL.md

@@ -9,7 +9,5 @@ or alternatively, in the [installation](installation/) folder:
 3. [Preparing database storage](installation/3_database.md)
 4. [Generating signing keys](installation/4_signingkey.md)
 5. [Installing as a monolith](installation/5_install_monolith.md)
-6. [Installing as a polylith](installation/6_install_polylith.md)
-7. [Populate the configuration](installation/7_configuration.md)
-8. [Starting the monolith](installation/8_starting_monolith.md)
-9. [Starting the polylith](installation/9_starting_polylith.md)
+6. [Populate the configuration](installation/7_configuration.md)
+7. [Starting the monolith](installation/8_starting_monolith.md)

docs/development/coverage.md

@@ -57,22 +57,16 @@ github.com/matrix-org/util/unique.go:55:										UniqueStrings					100.0%
 total:															(statements)					53.7%
 ```
 The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations,
-which will never be tested in a single test run (e.g sqlite or postgres, monolith or polylith). To get a more accurate value, additional processing is required
+which will never be tested in a single test run (e.g sqlite or postgres). To get a more accurate value, additional processing is required
 to remove packages which will never be tested and extension MSCs:
 ```bash
 # These commands are all similar but change which package paths are _removed_ from the output.
 
-# For Postgres (monolith)
+# For Postgres
 go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt
 
-# For Postgres (polylith)
-go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'sqlite|setup/mscs|api_trace' > coverage.txt
-
-# For SQLite (monolith)
+# For SQLite
 go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt
-
-# For SQLite (polylith)
-go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'postgres|setup/mscs|api_trace' > coverage.txt
 ```
 
 A total value can then be calculated using:

docs/development/tracing/setup.md

@@ -46,10 +46,10 @@ tracing:
       param: 1
 ```
 
-then run the monolith server with `--api true` to use polylith components which do tracing spans:
+then run the monolith server:
 
 ```
-./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml --api true
+./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
 ```
 
 ## Checking traces

docs/installation/10_starting_polylith.md

@@ -1,73 +0,0 @@
----
-title: Starting the polylith
-parent: Installation
-has_toc: true
-nav_order: 10
-permalink: /installation/start/polylith
----
-
-# Starting the polylith
-
-Once you have completed all of the preparation and installation steps,
-you can start your Dendrite polylith deployment by starting the various components
-using the `dendrite-polylith-multi` personalities.
-
-## Start the reverse proxy
-
-Ensure that your reverse proxy is started and is proxying the correct
-endpoints to the correct components. Software such as [NGINX](https://www.nginx.com) or
-[HAProxy](http://www.haproxy.org) can be used for this purpose. A [sample configuration
-for NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf)
-is provided.
-
-## Starting the components
-
-Each component must be started individually:
-
-### Client API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml clientapi
-```
-
-### Sync API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml syncapi
-```
-
-### Media API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml mediaapi
-```
-
-### Federation API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml federationapi
-```
-
-### Roomserver
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml roomserver
-```
-
-### Appservice API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml appservice
-```
-
-### User API
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml userapi
-```
-
-### Key server
-
-```bash
-./dendrite-polylith-multi -config /path/to/dendrite.yaml keyserver
-```

docs/installation/1_planning.md

@@ -16,12 +16,6 @@ Users can run Dendrite in one of two modes which dictate how these components ar
   server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the
   best balance between performance and resource usage for low-to-mid volume deployments.
 
-* **Polylith mode** runs all components in isolated processes. Components communicate through an external NATS
-  server and HTTP APIs, which incur considerable overhead. While this mode allows for more granular control of
-  resources dedicated toward individual processes, given the additional communications overhead, it is only
-  necessary for very large deployments.
-
-Given our current state of development, **we recommend monolith mode** for all deployments.
 
 ## Databases
 
@@ -85,21 +79,15 @@ If using the PostgreSQL database engine, you should install PostgreSQL 12 or lat
 
 ### NATS Server
 
-Monolith deployments come with a built-in [NATS Server](https://github.com/nats-io/nats-server) and
-therefore do not need this to be manually installed. If you are planning a monolith installation, you
+Dendrite comes with a built-in [NATS Server](https://github.com/nats-io/nats-server) and
+therefore does not need this to be manually installed. If you are planning a monolith installation, you
 do not need to do anything.
 
-Polylith deployments, however, currently need a standalone NATS Server installation with JetStream
-enabled.
-
-To do so, follow the [NATS Server installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation) and then [start your NATS deployment](https://docs.nats.io/running-a-nats-service/introduction/running). JetStream must be enabled, either by passing the `-js` flag to `nats-server`,
-or by specifying the `store_dir` option in the the `jetstream` configuration.
 
 ### Reverse proxy
 
 A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or
-[HAProxy](http://www.haproxy.org) is required for polylith deployments and is useful for monolith
-deployments. Configuring those is not covered in this documentation, although sample configurations
+[HAProxy](http://www.haproxy.org) is useful for deployments. Configuring those is not covered in this documentation, although sample configurations
 for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and
 [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided.
 

docs/installation/6_install_polylith.md

@@ -1,34 +0,0 @@
----
-title: Installing as a polylith
-parent: Installation
-has_toc: true
-nav_order: 6
-permalink: /installation/install/polylith
----
-
-# Installing as a polylith
-
-You can install the Dendrite polylith binary into `$GOPATH/bin` by using `go install`:
-
-```sh
-go install ./cmd/dendrite-polylith-multi
-```
-
-Alternatively, you can specify a custom path for the binary to be written to using `go build`:
-
-```sh
-go build -o /usr/local/bin/ ./cmd/dendrite-polylith-multi
-```
-
-The `dendrite-polylith-multi` binary is a "multi-personality" binary which can run as
-any of the components depending on the supplied command line parameters.
-
-## Reverse proxy
-
-Polylith deployments require a reverse proxy in order to ensure that requests are
-sent to the correct endpoint. You must ensure that a suitable reverse proxy is installed
-and configured.
-
-Sample configurations are provided
-for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy/polylith/Caddyfile)
-and [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx/polylith-sample.conf).

docs/installation/7_configuration.md

@@ -7,11 +7,10 @@ permalink: /installation/configuration
 
 # Configuring Dendrite
 
-A YAML configuration file is used to configure Dendrite. Sample configuration files are
+A YAML configuration file is used to configure Dendrite. A sample configuration file is
 present in the top level of the Dendrite repository:
 
 * [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml)
-* [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml)
 
 You will need to duplicate the sample, calling it `dendrite.yaml` for example, and then
 tailor it to your installation. At a minimum, you will need to populate the following
@@ -46,10 +45,9 @@ global:
 ## JetStream configuration
 
 Monolith deployments can use the built-in NATS Server rather than running a standalone
-server. If you are building a polylith deployment, or you want to use a standalone NATS
-Server anyway, you can also configure that too.
+server. If you want to use a standalone NATS Server anyway, you can also configure that too.
 
-### Built-in NATS Server (monolith only)
+### Built-in NATS Server
 
 In the `global` section, under the `jetstream` key, ensure that no server addresses are
 configured and set a `storage_path` to a persistent folder on the filesystem:
@@ -63,7 +61,7 @@ global:
     topic_prefix: Dendrite
 ```
 
-### Standalone NATS Server (monolith and polylith)
+### Standalone NATS Server
 
 To use a standalone NATS Server instance, you will need to configure `addresses` field
 to point to the port that your NATS Server is listening on:
@@ -86,7 +84,7 @@ one address in the `addresses` field.
 Configuring database connections varies based on the [database configuration](database)
 that you chose.
 
-### Global connection pool (monolith with a single PostgreSQL database only)
+### Global connection pool
 
 If you are running a monolith deployment and want to use a single connection pool to a
 single PostgreSQL database, then you must uncomment and configure the `database` section
@@ -109,7 +107,7 @@ override the `global` database configuration.
 
 ### Per-component connections (all other configurations)
 
-If you are building a polylith deployment, are using SQLite databases or separate PostgreSQL
+If you are are using SQLite databases or separate PostgreSQL
 databases per component, then you must instead configure the `database` sections under each
 of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`,
 `media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks.

federationapi/api/api.go

@@ -18,6 +18,7 @@ type FederationInternalAPI interface {
 	gomatrixserverlib.KeyDatabase
 	ClientFederationAPI
 	RoomserverFederationAPI
+	P2PFederationAPI
 
 	QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
 	LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
@@ -30,7 +31,6 @@ type FederationInternalAPI interface {
 		request *PerformBroadcastEDURequest,
 		response *PerformBroadcastEDUResponse,
 	) error
-
 	PerformWakeupServers(
 		ctx context.Context,
 		request *PerformWakeupServersRequest,
@@ -71,6 +71,29 @@ type RoomserverFederationAPI interface {
 	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
+type P2PFederationAPI interface {
+	// Get the relay servers associated for the given server.
+	P2PQueryRelayServers(
+		ctx context.Context,
+		request *P2PQueryRelayServersRequest,
+		response *P2PQueryRelayServersResponse,
+	) error
+
+	// Add relay server associations to the given server.
+	P2PAddRelayServers(
+		ctx context.Context,
+		request *P2PAddRelayServersRequest,
+		response *P2PAddRelayServersResponse,
+	) error
+
+	// Remove relay server associations from the given server.
+	P2PRemoveRelayServers(
+		ctx context.Context,
+		request *P2PRemoveRelayServersRequest,
+		response *P2PRemoveRelayServersResponse,
+	) error
+}
+
 // KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver
 // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
 // this interface are of type FederationClientError
@@ -82,6 +105,7 @@ type KeyserverFederationAPI interface {
 
 // an interface for gmsl.FederationClient - contains functions called by federationapi only.
 type FederationClient interface {
+	P2PFederationClient
 	gomatrixserverlib.KeyClient
 	SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
 
@@ -110,6 +134,11 @@ type FederationClient interface {
 	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
 }
 
+type P2PFederationClient interface {
+	P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error)
+	P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev gomatrixserverlib.RelayEntry, relayServer gomatrixserverlib.ServerName) (res gomatrixserverlib.RespGetRelayTransaction, err error)
+}
+
 // FederationClientError is returned from FederationClient methods in the event of a problem.
 type FederationClientError struct {
 	Err         string
@@ -233,3 +262,27 @@ type InputPublicKeysRequest struct {
 
 type InputPublicKeysResponse struct {
 }
+
+type P2PQueryRelayServersRequest struct {
+	Server gomatrixserverlib.ServerName
+}
+
+type P2PQueryRelayServersResponse struct {
+	RelayServers []gomatrixserverlib.ServerName
+}
+
+type P2PAddRelayServersRequest struct {
+	Server       gomatrixserverlib.ServerName
+	RelayServers []gomatrixserverlib.ServerName
+}
+
+type P2PAddRelayServersResponse struct {
+}
+
+type P2PRemoveRelayServersRequest struct {
+	Server       gomatrixserverlib.ServerName
+	RelayServers []gomatrixserverlib.ServerName
+}
+
+type P2PRemoveRelayServersResponse struct {
+}

federationapi/consumers/keychange.go

@@ -26,11 +26,11 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/types"
-	"github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/userapi/api"
 )
 
 // KeyChangeConsumer consumes events that originate in key server.

federationapi/consumers/roomserver.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/nats-io/nats.go"
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/federationapi/queue"
@@ -90,8 +91,10 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
 	msg := msgs[0] // Guaranteed to exist if onMessage is called
 	receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
 
-	// Only handle events we care about
-	if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek {
+	// Only handle events we care about, avoids unneeded unmarshalling
+	switch receivedType {
+	case api.OutputTypeNewRoomEvent, api.OutputTypeNewInboundPeek, api.OutputTypePurgeRoom:
+	default:
 		return true
 	}
 
@@ -126,6 +129,14 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
 			return false
 		}
 
+	case api.OutputTypePurgeRoom:
+		log.WithField("room_id", output.PurgeRoom.RoomID).Warn("Purging room from federation API")
+		if err := s.db.PurgeRoom(ctx, output.PurgeRoom.RoomID); err != nil {
+			logrus.WithField("room_id", output.PurgeRoom.RoomID).WithError(err).Error("Failed to purge room from federation API")
+		} else {
+			logrus.WithField("room_id", output.PurgeRoom.RoomID).Warn("Room purged from federation API")
+		}
+
 	default:
 		log.WithField("type", output.Type).Debug(
 			"roomserver output log: ignoring unknown output type",
@@ -195,7 +206,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 	}
 
 	// If we added new hosts, inform them about our known presence events for this room
-	if len(addsJoinedHosts) > 0 && ore.Event.Type() == gomatrixserverlib.MRoomMember && ore.Event.StateKey() != nil {
+	if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == gomatrixserverlib.MRoomMember && ore.Event.StateKey() != nil {
 		membership, _ := ore.Event.Membership()
 		if membership == gomatrixserverlib.Join {
 			s.sendPresence(ore.Event.RoomID(), addsJoinedHosts)

federationapi/federationapi.go

@@ -17,20 +17,17 @@ package federationapi
 import (
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
 	"github.com/matrix-org/dendrite/federationapi/internal"
-	"github.com/matrix-org/dendrite/federationapi/inthttp"
 	"github.com/matrix-org/dendrite/federationapi/producers"
 	"github.com/matrix-org/dendrite/federationapi/queue"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/internal/caching"
-	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -41,21 +38,14 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/routing"
 )
 
-// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
-// on the given input API.
-func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI, enableMetrics bool) {
-	inthttp.AddRoutes(intAPI, router, enableMetrics)
-}
-
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
 func AddPublicRoutes(
 	base *base.BaseDendrite,
-	userAPI userapi.UserInternalAPI,
+	userAPI userapi.FederationUserAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fedAPI federationAPI.FederationInternalAPI,
-	keyAPI keyserverAPI.FederationKeyAPI,
 	servers federationAPI.ServersInRoomProvider,
 ) {
 	cfg := &base.Cfg.FederationAPI
@@ -85,12 +75,9 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		base.PublicFederationAPIMux,
-		base.PublicKeyAPIMux,
-		base.PublicWellKnownAPIMux,
-		cfg,
+		base,
 		rsAPI, f, keyRing,
-		federation, userAPI, keyAPI, mscCfg,
+		federation, userAPI, mscCfg,
 		servers, producer,
 	)
 }
@@ -116,7 +103,10 @@ func NewInternalAPI(
 		_ = federationDB.RemoveAllServersFromBlacklist()
 	}
 
-	stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1)
+	stats := statistics.NewStatistics(
+		federationDB,
+		cfg.FederationMaxRetries+1,
+		cfg.P2PFederationRetriesUntilAssumedOffline+1)
 
 	js, nats := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 

federationapi/federationapi_keys_test.go

@@ -77,8 +77,8 @@ func TestMain(m *testing.M) {
 			// API to work.
 			cfg := &config.Dendrite{}
 			cfg.Defaults(config.DefaultOpts{
-				Generate:   true,
-				Monolithic: true,
+				Generate:       true,
+				SingleDatabase: false,
 			})
 			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
 			cfg.Global.PrivateKey = testPriv
@@ -109,7 +109,7 @@ func TestMain(m *testing.M) {
 			)
 
 			// Finally, build the server key APIs.
-			sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
+			sbase := base.NewBaseDendrite(cfg, base.DisableMetrics)
 			s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
 		}
 

federationapi/federationapi_test.go

@@ -17,13 +17,13 @@ import (
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/internal"
-	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
 type fedRoomserverAPI struct {
@@ -230,9 +230,9 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 	// Inject a keyserver key change event and ensure we try to send it out. If we don't, then the
 	// federationapi is incorrectly waiting for an output room event to arrive to update the joined
 	// hosts table.
-	key := keyapi.DeviceMessage{
-		Type: keyapi.TypeDeviceKeyUpdate,
-		DeviceKeys: &keyapi.DeviceKeys{
+	key := userapi.DeviceMessage{
+		Type: userapi.TypeDeviceKeyUpdate,
+		DeviceKeys: &userapi.DeviceKeys{
 			UserID:      joiningUser.ID,
 			DeviceID:    "MY_DEVICE",
 			DisplayName: "BLARGLE",
@@ -266,19 +266,19 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	_, privKey, _ := ed25519.GenerateKey(nil)
 	cfg := &config.Dendrite{}
 	cfg.Defaults(config.DefaultOpts{
-		Generate:   true,
-		Monolithic: true,
+		Generate:       true,
+		SingleDatabase: false,
 	})
 	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
 	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
 	cfg.Global.PrivateKey = privKey
 	cfg.Global.JetStream.InMemory = true
-	base := base.NewBaseDendrite(cfg, "Monolith")
+	b := base.NewBaseDendrite(cfg, base.DisableMetrics)
 	keyRing := &test.NopJSONVerifier{}
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, nil)
-	baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true)
+	federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil)
+	baseURL, cancel := test.ListenAndServe(t, b.PublicFederationAPIMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
 

federationapi/internal/api.go

@@ -109,13 +109,14 @@ func NewFederationInternalAPI(
 
 func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s gomatrixserverlib.ServerName) (*statistics.ServerStatistics, error) {
 	stats := a.statistics.ForServer(s)
-	until, blacklisted := stats.BackoffInfo()
-	if blacklisted {
+	if stats.Blacklisted() {
 		return stats, &api.FederationClientError{
 			Blacklisted: true,
 		}
 	}
+
 	now := time.Now()
+	until := stats.BackoffInfo()
 	if until != nil && now.Before(*until) {
 		return stats, &api.FederationClientError{
 			RetryAfter: time.Until(*until),
@@ -163,7 +164,7 @@ func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
 			RetryAfter:  retryAfter,
 		}
 	}
-	stats.Success()
+	stats.Success(statistics.SendDirect)
 	return res, nil
 }
 
@@ -171,7 +172,7 @@ func (a *FederationInternalAPI) doRequestIfNotBlacklisted(
 	s gomatrixserverlib.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
 	stats := a.statistics.ForServer(s)
-	if _, blacklisted := stats.BackoffInfo(); blacklisted {
+	if blacklisted := stats.Blacklisted(); blacklisted {
 		return stats, &api.FederationClientError{
 			Err:         fmt.Sprintf("server %q is blacklisted", s),
 			Blacklisted: true,

federationapi/internal/federationclient_test.go

@@ -0,0 +1,202 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	FailuresUntilAssumedOffline = 3
+	FailuresUntilBlacklist      = 8
+)
+
+func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) {
+	t.queryKeysCalled = true
+	if t.shouldFail {
+		return gomatrixserverlib.RespQueryKeys{}, fmt.Errorf("Failure")
+	}
+	return gomatrixserverlib.RespQueryKeys{}, nil
+}
+
+func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) {
+	t.claimKeysCalled = true
+	if t.shouldFail {
+		return gomatrixserverlib.RespClaimKeys{}, fmt.Errorf("Failure")
+	}
+	return gomatrixserverlib.RespClaimKeys{}, nil
+}
+
+func TestFederationClientQueryKeys(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.Nil(t, err)
+	assert.True(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+	testDB.AddServerToBlacklist("server")
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.False(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientQueryKeysFailure(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{shouldFail: true}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.True(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientClaimKeys(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.ClaimKeys(context.Background(), "origin", "server", nil)
+	assert.Nil(t, err)
+	assert.True(t, fedClient.claimKeysCalled)
+}
+
+func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+	testDB.AddServerToBlacklist("server")
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.ClaimKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.False(t, fedClient.claimKeysCalled)
+}

federationapi/internal/perform.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 )
@@ -24,6 +25,10 @@ func (r *FederationInternalAPI) PerformDirectoryLookup(
 	request *api.PerformDirectoryLookupRequest,
 	response *api.PerformDirectoryLookupResponse,
 ) (err error) {
+	if !r.shouldAttemptDirectFederation(request.ServerName) {
+		return fmt.Errorf("relay servers have no meaningful response for directory lookup.")
+	}
+
 	dir, err := r.federation.LookupRoomAlias(
 		ctx,
 		r.cfg.Matrix.ServerName,
@@ -36,7 +41,7 @@ func (r *FederationInternalAPI) PerformDirectoryLookup(
 	}
 	response.RoomID = dir.RoomID
 	response.ServerNames = dir.Servers
-	r.statistics.ForServer(request.ServerName).Success()
+	r.statistics.ForServer(request.ServerName).Success(statistics.SendDirect)
 	return nil
 }
 
@@ -116,8 +121,6 @@ func (r *FederationInternalAPI) PerformJoin(
 	var httpErr gomatrix.HTTPError
 	if ok := errors.As(lastErr, &httpErr); ok {
 		httpErr.Message = string(httpErr.Contents)
-		// Clear the wrapped error, else serialising to JSON (in polylith mode) will fail
-		httpErr.WrappedError = nil
 		response.LastError = &httpErr
 	} else {
 		response.LastError = &gomatrix.HTTPError{
@@ -144,6 +147,10 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	supportedVersions []gomatrixserverlib.RoomVersion,
 	unsigned map[string]interface{},
 ) error {
+	if !r.shouldAttemptDirectFederation(serverName) {
+		return fmt.Errorf("relay servers have no meaningful response for join.")
+	}
+
 	_, origin, err := r.cfg.Matrix.SplitLocalID('@', userID)
 	if err != nil {
 		return err
@@ -164,7 +171,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.MakeJoin: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// Set all the fields to be what they should be, this should be a no-op
 	// but it's possible that the remote server returned us something "odd"
@@ -219,7 +226,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.SendJoin: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// If the remote server returned an event in the "event" key of
 	// the send_join request then we should use that instead. It may
@@ -382,8 +389,6 @@ func (r *FederationInternalAPI) PerformOutboundPeek(
 	var httpErr gomatrix.HTTPError
 	if ok := errors.As(lastErr, &httpErr); ok {
 		httpErr.Message = string(httpErr.Contents)
-		// Clear the wrapped error, else serialising to JSON (in polylith mode) will fail
-		httpErr.WrappedError = nil
 		response.LastError = &httpErr
 	} else {
 		response.LastError = &gomatrix.HTTPError{
@@ -407,6 +412,10 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 	serverName gomatrixserverlib.ServerName,
 	supportedVersions []gomatrixserverlib.RoomVersion,
 ) error {
+	if !r.shouldAttemptDirectFederation(serverName) {
+		return fmt.Errorf("relay servers have no meaningful response for outbound peek.")
+	}
+
 	// create a unique ID for this peek.
 	// for now we just use the room ID again. In future, if we ever
 	// support concurrent peeks to the same room with different filters
@@ -446,7 +455,7 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.Peek: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// Work out if we support the room version that has been supplied in
 	// the peek response.
@@ -516,6 +525,10 @@ func (r *FederationInternalAPI) PerformLeave(
 	// Try each server that we were provided until we land on one that
 	// successfully completes the make-leave send-leave dance.
 	for _, serverName := range request.ServerNames {
+		if !r.shouldAttemptDirectFederation(serverName) {
+			continue
+		}
+
 		// Try to perform a make_leave using the information supplied in the
 		// request.
 		respMakeLeave, err := r.federation.MakeLeave(
@@ -585,7 +598,7 @@ func (r *FederationInternalAPI) PerformLeave(
 			continue
 		}
 
-		r.statistics.ForServer(serverName).Success()
+		r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 		return nil
 	}
 
@@ -616,6 +629,12 @@ func (r *FederationInternalAPI) PerformInvite(
 		return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 	}
 
+	// TODO (devon): This should be allowed via a relay. Currently only transactions
+	// can be sent to relays. Would need to extend relays to handle invites.
+	if !r.shouldAttemptDirectFederation(destination) {
+		return fmt.Errorf("relay servers have no meaningful response for invite.")
+	}
+
 	logrus.WithFields(logrus.Fields{
 		"event_id":     request.Event.EventID(),
 		"user_id":      *request.Event.StateKey(),
@@ -682,12 +701,8 @@ func (r *FederationInternalAPI) PerformWakeupServers(
 
 func (r *FederationInternalAPI) MarkServersAlive(destinations []gomatrixserverlib.ServerName) {
 	for _, srv := range destinations {
-		// Check the statistics cache for the blacklist status to prevent hitting
-		// the database unnecessarily.
-		if r.queues.IsServerBlacklisted(srv) {
-			_ = r.db.RemoveServerFromBlacklist(srv)
-		}
-		r.queues.RetryServer(srv)
+		wasBlacklisted := r.statistics.ForServer(srv).MarkServerAlive()
+		r.queues.RetryServer(srv, wasBlacklisted)
 	}
 }
 
@@ -719,7 +734,9 @@ func sanityCheckAuthChain(authChain []*gomatrixserverlib.Event) error {
 	return fmt.Errorf("auth chain response is missing m.room.create event")
 }
 
-func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder) gomatrixserverlib.RoomVersion {
+func setDefaultRoomVersionFromJoinEvent(
+	joinEvent gomatrixserverlib.EventBuilder,
+) gomatrixserverlib.RoomVersion {
 	// if auth events are not event references we know it must be v3+
 	// we have to do these shenanigans to satisfy sytest, specifically for:
 	// "Outbound federation rejects m.room.create events with an unknown room version"
@@ -802,3 +819,61 @@ func federatedAuthProvider(
 		return returning, nil
 	}
 }
+
+// P2PQueryRelayServers implements api.FederationInternalAPI
+func (r *FederationInternalAPI) P2PQueryRelayServers(
+	ctx context.Context,
+	request *api.P2PQueryRelayServersRequest,
+	response *api.P2PQueryRelayServersResponse,
+) error {
+	logrus.Infof("Getting relay servers for: %s", request.Server)
+	relayServers, err := r.db.P2PGetRelayServersForServer(ctx, request.Server)
+	if err != nil {
+		return err
+	}
+
+	response.RelayServers = relayServers
+	return nil
+}
+
+// P2PAddRelayServers implements api.FederationInternalAPI
+func (r *FederationInternalAPI) P2PAddRelayServers(
+	ctx context.Context,
+	request *api.P2PAddRelayServersRequest,
+	response *api.P2PAddRelayServersResponse,
+) error {
+	logrus.Infof("Adding relay servers for: %s", request.Server)
+	err := r.db.P2PAddRelayServersForServer(ctx, request.Server, request.RelayServers)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// P2PRemoveRelayServers implements api.FederationInternalAPI
+func (r *FederationInternalAPI) P2PRemoveRelayServers(
+	ctx context.Context,
+	request *api.P2PRemoveRelayServersRequest,
+	response *api.P2PRemoveRelayServersResponse,
+) error {
+	logrus.Infof("Adding relay servers for: %s", request.Server)
+	err := r.db.P2PRemoveRelayServersForServer(ctx, request.Server, request.RelayServers)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *FederationInternalAPI) shouldAttemptDirectFederation(
+	destination gomatrixserverlib.ServerName,
+) bool {
+	var shouldRelay bool
+	stats := r.statistics.ForServer(destination)
+	if stats.AssumedOffline() && len(stats.KnownRelayServers()) > 0 {
+		shouldRelay = true
+	}
+
+	return !shouldRelay
+}

federationapi/internal/perform_test.go

@@ -0,0 +1,231 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+type testFedClient struct {
+	api.FederationClient
+	queryKeysCalled bool
+	claimKeysCalled bool
+	shouldFail      bool
+}
+
+func (t *testFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) {
+	return gomatrixserverlib.RespDirectory{}, nil
+}
+
+func TestPerformWakeupServers(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	testDB.AddServerToBlacklist(server)
+	testDB.SetServerAssumedOffline(context.Background(), server)
+	blacklisted, err := testDB.IsServerBlacklisted(server)
+	assert.NoError(t, err)
+	assert.True(t, blacklisted)
+	offline, err := testDB.IsServerAssumedOffline(context.Background(), server)
+	assert.NoError(t, err)
+	assert.True(t, offline)
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformWakeupServersRequest{
+		ServerNames: []gomatrixserverlib.ServerName{server},
+	}
+	res := api.PerformWakeupServersResponse{}
+	err = fedAPI.PerformWakeupServers(context.Background(), &req, &res)
+	assert.NoError(t, err)
+
+	blacklisted, err = testDB.IsServerBlacklisted(server)
+	assert.NoError(t, err)
+	assert.False(t, blacklisted)
+	offline, err = testDB.IsServerAssumedOffline(context.Background(), server)
+	assert.NoError(t, err)
+	assert.False(t, offline)
+}
+
+func TestQueryRelayServers(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	relayServers := []gomatrixserverlib.ServerName{"relay1", "relay2"}
+	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
+	assert.NoError(t, err)
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.P2PQueryRelayServersRequest{
+		Server: server,
+	}
+	res := api.P2PQueryRelayServersResponse{}
+	err = fedAPI.P2PQueryRelayServers(context.Background(), &req, &res)
+	assert.NoError(t, err)
+
+	assert.Equal(t, len(relayServers), len(res.RelayServers))
+}
+
+func TestRemoveRelayServers(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	relayServers := []gomatrixserverlib.ServerName{"relay1", "relay2"}
+	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
+	assert.NoError(t, err)
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.P2PRemoveRelayServersRequest{
+		Server:       server,
+		RelayServers: []gomatrixserverlib.ServerName{"relay1"},
+	}
+	res := api.P2PRemoveRelayServersResponse{}
+	err = fedAPI.P2PRemoveRelayServers(context.Background(), &req, &res)
+	assert.NoError(t, err)
+
+	finalRelays, err := testDB.P2PGetRelayServersForServer(context.Background(), server)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(finalRelays))
+	assert.Equal(t, gomatrixserverlib.ServerName("relay2"), finalRelays[0])
+}
+
+func TestPerformDirectoryLookup(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformDirectoryLookupRequest{
+		RoomAlias:  "room",
+		ServerName: "server",
+	}
+	res := api.PerformDirectoryLookupResponse{}
+	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	assert.NoError(t, err)
+}
+
+func TestPerformDirectoryLookupRelaying(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	testDB.SetServerAssumedOffline(context.Background(), server)
+	testDB.P2PAddRelayServersForServer(context.Background(), server, []gomatrixserverlib.ServerName{"relay"})
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: server,
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformDirectoryLookupRequest{
+		RoomAlias:  "room",
+		ServerName: server,
+	}
+	res := api.PerformDirectoryLookupResponse{}
+	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	assert.Error(t, err)
+}

federationapi/inthttp/client.go

@@ -1,512 +0,0 @@
-package inthttp
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/internal/caching"
-	"github.com/matrix-org/dendrite/internal/httputil"
-	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// HTTP paths for the internal HTTP API
-const (
-	FederationAPIQueryJoinedHostServerNamesInRoomPath = "/federationapi/queryJoinedHostServerNamesInRoom"
-	FederationAPIQueryServerKeysPath                  = "/federationapi/queryServerKeys"
-
-	FederationAPIPerformDirectoryLookupRequestPath = "/federationapi/performDirectoryLookup"
-	FederationAPIPerformJoinRequestPath            = "/federationapi/performJoinRequest"
-	FederationAPIPerformLeaveRequestPath           = "/federationapi/performLeaveRequest"
-	FederationAPIPerformInviteRequestPath          = "/federationapi/performInviteRequest"
-	FederationAPIPerformOutboundPeekRequestPath    = "/federationapi/performOutboundPeekRequest"
-	FederationAPIPerformBroadcastEDUPath           = "/federationapi/performBroadcastEDU"
-	FederationAPIPerformWakeupServers              = "/federationapi/performWakeupServers"
-
-	FederationAPIGetUserDevicesPath      = "/federationapi/client/getUserDevices"
-	FederationAPIClaimKeysPath           = "/federationapi/client/claimKeys"
-	FederationAPIQueryKeysPath           = "/federationapi/client/queryKeys"
-	FederationAPIBackfillPath            = "/federationapi/client/backfill"
-	FederationAPILookupStatePath         = "/federationapi/client/lookupState"
-	FederationAPILookupStateIDsPath      = "/federationapi/client/lookupStateIDs"
-	FederationAPILookupMissingEventsPath = "/federationapi/client/lookupMissingEvents"
-	FederationAPIGetEventPath            = "/federationapi/client/getEvent"
-	FederationAPILookupServerKeysPath    = "/federationapi/client/lookupServerKeys"
-	FederationAPIEventRelationshipsPath  = "/federationapi/client/msc2836eventRelationships"
-	FederationAPISpacesSummaryPath       = "/federationapi/client/msc2946spacesSummary"
-	FederationAPIGetEventAuthPath        = "/federationapi/client/getEventAuth"
-
-	FederationAPIInputPublicKeyPath = "/federationapi/inputPublicKey"
-	FederationAPIQueryPublicKeyPath = "/federationapi/queryPublicKey"
-)
-
-// NewFederationAPIClient creates a FederationInternalAPI implemented by talking to a HTTP POST API.
-// If httpClient is nil an error is returned
-func NewFederationAPIClient(federationSenderURL string, httpClient *http.Client, cache caching.ServerKeyCache) (api.FederationInternalAPI, error) {
-	if httpClient == nil {
-		return nil, errors.New("NewFederationInternalAPIHTTP: httpClient is <nil>")
-	}
-	return &httpFederationInternalAPI{
-		federationAPIURL: federationSenderURL,
-		httpClient:       httpClient,
-		cache:            cache,
-	}, nil
-}
-
-type httpFederationInternalAPI struct {
-	federationAPIURL string
-	httpClient       *http.Client
-	cache            caching.ServerKeyCache
-}
-
-// Handle an instruction to make_leave & send_leave with a remote server.
-func (h *httpFederationInternalAPI) PerformLeave(
-	ctx context.Context,
-	request *api.PerformLeaveRequest,
-	response *api.PerformLeaveResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"PerformLeave", h.federationAPIURL+FederationAPIPerformLeaveRequestPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// Handle sending an invite to a remote server.
-func (h *httpFederationInternalAPI) PerformInvite(
-	ctx context.Context,
-	request *api.PerformInviteRequest,
-	response *api.PerformInviteResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"PerformInvite", h.federationAPIURL+FederationAPIPerformInviteRequestPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// Handle starting a peek on a remote server.
-func (h *httpFederationInternalAPI) PerformOutboundPeek(
-	ctx context.Context,
-	request *api.PerformOutboundPeekRequest,
-	response *api.PerformOutboundPeekResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"PerformOutboundPeek", h.federationAPIURL+FederationAPIPerformOutboundPeekRequestPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// QueryJoinedHostServerNamesInRoom implements FederationInternalAPI
-func (h *httpFederationInternalAPI) QueryJoinedHostServerNamesInRoom(
-	ctx context.Context,
-	request *api.QueryJoinedHostServerNamesInRoomRequest,
-	response *api.QueryJoinedHostServerNamesInRoomResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"QueryJoinedHostServerNamesInRoom", h.federationAPIURL+FederationAPIQueryJoinedHostServerNamesInRoomPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// Handle an instruction to make_join & send_join with a remote server.
-func (h *httpFederationInternalAPI) PerformJoin(
-	ctx context.Context,
-	request *api.PerformJoinRequest,
-	response *api.PerformJoinResponse,
-) {
-	if err := httputil.CallInternalRPCAPI(
-		"PerformJoinRequest", h.federationAPIURL+FederationAPIPerformJoinRequestPath,
-		h.httpClient, ctx, request, response,
-	); err != nil {
-		response.LastError = &gomatrix.HTTPError{
-			Message:      err.Error(),
-			Code:         0,
-			WrappedError: err,
-		}
-	}
-}
-
-// Handle an instruction to make_join & send_join with a remote server.
-func (h *httpFederationInternalAPI) PerformDirectoryLookup(
-	ctx context.Context,
-	request *api.PerformDirectoryLookupRequest,
-	response *api.PerformDirectoryLookupResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"PerformDirectoryLookup", h.federationAPIURL+FederationAPIPerformDirectoryLookupRequestPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// Handle an instruction to broadcast an EDU to all servers in rooms we are joined to.
-func (h *httpFederationInternalAPI) PerformBroadcastEDU(
-	ctx context.Context,
-	request *api.PerformBroadcastEDURequest,
-	response *api.PerformBroadcastEDUResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"PerformBroadcastEDU", h.federationAPIURL+FederationAPIPerformBroadcastEDUPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-// Handle an instruction to remove the respective servers from being blacklisted.
-func (h *httpFederationInternalAPI) PerformWakeupServers(
-	ctx context.Context,
-	request *api.PerformWakeupServersRequest,
-	response *api.PerformWakeupServersResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"PerformWakeupServers", h.federationAPIURL+FederationAPIPerformWakeupServers,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-type getUserDevices struct {
-	S      gomatrixserverlib.ServerName
-	Origin gomatrixserverlib.ServerName
-	UserID string
-}
-
-func (h *httpFederationInternalAPI) GetUserDevices(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string,
-) (gomatrixserverlib.RespUserDevices, error) {
-	return httputil.CallInternalProxyAPI[getUserDevices, gomatrixserverlib.RespUserDevices, *api.FederationClientError](
-		"GetUserDevices", h.federationAPIURL+FederationAPIGetUserDevicesPath, h.httpClient,
-		ctx, &getUserDevices{
-			S:      s,
-			Origin: origin,
-			UserID: userID,
-		},
-	)
-}
-
-type claimKeys struct {
-	S           gomatrixserverlib.ServerName
-	Origin      gomatrixserverlib.ServerName
-	OneTimeKeys map[string]map[string]string
-}
-
-func (h *httpFederationInternalAPI) ClaimKeys(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
-) (gomatrixserverlib.RespClaimKeys, error) {
-	return httputil.CallInternalProxyAPI[claimKeys, gomatrixserverlib.RespClaimKeys, *api.FederationClientError](
-		"ClaimKeys", h.federationAPIURL+FederationAPIClaimKeysPath, h.httpClient,
-		ctx, &claimKeys{
-			S:           s,
-			Origin:      origin,
-			OneTimeKeys: oneTimeKeys,
-		},
-	)
-}
-
-type queryKeys struct {
-	S      gomatrixserverlib.ServerName
-	Origin gomatrixserverlib.ServerName
-	Keys   map[string][]string
-}
-
-func (h *httpFederationInternalAPI) QueryKeys(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string,
-) (gomatrixserverlib.RespQueryKeys, error) {
-	return httputil.CallInternalProxyAPI[queryKeys, gomatrixserverlib.RespQueryKeys, *api.FederationClientError](
-		"QueryKeys", h.federationAPIURL+FederationAPIQueryKeysPath, h.httpClient,
-		ctx, &queryKeys{
-			S:      s,
-			Origin: origin,
-			Keys:   keys,
-		},
-	)
-}
-
-type backfill struct {
-	S        gomatrixserverlib.ServerName
-	Origin   gomatrixserverlib.ServerName
-	RoomID   string
-	Limit    int
-	EventIDs []string
-}
-
-func (h *httpFederationInternalAPI) Backfill(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string,
-) (gomatrixserverlib.Transaction, error) {
-	return httputil.CallInternalProxyAPI[backfill, gomatrixserverlib.Transaction, *api.FederationClientError](
-		"Backfill", h.federationAPIURL+FederationAPIBackfillPath, h.httpClient,
-		ctx, &backfill{
-			S:        s,
-			Origin:   origin,
-			RoomID:   roomID,
-			Limit:    limit,
-			EventIDs: eventIDs,
-		},
-	)
-}
-
-type lookupState struct {
-	S           gomatrixserverlib.ServerName
-	Origin      gomatrixserverlib.ServerName
-	RoomID      string
-	EventID     string
-	RoomVersion gomatrixserverlib.RoomVersion
-}
-
-func (h *httpFederationInternalAPI) LookupState(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
-) (gomatrixserverlib.RespState, error) {
-	return httputil.CallInternalProxyAPI[lookupState, gomatrixserverlib.RespState, *api.FederationClientError](
-		"LookupState", h.federationAPIURL+FederationAPILookupStatePath, h.httpClient,
-		ctx, &lookupState{
-			S:           s,
-			Origin:      origin,
-			RoomID:      roomID,
-			EventID:     eventID,
-			RoomVersion: roomVersion,
-		},
-	)
-}
-
-type lookupStateIDs struct {
-	S       gomatrixserverlib.ServerName
-	Origin  gomatrixserverlib.ServerName
-	RoomID  string
-	EventID string
-}
-
-func (h *httpFederationInternalAPI) LookupStateIDs(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string,
-) (gomatrixserverlib.RespStateIDs, error) {
-	return httputil.CallInternalProxyAPI[lookupStateIDs, gomatrixserverlib.RespStateIDs, *api.FederationClientError](
-		"LookupStateIDs", h.federationAPIURL+FederationAPILookupStateIDsPath, h.httpClient,
-		ctx, &lookupStateIDs{
-			S:       s,
-			Origin:  origin,
-			RoomID:  roomID,
-			EventID: eventID,
-		},
-	)
-}
-
-type lookupMissingEvents struct {
-	S           gomatrixserverlib.ServerName
-	Origin      gomatrixserverlib.ServerName
-	RoomID      string
-	Missing     gomatrixserverlib.MissingEvents
-	RoomVersion gomatrixserverlib.RoomVersion
-}
-
-func (h *httpFederationInternalAPI) LookupMissingEvents(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string,
-	missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.RespMissingEvents, err error) {
-	return httputil.CallInternalProxyAPI[lookupMissingEvents, gomatrixserverlib.RespMissingEvents, *api.FederationClientError](
-		"LookupMissingEvents", h.federationAPIURL+FederationAPILookupMissingEventsPath, h.httpClient,
-		ctx, &lookupMissingEvents{
-			S:           s,
-			Origin:      origin,
-			RoomID:      roomID,
-			Missing:     missing,
-			RoomVersion: roomVersion,
-		},
-	)
-}
-
-type getEvent struct {
-	S       gomatrixserverlib.ServerName
-	Origin  gomatrixserverlib.ServerName
-	EventID string
-}
-
-func (h *httpFederationInternalAPI) GetEvent(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string,
-) (gomatrixserverlib.Transaction, error) {
-	return httputil.CallInternalProxyAPI[getEvent, gomatrixserverlib.Transaction, *api.FederationClientError](
-		"GetEvent", h.federationAPIURL+FederationAPIGetEventPath, h.httpClient,
-		ctx, &getEvent{
-			S:       s,
-			Origin:  origin,
-			EventID: eventID,
-		},
-	)
-}
-
-type getEventAuth struct {
-	S           gomatrixserverlib.ServerName
-	Origin      gomatrixserverlib.ServerName
-	RoomVersion gomatrixserverlib.RoomVersion
-	RoomID      string
-	EventID     string
-}
-
-func (h *httpFederationInternalAPI) GetEventAuth(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName,
-	roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string,
-) (gomatrixserverlib.RespEventAuth, error) {
-	return httputil.CallInternalProxyAPI[getEventAuth, gomatrixserverlib.RespEventAuth, *api.FederationClientError](
-		"GetEventAuth", h.federationAPIURL+FederationAPIGetEventAuthPath, h.httpClient,
-		ctx, &getEventAuth{
-			S:           s,
-			Origin:      origin,
-			RoomVersion: roomVersion,
-			RoomID:      roomID,
-			EventID:     eventID,
-		},
-	)
-}
-
-func (h *httpFederationInternalAPI) QueryServerKeys(
-	ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"QueryServerKeys", h.federationAPIURL+FederationAPIQueryServerKeysPath,
-		h.httpClient, ctx, req, res,
-	)
-}
-
-type lookupServerKeys struct {
-	S           gomatrixserverlib.ServerName
-	KeyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp
-}
-
-func (h *httpFederationInternalAPI) LookupServerKeys(
-	ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
-) ([]gomatrixserverlib.ServerKeys, error) {
-	return httputil.CallInternalProxyAPI[lookupServerKeys, []gomatrixserverlib.ServerKeys, *api.FederationClientError](
-		"LookupServerKeys", h.federationAPIURL+FederationAPILookupServerKeysPath, h.httpClient,
-		ctx, &lookupServerKeys{
-			S:           s,
-			KeyRequests: keyRequests,
-		},
-	)
-}
-
-type eventRelationships struct {
-	S       gomatrixserverlib.ServerName
-	Origin  gomatrixserverlib.ServerName
-	Req     gomatrixserverlib.MSC2836EventRelationshipsRequest
-	RoomVer gomatrixserverlib.RoomVersion
-}
-
-func (h *httpFederationInternalAPI) MSC2836EventRelationships(
-	ctx context.Context, origin, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest,
-	roomVersion gomatrixserverlib.RoomVersion,
-) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) {
-	return httputil.CallInternalProxyAPI[eventRelationships, gomatrixserverlib.MSC2836EventRelationshipsResponse, *api.FederationClientError](
-		"MSC2836EventRelationships", h.federationAPIURL+FederationAPIEventRelationshipsPath, h.httpClient,
-		ctx, &eventRelationships{
-			S:       s,
-			Origin:  origin,
-			Req:     r,
-			RoomVer: roomVersion,
-		},
-	)
-}
-
-type spacesReq struct {
-	S             gomatrixserverlib.ServerName
-	Origin        gomatrixserverlib.ServerName
-	SuggestedOnly bool
-	RoomID        string
-}
-
-func (h *httpFederationInternalAPI) MSC2946Spaces(
-	ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool,
-) (res gomatrixserverlib.MSC2946SpacesResponse, err error) {
-	return httputil.CallInternalProxyAPI[spacesReq, gomatrixserverlib.MSC2946SpacesResponse, *api.FederationClientError](
-		"MSC2836EventRelationships", h.federationAPIURL+FederationAPISpacesSummaryPath, h.httpClient,
-		ctx, &spacesReq{
-			S:             dst,
-			Origin:        origin,
-			SuggestedOnly: suggestedOnly,
-			RoomID:        roomID,
-		},
-	)
-}
-
-func (s *httpFederationInternalAPI) KeyRing() *gomatrixserverlib.KeyRing {
-	// This is a bit of a cheat - we tell gomatrixserverlib that this API is
-	// both the key database and the key fetcher. While this does have the
-	// rather unfortunate effect of preventing gomatrixserverlib from handling
-	// key fetchers directly, we can at least reimplement this behaviour on
-	// the other end of the API.
-	return &gomatrixserverlib.KeyRing{
-		KeyDatabase: s,
-		KeyFetchers: []gomatrixserverlib.KeyFetcher{},
-	}
-}
-
-func (s *httpFederationInternalAPI) FetcherName() string {
-	return "httpServerKeyInternalAPI"
-}
-
-func (s *httpFederationInternalAPI) StoreKeys(
-	_ context.Context,
-	results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult,
-) error {
-	// Run in a background context - we don't want to stop this work just
-	// because the caller gives up waiting.
-	ctx := context.Background()
-	request := api.InputPublicKeysRequest{
-		Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult),
-	}
-	response := api.InputPublicKeysResponse{}
-	for req, res := range results {
-		request.Keys[req] = res
-		s.cache.StoreServerKey(req, res)
-	}
-	return s.InputPublicKeys(ctx, &request, &response)
-}
-
-func (s *httpFederationInternalAPI) FetchKeys(
-	_ context.Context,
-	requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
-) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
-	// Run in a background context - we don't want to stop this work just
-	// because the caller gives up waiting.
-	ctx := context.Background()
-	result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
-	request := api.QueryPublicKeysRequest{
-		Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp),
-	}
-	response := api.QueryPublicKeysResponse{
-		Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult),
-	}
-	for req, ts := range requests {
-		if res, ok := s.cache.GetServerKey(req, ts); ok {
-			result[req] = res
-			continue
-		}
-		request.Requests[req] = ts
-	}
-	err := s.QueryPublicKeys(ctx, &request, &response)
-	if err != nil {
-		return nil, err
-	}
-	for req, res := range response.Results {
-		result[req] = res
-		s.cache.StoreServerKey(req, res)
-	}
-	return result, nil
-}
-
-func (h *httpFederationInternalAPI) InputPublicKeys(
-	ctx context.Context,
-	request *api.InputPublicKeysRequest,
-	response *api.InputPublicKeysResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"InputPublicKey", h.federationAPIURL+FederationAPIInputPublicKeyPath,
-		h.httpClient, ctx, request, response,
-	)
-}
-
-func (h *httpFederationInternalAPI) QueryPublicKeys(
-	ctx context.Context,
-	request *api.QueryPublicKeysRequest,
-	response *api.QueryPublicKeysResponse,
-) error {
-	return httputil.CallInternalRPCAPI(
-		"QueryPublicKeys", h.federationAPIURL+FederationAPIQueryPublicKeyPath,
-		h.httpClient, ctx, request, response,
-	)
-}

federationapi/inthttp/server.go

@@ -1,257 +0,0 @@
-package inthttp
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/url"
-
-	"github.com/gorilla/mux"
-	"github.com/matrix-org/gomatrix"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/util"
-
-	"github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/internal/httputil"
-)
-
-// AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux.
-// nolint:gocyclo
-func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
-	internalAPIMux.Handle(
-		FederationAPIQueryJoinedHostServerNamesInRoomPath,
-		httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", enableMetrics, intAPI.QueryJoinedHostServerNamesInRoom),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIPerformInviteRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", enableMetrics, intAPI.PerformInvite),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIPerformLeaveRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", enableMetrics, intAPI.PerformLeave),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIPerformDirectoryLookupRequestPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", enableMetrics, intAPI.PerformDirectoryLookup),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIPerformBroadcastEDUPath,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", enableMetrics, intAPI.PerformBroadcastEDU),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIPerformWakeupServers,
-		httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", enableMetrics, intAPI.PerformWakeupServers),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIPerformJoinRequestPath,
-		httputil.MakeInternalRPCAPI(
-			"FederationAPIPerformJoinRequest", enableMetrics,
-			func(ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse) error {
-				intAPI.PerformJoin(ctx, req, res)
-				return nil
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIGetUserDevicesPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetUserDevices", enableMetrics,
-			func(ctx context.Context, req *getUserDevices) (*gomatrixserverlib.RespUserDevices, error) {
-				res, err := intAPI.GetUserDevices(ctx, req.Origin, req.S, req.UserID)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIClaimKeysPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIClaimKeys", enableMetrics,
-			func(ctx context.Context, req *claimKeys) (*gomatrixserverlib.RespClaimKeys, error) {
-				res, err := intAPI.ClaimKeys(ctx, req.Origin, req.S, req.OneTimeKeys)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIQueryKeysPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIQueryKeys", enableMetrics,
-			func(ctx context.Context, req *queryKeys) (*gomatrixserverlib.RespQueryKeys, error) {
-				res, err := intAPI.QueryKeys(ctx, req.Origin, req.S, req.Keys)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIBackfillPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIBackfill", enableMetrics,
-			func(ctx context.Context, req *backfill) (*gomatrixserverlib.Transaction, error) {
-				res, err := intAPI.Backfill(ctx, req.Origin, req.S, req.RoomID, req.Limit, req.EventIDs)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPILookupStatePath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupState", enableMetrics,
-			func(ctx context.Context, req *lookupState) (*gomatrixserverlib.RespState, error) {
-				res, err := intAPI.LookupState(ctx, req.Origin, req.S, req.RoomID, req.EventID, req.RoomVersion)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPILookupStateIDsPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupStateIDs", enableMetrics,
-			func(ctx context.Context, req *lookupStateIDs) (*gomatrixserverlib.RespStateIDs, error) {
-				res, err := intAPI.LookupStateIDs(ctx, req.Origin, req.S, req.RoomID, req.EventID)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPILookupMissingEventsPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupMissingEvents", enableMetrics,
-			func(ctx context.Context, req *lookupMissingEvents) (*gomatrixserverlib.RespMissingEvents, error) {
-				res, err := intAPI.LookupMissingEvents(ctx, req.Origin, req.S, req.RoomID, req.Missing, req.RoomVersion)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIGetEventPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetEvent", enableMetrics,
-			func(ctx context.Context, req *getEvent) (*gomatrixserverlib.Transaction, error) {
-				res, err := intAPI.GetEvent(ctx, req.Origin, req.S, req.EventID)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIGetEventAuthPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIGetEventAuth", enableMetrics,
-			func(ctx context.Context, req *getEventAuth) (*gomatrixserverlib.RespEventAuth, error) {
-				res, err := intAPI.GetEventAuth(ctx, req.Origin, req.S, req.RoomVersion, req.RoomID, req.EventID)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIQueryServerKeysPath,
-		httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", enableMetrics, intAPI.QueryServerKeys),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPILookupServerKeysPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPILookupServerKeys", enableMetrics,
-			func(ctx context.Context, req *lookupServerKeys) (*[]gomatrixserverlib.ServerKeys, error) {
-				res, err := intAPI.LookupServerKeys(ctx, req.S, req.KeyRequests)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPIEventRelationshipsPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIMSC2836EventRelationships", enableMetrics,
-			func(ctx context.Context, req *eventRelationships) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) {
-				res, err := intAPI.MSC2836EventRelationships(ctx, req.Origin, req.S, req.Req, req.RoomVer)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	internalAPIMux.Handle(
-		FederationAPISpacesSummaryPath,
-		httputil.MakeInternalProxyAPI(
-			"FederationAPIMSC2946SpacesSummary", enableMetrics,
-			func(ctx context.Context, req *spacesReq) (*gomatrixserverlib.MSC2946SpacesResponse, error) {
-				res, err := intAPI.MSC2946Spaces(ctx, req.Origin, req.S, req.RoomID, req.SuggestedOnly)
-				return &res, federationClientError(err)
-			},
-		),
-	)
-
-	// TODO: Look at this shape
-	internalAPIMux.Handle(FederationAPIQueryPublicKeyPath,
-		httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
-			request := api.QueryPublicKeysRequest{}
-			response := api.QueryPublicKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			keys, err := intAPI.FetchKeys(req.Context(), request.Requests)
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			response.Results = keys
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-
-	// TODO: Look at this shape
-	internalAPIMux.Handle(FederationAPIInputPublicKeyPath,
-		httputil.MakeInternalAPI("FederationAPIInputPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse {
-			request := api.InputPublicKeysRequest{}
-			response := api.InputPublicKeysResponse{}
-			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
-				return util.MessageResponse(http.StatusBadRequest, err.Error())
-			}
-			if err := intAPI.StoreKeys(req.Context(), request.Keys); err != nil {
-				return util.ErrorResponse(err)
-			}
-			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
-		}),
-	)
-}
-
-func federationClientError(err error) error {
-	switch ferr := err.(type) {
-	case nil:
-		return nil
-	case api.FederationClientError:
-		return &ferr
-	case *api.FederationClientError:
-		return ferr
-	case gomatrix.HTTPError:
-		return &api.FederationClientError{
-			Code: ferr.Code,
-		}
-	case *url.Error: // e.g. certificate error, unable to connect
-		return &api.FederationClientError{
-			Err:  ferr.Error(),
-			Code: 400,
-		}
-	default:
-		// We don't know what exactly failed, but we probably don't
-		// want to retry the request immediately in the device list updater
-		return &api.FederationClientError{
-			Err:  err.Error(),
-			Code: 400,
-		}
-	}
-}

federationapi/producers/syncapi.go

@@ -41,7 +41,7 @@ type SyncAPIProducer struct {
 	TopicSigningKeyUpdate  string
 	JetStream              nats.JetStreamContext
 	Config                 *config.FederationAPI
-	UserAPI                userapi.UserInternalAPI
+	UserAPI                userapi.FederationUserAPI
 }
 
 func (p *SyncAPIProducer) SendReceipt(

federationapi/queue/destinationqueue.go

@@ -29,7 +29,7 @@ import (
 	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -70,7 +70,7 @@ type destinationQueue struct {
 // Send event adds the event to the pending queue for the destination.
 // If the queue is empty then it starts a background goroutine to
 // start sending events to that destination.
-func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, receipt *shared.Receipt) {
+func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, dbReceipt *receipt.Receipt) {
 	if event == nil {
 		logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination)
 		return
@@ -84,8 +84,8 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re
 		oq.pendingMutex.Lock()
 		if len(oq.pendingPDUs) < maxPDUsInMemory {
 			oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{
-				pdu:     event,
-				receipt: receipt,
+				pdu:       event,
+				dbReceipt: dbReceipt,
 			})
 		} else {
 			oq.overflowed.Store(true)
@@ -101,7 +101,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re
 // sendEDU adds the EDU event to the pending queue for the destination.
 // If the queue is empty then it starts a background goroutine to
 // start sending events to that destination.
-func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *shared.Receipt) {
+func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, dbReceipt *receipt.Receipt) {
 	if event == nil {
 		logrus.Errorf("attempt to send nil EDU with destination %q", oq.destination)
 		return
@@ -115,8 +115,8 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
 		oq.pendingMutex.Lock()
 		if len(oq.pendingEDUs) < maxEDUsInMemory {
 			oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{
-				edu:     event,
-				receipt: receipt,
+				edu:       event,
+				dbReceipt: dbReceipt,
 			})
 		} else {
 			oq.overflowed.Store(true)
@@ -210,10 +210,10 @@ func (oq *destinationQueue) getPendingFromDatabase() {
 	gotPDUs := map[string]struct{}{}
 	gotEDUs := map[string]struct{}{}
 	for _, pdu := range oq.pendingPDUs {
-		gotPDUs[pdu.receipt.String()] = struct{}{}
+		gotPDUs[pdu.dbReceipt.String()] = struct{}{}
 	}
 	for _, edu := range oq.pendingEDUs {
-		gotEDUs[edu.receipt.String()] = struct{}{}
+		gotEDUs[edu.dbReceipt.String()] = struct{}{}
 	}
 
 	overflowed := false
@@ -371,7 +371,7 @@ func (oq *destinationQueue) backgroundSend() {
 
 		// If we have pending PDUs or EDUs then construct a transaction.
 		// Try sending the next transaction and see what happens.
-		terr := oq.nextTransaction(toSendPDUs, toSendEDUs)
+		terr, sendMethod := oq.nextTransaction(toSendPDUs, toSendEDUs)
 		if terr != nil {
 			// We failed to send the transaction. Mark it as a failure.
 			_, blacklisted := oq.statistics.Failure()
@@ -388,18 +388,19 @@ func (oq *destinationQueue) backgroundSend() {
 				return
 			}
 		} else {
-			oq.handleTransactionSuccess(pduCount, eduCount)
+			oq.handleTransactionSuccess(pduCount, eduCount, sendMethod)
 		}
 	}
 }
 
 // nextTransaction creates a new transaction from the pending event
 // queue and sends it.
-// Returns an error if the transaction wasn't sent.
+// Returns an error if the transaction wasn't sent. And whether the success
+// was to a relay server or not.
 func (oq *destinationQueue) nextTransaction(
 	pdus []*queuedPDU,
 	edus []*queuedEDU,
-) error {
+) (err error, sendMethod statistics.SendMethod) {
 	// Create the transaction.
 	t, pduReceipts, eduReceipts := oq.createTransaction(pdus, edus)
 	logrus.WithField("server_name", oq.destination).Debugf("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs))
@@ -407,7 +408,52 @@ func (oq *destinationQueue) nextTransaction(
 	// Try to send the transaction to the destination server.
 	ctx, cancel := context.WithTimeout(oq.process.Context(), time.Minute*5)
 	defer cancel()
-	_, err := oq.client.SendTransaction(ctx, t)
+
+	relayServers := oq.statistics.KnownRelayServers()
+	hasRelayServers := len(relayServers) > 0
+	shouldSendToRelays := oq.statistics.AssumedOffline() && hasRelayServers
+	if !shouldSendToRelays {
+		sendMethod = statistics.SendDirect
+		_, err = oq.client.SendTransaction(ctx, t)
+	} else {
+		// Try sending directly to the destination first in case they came back online.
+		sendMethod = statistics.SendDirect
+		_, err = oq.client.SendTransaction(ctx, t)
+		if err != nil {
+			// The destination is still offline, try sending to relays.
+			sendMethod = statistics.SendViaRelay
+			relaySuccess := false
+			logrus.Infof("Sending %q to relay servers: %v", t.TransactionID, relayServers)
+			// TODO : how to pass through actual userID here?!?!?!?!
+			userID, userErr := gomatrixserverlib.NewUserID("@user:"+string(oq.destination), false)
+			if userErr != nil {
+				return userErr, sendMethod
+			}
+
+			// Attempt sending to each known relay server.
+			for _, relayServer := range relayServers {
+				_, relayErr := oq.client.P2PSendTransactionToRelay(ctx, *userID, t, relayServer)
+				if relayErr != nil {
+					err = relayErr
+				} else {
+					// If sending to one of the relay servers succeeds, consider the send successful.
+					relaySuccess = true
+
+					// TODO : what about if the dest comes back online but can't see their relay?
+					// How do I sync with the dest in that case?
+					// Should change the database to have a "relay success" flag on events and if
+					// I see the node back online, maybe directly send through the backlog of events
+					// with "relay success"... could lead to duplicate events, but only those that
+					// I sent. And will lead to a much more consistent experience.
+				}
+			}
+
+			// Clear the error if sending to any of the relay servers succeeded.
+			if relaySuccess {
+				err = nil
+			}
+		}
+	}
 	switch errResponse := err.(type) {
 	case nil:
 		// Clean up the transaction in the database.
@@ -427,7 +473,7 @@ func (oq *destinationQueue) nextTransaction(
 		oq.transactionIDMutex.Lock()
 		oq.transactionID = ""
 		oq.transactionIDMutex.Unlock()
-		return nil
+		return nil, sendMethod
 	case gomatrix.HTTPError:
 		// Report that we failed to send the transaction and we
 		// will retry again, subject to backoff.
@@ -437,13 +483,13 @@ func (oq *destinationQueue) nextTransaction(
 		// to a 400-ish error
 		code := errResponse.Code
 		logrus.Debug("Transaction failed with HTTP", code)
-		return err
+		return err, sendMethod
 	default:
 		logrus.WithFields(logrus.Fields{
 			"destination":   oq.destination,
 			logrus.ErrorKey: err,
 		}).Debugf("Failed to send transaction %q", t.TransactionID)
-		return err
+		return err, sendMethod
 	}
 }
 
@@ -453,7 +499,7 @@ func (oq *destinationQueue) nextTransaction(
 func (oq *destinationQueue) createTransaction(
 	pdus []*queuedPDU,
 	edus []*queuedEDU,
-) (gomatrixserverlib.Transaction, []*shared.Receipt, []*shared.Receipt) {
+) (gomatrixserverlib.Transaction, []*receipt.Receipt, []*receipt.Receipt) {
 	// If there's no projected transaction ID then generate one. If
 	// the transaction succeeds then we'll set it back to "" so that
 	// we generate a new one next time. If it fails, we'll preserve
@@ -474,8 +520,8 @@ func (oq *destinationQueue) createTransaction(
 	t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now())
 	t.TransactionID = oq.transactionID
 
-	var pduReceipts []*shared.Receipt
-	var eduReceipts []*shared.Receipt
+	var pduReceipts []*receipt.Receipt
+	var eduReceipts []*receipt.Receipt
 
 	// Go through PDUs that we retrieved from the database, if any,
 	// and add them into the transaction.
@@ -487,7 +533,7 @@ func (oq *destinationQueue) createTransaction(
 		// Append the JSON of the event, since this is a json.RawMessage type in the
 		// gomatrixserverlib.Transaction struct
 		t.PDUs = append(t.PDUs, pdu.pdu.JSON())
-		pduReceipts = append(pduReceipts, pdu.receipt)
+		pduReceipts = append(pduReceipts, pdu.dbReceipt)
 	}
 
 	// Do the same for pending EDUS in the queue.
@@ -497,7 +543,7 @@ func (oq *destinationQueue) createTransaction(
 			continue
 		}
 		t.EDUs = append(t.EDUs, *edu.edu)
-		eduReceipts = append(eduReceipts, edu.receipt)
+		eduReceipts = append(eduReceipts, edu.dbReceipt)
 	}
 
 	return t, pduReceipts, eduReceipts
@@ -530,10 +576,11 @@ func (oq *destinationQueue) blacklistDestination() {
 
 // handleTransactionSuccess updates the cached event queues as well as the success and
 // backoff information for this server.
-func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int) {
+func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int, sendMethod statistics.SendMethod) {
 	// If we successfully sent the transaction then clear out
 	// the pending events and EDUs, and wipe our transaction ID.
-	oq.statistics.Success()
+
+	oq.statistics.Success(sendMethod)
 	oq.pendingMutex.Lock()
 	defer oq.pendingMutex.Unlock()
 

federationapi/queue/queue.go

@@ -30,7 +30,7 @@ import (
 	fedapi "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -138,13 +138,13 @@ func NewOutgoingQueues(
 }
 
 type queuedPDU struct {
-	receipt *shared.Receipt
-	pdu     *gomatrixserverlib.HeaderedEvent
+	dbReceipt *receipt.Receipt
+	pdu       *gomatrixserverlib.HeaderedEvent
 }
 
 type queuedEDU struct {
-	receipt *shared.Receipt
-	edu     *gomatrixserverlib.EDU
+	dbReceipt *receipt.Receipt
+	edu       *gomatrixserverlib.EDU
 }
 
 func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue {
@@ -374,24 +374,13 @@ func (oqs *OutgoingQueues) SendEDU(
 	return nil
 }
 
-// IsServerBlacklisted returns whether or not the provided server is currently
-// blacklisted.
-func (oqs *OutgoingQueues) IsServerBlacklisted(srv gomatrixserverlib.ServerName) bool {
-	return oqs.statistics.ForServer(srv).Blacklisted()
-}
-
 // RetryServer attempts to resend events to the given server if we had given up.
-func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) {
+func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName, wasBlacklisted bool) {
 	if oqs.disabled {
 		return
 	}
 
-	serverStatistics := oqs.statistics.ForServer(srv)
-	forceWakeup := serverStatistics.Blacklisted()
-	serverStatistics.RemoveBlacklist()
-	serverStatistics.ClearBackoff()
-
 	if queue := oqs.getQueue(srv); queue != nil {
-		queue.wakeQueueIfEventsPending(forceWakeup)
+		queue.wakeQueueIfEventsPending(wasBlacklisted)
 	}
 }

federationapi/queue/queue_test.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"sync"
 	"testing"
 	"time"
 
@@ -26,13 +25,11 @@ import (
 	"gotest.tools/v3/poll"
 
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -57,7 +54,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
 		}
 	} else {
 		// Fake Database
-		db := createDatabase()
+		db := test.NewInMemoryFederationDatabase()
 		b := struct {
 			ProcessContext *process.ProcessContext
 		}{ProcessContext: process.NewProcessContext()}
@@ -65,220 +62,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
 	}
 }
 
-func createDatabase() storage.Database {
-	return &fakeDatabase{
-		pendingPDUServers:  make(map[gomatrixserverlib.ServerName]struct{}),
-		pendingEDUServers:  make(map[gomatrixserverlib.ServerName]struct{}),
-		blacklistedServers: make(map[gomatrixserverlib.ServerName]struct{}),
-		pendingPDUs:        make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent),
-		pendingEDUs:        make(map[*shared.Receipt]*gomatrixserverlib.EDU),
-		associatedPDUs:     make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}),
-		associatedEDUs:     make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}),
-	}
-}
-
-type fakeDatabase struct {
-	storage.Database
-	dbMutex            sync.Mutex
-	pendingPDUServers  map[gomatrixserverlib.ServerName]struct{}
-	pendingEDUServers  map[gomatrixserverlib.ServerName]struct{}
-	blacklistedServers map[gomatrixserverlib.ServerName]struct{}
-	pendingPDUs        map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent
-	pendingEDUs        map[*shared.Receipt]*gomatrixserverlib.EDU
-	associatedPDUs     map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}
-	associatedEDUs     map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}
-}
-
-var nidMutex sync.Mutex
-var nid = int64(0)
-
-func (d *fakeDatabase) StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	var event gomatrixserverlib.HeaderedEvent
-	if err := json.Unmarshal([]byte(js), &event); err == nil {
-		nidMutex.Lock()
-		defer nidMutex.Unlock()
-		nid++
-		receipt := shared.NewReceipt(nid)
-		d.pendingPDUs[&receipt] = &event
-		return &receipt, nil
-	}
-
-	var edu gomatrixserverlib.EDU
-	if err := json.Unmarshal([]byte(js), &edu); err == nil {
-		nidMutex.Lock()
-		defer nidMutex.Unlock()
-		nid++
-		receipt := shared.NewReceipt(nid)
-		d.pendingEDUs[&receipt] = &edu
-		return &receipt, nil
-	}
-
-	return nil, errors.New("Failed to determine type of json to store")
-}
-
-func (d *fakeDatabase) GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	pduCount := 0
-	pdus = make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent)
-	if receipts, ok := d.associatedPDUs[serverName]; ok {
-		for receipt := range receipts {
-			if event, ok := d.pendingPDUs[receipt]; ok {
-				pdus[receipt] = event
-				pduCount++
-				if pduCount == limit {
-					break
-				}
-			}
-		}
-	}
-	return pdus, nil
-}
-
-func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	eduCount := 0
-	edus = make(map[*shared.Receipt]*gomatrixserverlib.EDU)
-	if receipts, ok := d.associatedEDUs[serverName]; ok {
-		for receipt := range receipts {
-			if event, ok := d.pendingEDUs[receipt]; ok {
-				edus[receipt] = event
-				eduCount++
-				if eduCount == limit {
-					break
-				}
-			}
-		}
-	}
-	return edus, nil
-}
-
-func (d *fakeDatabase) AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if _, ok := d.pendingPDUs[receipt]; ok {
-		for destination := range destinations {
-			if _, ok := d.associatedPDUs[destination]; !ok {
-				d.associatedPDUs[destination] = make(map[*shared.Receipt]struct{})
-			}
-			d.associatedPDUs[destination][receipt] = struct{}{}
-		}
-
-		return nil
-	} else {
-		return errors.New("PDU doesn't exist")
-	}
-}
-
-func (d *fakeDatabase) AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if _, ok := d.pendingEDUs[receipt]; ok {
-		for destination := range destinations {
-			if _, ok := d.associatedEDUs[destination]; !ok {
-				d.associatedEDUs[destination] = make(map[*shared.Receipt]struct{})
-			}
-			d.associatedEDUs[destination][receipt] = struct{}{}
-		}
-
-		return nil
-	} else {
-		return errors.New("EDU doesn't exist")
-	}
-}
-
-func (d *fakeDatabase) CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if pdus, ok := d.associatedPDUs[serverName]; ok {
-		for _, receipt := range receipts {
-			delete(pdus, receipt)
-		}
-	}
-
-	return nil
-}
-
-func (d *fakeDatabase) CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	if edus, ok := d.associatedEDUs[serverName]; ok {
-		for _, receipt := range receipts {
-			delete(edus, receipt)
-		}
-	}
-
-	return nil
-}
-
-func (d *fakeDatabase) GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	servers := []gomatrixserverlib.ServerName{}
-	for server := range d.pendingPDUServers {
-		servers = append(servers, server)
-	}
-	return servers, nil
-}
-
-func (d *fakeDatabase) GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	servers := []gomatrixserverlib.ServerName{}
-	for server := range d.pendingEDUServers {
-		servers = append(servers, server)
-	}
-	return servers, nil
-}
-
-func (d *fakeDatabase) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	d.blacklistedServers[serverName] = struct{}{}
-	return nil
-}
-
-func (d *fakeDatabase) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	delete(d.blacklistedServers, serverName)
-	return nil
-}
-
-func (d *fakeDatabase) RemoveAllServersFromBlacklist() error {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	d.blacklistedServers = make(map[gomatrixserverlib.ServerName]struct{})
-	return nil
-}
-
-func (d *fakeDatabase) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) {
-	d.dbMutex.Lock()
-	defer d.dbMutex.Unlock()
-
-	isBlacklisted := false
-	if _, ok := d.blacklistedServers[serverName]; ok {
-		isBlacklisted = true
-	}
-
-	return isBlacklisted, nil
-}
-
 type stubFederationRoomServerAPI struct {
 	rsapi.FederationRoomserverAPI
 }
@@ -290,8 +73,10 @@ func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Cont
 
 type stubFederationClient struct {
 	api.FederationClient
-	shouldTxSucceed bool
-	txCount         atomic.Uint32
+	shouldTxSucceed      bool
+	shouldTxRelaySucceed bool
+	txCount              atomic.Uint32
+	txRelayCount         atomic.Uint32
 }
 
 func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) {
@@ -304,6 +89,16 @@ func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixse
 	return gomatrixserverlib.RespSend{}, result
 }
 
+func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error) {
+	var result error
+	if !f.shouldTxRelaySucceed {
+		result = fmt.Errorf("relay transaction failed")
+	}
+
+	f.txRelayCount.Add(1)
+	return gomatrixserverlib.EmptyResp{}, result
+}
+
 func mustCreatePDU(t *testing.T) *gomatrixserverlib.HeaderedEvent {
 	t.Helper()
 	content := `{"type":"m.room.message"}`
@@ -319,15 +114,18 @@ func mustCreateEDU(t *testing.T) *gomatrixserverlib.EDU {
 	return &gomatrixserverlib.EDU{Type: gomatrixserverlib.MTyping}
 }
 
-func testSetup(failuresUntilBlacklist uint32, shouldTxSucceed bool, t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *stubFederationClient, *OutgoingQueues, *process.ProcessContext, func()) {
+func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32, shouldTxSucceed bool, shouldTxRelaySucceed bool, t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *stubFederationClient, *OutgoingQueues, *process.ProcessContext, func()) {
 	db, processContext, close := mustCreateFederationDatabase(t, dbType, realDatabase)
 
 	fc := &stubFederationClient{
-		shouldTxSucceed: shouldTxSucceed,
-		txCount:         *atomic.NewUint32(0),
+		shouldTxSucceed:      shouldTxSucceed,
+		shouldTxRelaySucceed: shouldTxRelaySucceed,
+		txCount:              *atomic.NewUint32(0),
+		txRelayCount:         *atomic.NewUint32(0),
 	}
 	rs := &stubFederationRoomServerAPI{}
-	stats := statistics.NewStatistics(db, failuresUntilBlacklist)
+
+	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
 	signingInfo := []*gomatrixserverlib.SigningIdentity{
 		{
 			KeyID:      "ed21019:auto",
@@ -344,7 +142,7 @@ func TestSendPDUOnSuccessRemovedFromDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -373,7 +171,7 @@ func TestSendEDUOnSuccessRemovedFromDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -402,7 +200,7 @@ func TestSendPDUOnFailStoredInDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -432,7 +230,7 @@ func TestSendEDUOnFailStoredInDB(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -462,7 +260,7 @@ func TestSendPDUAgainDoesntInterruptBackoff(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -513,7 +311,7 @@ func TestSendEDUAgainDoesntInterruptBackoff(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -564,7 +362,7 @@ func TestSendPDUMultipleFailuresBlacklisted(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -596,7 +394,7 @@ func TestSendEDUMultipleFailuresBlacklisted(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -628,7 +426,7 @@ func TestSendPDUBlacklistedWithPriorExternalFailure(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -662,7 +460,7 @@ func TestSendEDUBlacklistedWithPriorExternalFailure(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(2)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -696,7 +494,7 @@ func TestRetryServerSendsPDUSuccessfully(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(1)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -730,8 +528,8 @@ func TestRetryServerSendsPDUSuccessfully(t *testing.T) {
 	poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
 
 	fc.shouldTxSucceed = true
-	db.RemoveServerFromBlacklist(destination)
-	queues.RetryServer(destination)
+	wasBlacklisted := dest.statistics.MarkServerAlive()
+	queues.RetryServer(destination, wasBlacklisted)
 	checkRetry := func(log poll.LogT) poll.Result {
 		data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100)
 		assert.NoError(t, dbErr)
@@ -747,7 +545,7 @@ func TestRetryServerSendsEDUSuccessfully(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(1)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -781,8 +579,8 @@ func TestRetryServerSendsEDUSuccessfully(t *testing.T) {
 	poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
 
 	fc.shouldTxSucceed = true
-	db.RemoveServerFromBlacklist(destination)
-	queues.RetryServer(destination)
+	wasBlacklisted := dest.statistics.MarkServerAlive()
+	queues.RetryServer(destination, wasBlacklisted)
 	checkRetry := func(log poll.LogT) poll.Result {
 		data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100)
 		assert.NoError(t, dbErr)
@@ -801,7 +599,7 @@ func TestSendPDUBatches(t *testing.T) {
 
 	// test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 	// db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true)
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -845,7 +643,7 @@ func TestSendEDUBatches(t *testing.T) {
 
 	// test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 	// db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true)
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -889,7 +687,7 @@ func TestSendPDUAndEDUBatches(t *testing.T) {
 
 	// test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 	// db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true)
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -940,7 +738,7 @@ func TestExternalFailureBackoffDoesntStartQueue(t *testing.T) {
 	t.Parallel()
 	failuresUntilBlacklist := uint32(16)
 	destination := gomatrixserverlib.ServerName("remotehost")
-	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false)
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, true, false, t, test.DBTypeSQLite, false)
 	defer close()
 	defer func() {
 		pc.ShutdownDendrite()
@@ -978,7 +776,7 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) {
 	destination := gomatrixserverlib.ServerName("remotehost")
 	destinations := map[gomatrixserverlib.ServerName]struct{}{destination: {}}
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-		db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, dbType, true)
+		db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilBlacklist+1, false, false, t, dbType, true)
 		// NOTE : These defers aren't called if go test is killed so the dbs may not get cleaned up.
 		defer close()
 		defer func() {
@@ -1023,8 +821,8 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) {
 		poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(10*time.Second), poll.WithDelay(100*time.Millisecond))
 
 		fc.shouldTxSucceed = true
-		db.RemoveServerFromBlacklist(destination)
-		queues.RetryServer(destination)
+		wasBlacklisted := dest.statistics.MarkServerAlive()
+		queues.RetryServer(destination, wasBlacklisted)
 		checkRetry := func(log poll.LogT) poll.Result {
 			pduData, dbErrPDU := db.GetPendingPDUs(pc.Context(), destination, 200)
 			assert.NoError(t, dbErrPDU)
@@ -1038,3 +836,147 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) {
 		poll.WaitOn(t, checkRetry, poll.WithTimeout(10*time.Second), poll.WithDelay(100*time.Millisecond))
 	})
 }
+
+func TestSendPDUMultipleFailuresAssumedOffline(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(7)
+	failuresUntilAssumedOffline := uint32(2)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, false, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	ev := mustCreatePDU(t)
+	err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() == failuresUntilAssumedOffline {
+			data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100)
+			assert.NoError(t, dbErr)
+			if len(data) == 1 {
+				if val, _ := db.IsServerAssumedOffline(context.Background(), destination); val {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for server to be assumed offline")
+			}
+			return poll.Continue("waiting for event to be added to database. Currently present PDU: %d", len(data))
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+}
+
+func TestSendEDUMultipleFailuresAssumedOffline(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(7)
+	failuresUntilAssumedOffline := uint32(2)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, false, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	ev := mustCreateEDU(t)
+	err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() == failuresUntilAssumedOffline {
+			data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100)
+			assert.NoError(t, dbErr)
+			if len(data) == 1 {
+				if val, _ := db.IsServerAssumedOffline(context.Background(), destination); val {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for server to be assumed offline")
+			}
+			return poll.Continue("waiting for event to be added to database. Currently present EDU: %d", len(data))
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+}
+
+func TestSendPDUOnRelaySuccessRemovedFromDB(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(16)
+	failuresUntilAssumedOffline := uint32(1)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, true, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	relayServers := []gomatrixserverlib.ServerName{"relayserver"}
+	queues.statistics.ForServer(destination).AddRelayServers(relayServers)
+
+	ev := mustCreatePDU(t)
+	err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() >= 1 {
+			if fc.txRelayCount.Load() == 1 {
+				data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100)
+				assert.NoError(t, dbErr)
+				if len(data) == 0 {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for event to be removed from database. Currently present PDU: %d", len(data))
+			}
+			return poll.Continue("waiting for more relay send attempts before checking database. Currently %d", fc.txRelayCount.Load())
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+
+	assumedOffline, _ := db.IsServerAssumedOffline(context.Background(), destination)
+	assert.Equal(t, true, assumedOffline)
+}
+
+func TestSendEDUOnRelaySuccessRemovedFromDB(t *testing.T) {
+	t.Parallel()
+	failuresUntilBlacklist := uint32(16)
+	failuresUntilAssumedOffline := uint32(1)
+	destination := gomatrixserverlib.ServerName("remotehost")
+	db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, failuresUntilAssumedOffline, false, true, t, test.DBTypeSQLite, false)
+	defer close()
+	defer func() {
+		pc.ShutdownDendrite()
+		<-pc.WaitForShutdown()
+	}()
+
+	relayServers := []gomatrixserverlib.ServerName{"relayserver"}
+	queues.statistics.ForServer(destination).AddRelayServers(relayServers)
+
+	ev := mustCreateEDU(t)
+	err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination})
+	assert.NoError(t, err)
+
+	check := func(log poll.LogT) poll.Result {
+		if fc.txCount.Load() >= 1 {
+			if fc.txRelayCount.Load() == 1 {
+				data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100)
+				assert.NoError(t, dbErr)
+				if len(data) == 0 {
+					return poll.Success()
+				}
+				return poll.Continue("waiting for event to be removed from database. Currently present EDU: %d", len(data))
+			}
+			return poll.Continue("waiting for more relay send attempts before checking database. Currently %d", fc.txRelayCount.Load())
+		}
+		return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load())
+	}
+	poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
+
+	assumedOffline, _ := db.IsServerAssumedOffline(context.Background(), destination)
+	assert.Equal(t, true, assumedOffline)
+}

federationapi/routing/devices.go

@@ -17,7 +17,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	keyapi "github.com/matrix-org/dendrite/keyserver/api"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/tidwall/gjson"
@@ -26,11 +26,11 @@ import (
 // GetUserDevices for the given user id
 func GetUserDevices(
 	req *http.Request,
-	keyAPI keyapi.FederationKeyAPI,
+	keyAPI api.FederationKeyAPI,
 	userID string,
 ) util.JSONResponse {
-	var res keyapi.QueryDeviceMessagesResponse
-	if err := keyAPI.QueryDeviceMessages(req.Context(), &keyapi.QueryDeviceMessagesRequest{
+	var res api.QueryDeviceMessagesResponse
+	if err := keyAPI.QueryDeviceMessages(req.Context(), &api.QueryDeviceMessagesRequest{
 		UserID: userID,
 	}, &res); err != nil {
 		return util.ErrorResponse(err)
@@ -40,12 +40,12 @@ func GetUserDevices(
 		return jsonerror.InternalServerError()
 	}
 
-	sigReq := &keyapi.QuerySignaturesRequest{
+	sigReq := &api.QuerySignaturesRequest{
 		TargetIDs: map[string][]gomatrixserverlib.KeyID{
 			userID: {},
 		},
 	}
-	sigRes := &keyapi.QuerySignaturesResponse{}
+	sigRes := &api.QuerySignaturesResponse{}
 	for _, dev := range res.Devices {
 		sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID))
 	}

federationapi/routing/keys.go

@@ -22,8 +22,8 @@ import (
 	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
-	"github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"

federationapi/routing/profile_test.go

@@ -0,0 +1,94 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing_test
+
+import (
+	"context"
+	"encoding/hex"
+	"io"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	fedAPI "github.com/matrix-org/dendrite/federationapi"
+	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
+	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ed25519"
+)
+
+type fakeUserAPI struct {
+	userAPI.FederationUserAPI
+}
+
+func (u *fakeUserAPI) QueryProfile(ctx context.Context, req *userAPI.QueryProfileRequest, res *userAPI.QueryProfileResponse) error {
+	return nil
+}
+
+func TestHandleQueryProfile(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
+
+		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
+		base.PublicFederationAPIMux = fedMux
+		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		fedClient := fakeFedClient{}
+		serverKeyAPI := &signing.YggdrasilKeys{}
+		keyRing := serverKeyAPI.KeyRing()
+		fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true)
+		userapi := fakeUserAPI{}
+		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+		if !ok {
+			panic("This is a programming error.")
+		}
+		routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil)
+
+		handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP
+		_, sk, _ := ed25519.GenerateKey(nil)
+		keyID := signing.KeyID
+		pk := sk.Public().(ed25519.PublicKey)
+		serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+		req := gomatrixserverlib.NewFederationRequest("GET", serverName, testOrigin, "/query/profile?user_id="+url.QueryEscape("@user:"+string(testOrigin)))
+		type queryContent struct{}
+		content := queryContent{}
+		err := req.SetContent(content)
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
+		httpReq, err := req.HTTPRequest()
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		// vars := map[string]string{"room_alias": "#room:server"}
+		w := httptest.NewRecorder()
+		// httpReq = mux.SetURLVars(httpReq, vars)
+		handler(w, httpReq)
+
+		res := w.Result()
+		data, _ := io.ReadAll(res.Body)
+		println(string(data))
+		assert.Equal(t, 200, res.StatusCode)
+	})
+}

federationapi/routing/query_test.go

@@ -0,0 +1,94 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing_test
+
+import (
+	"context"
+	"encoding/hex"
+	"io"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	fedAPI "github.com/matrix-org/dendrite/federationapi"
+	fedclient "github.com/matrix-org/dendrite/federationapi/api"
+	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
+	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ed25519"
+)
+
+type fakeFedClient struct {
+	fedclient.FederationClient
+}
+
+func (f *fakeFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) {
+	return
+}
+
+func TestHandleQueryDirectory(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
+
+		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
+		base.PublicFederationAPIMux = fedMux
+		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		fedClient := fakeFedClient{}
+		serverKeyAPI := &signing.YggdrasilKeys{}
+		keyRing := serverKeyAPI.KeyRing()
+		fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true)
+		userapi := fakeUserAPI{}
+		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+		if !ok {
+			panic("This is a programming error.")
+		}
+		routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil)
+
+		handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP
+		_, sk, _ := ed25519.GenerateKey(nil)
+		keyID := signing.KeyID
+		pk := sk.Public().(ed25519.PublicKey)
+		serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+		req := gomatrixserverlib.NewFederationRequest("GET", serverName, testOrigin, "/query/directory?room_alias="+url.QueryEscape("#room:server"))
+		type queryContent struct{}
+		content := queryContent{}
+		err := req.SetContent(content)
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
+		httpReq, err := req.HTTPRequest()
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
+		}
+		// vars := map[string]string{"room_alias": "#room:server"}
+		w := httptest.NewRecorder()
+		// httpReq = mux.SetURLVars(httpReq, vars)
+		handler(w, httpReq)
+
+		res := w.Result()
+		data, _ := io.ReadAll(res.Body)
+		println(string(data))
+		assert.Equal(t, 200, res.StatusCode)
+	})
+}

federationapi/routing/routing.go

@@ -29,9 +29,9 @@ import (
 	"github.com/matrix-org/dendrite/federationapi/producers"
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/httputil"
-	keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -40,6 +40,12 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	SendRouteName           = "Send"
+	QueryDirectoryRouteName = "QueryDirectory"
+	QueryProfileRouteName   = "QueryProfile"
+)
+
 // Setup registers HTTP handlers with the given ServeMux.
 // The provided publicAPIMux MUST have `UseEncodedPath()` enabled or else routes will incorrectly
 // path unescape twice (once from the router, once from MakeFedAPI). We need to have this enabled
@@ -49,21 +55,26 @@ import (
 // applied:
 // nolint: gocyclo
 func Setup(
-	fedMux, keyMux, wkMux *mux.Router,
-	cfg *config.FederationAPI,
+	base *base.BaseDendrite,
 	rsAPI roomserverAPI.FederationRoomserverAPI,
 	fsAPI *fedInternal.FederationInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation federationAPI.FederationClient,
 	userAPI userapi.FederationUserAPI,
-	keyAPI keyserverAPI.FederationKeyAPI,
 	mscCfg *config.MSCs,
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,
 ) {
-	prometheus.MustRegister(
-		pduCountTotal, eduCountTotal,
-	)
+	fedMux := base.PublicFederationAPIMux
+	keyMux := base.PublicKeyAPIMux
+	wkMux := base.PublicWellKnownAPIMux
+	cfg := &base.Cfg.FederationAPI
+
+	if base.EnableMetrics {
+		prometheus.MustRegister(
+			internal.PDUCountTotal, internal.EDUCountTotal,
+		)
+	}
 
 	v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
 	v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
@@ -128,10 +139,10 @@ func Setup(
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return Send(
 				httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]),
-				cfg, rsAPI, keyAPI, keys, federation, mu, servers, producer,
+				cfg, rsAPI, userAPI, keys, federation, mu, servers, producer,
 			)
 		},
-	)).Methods(http.MethodPut, http.MethodOptions)
+	)).Methods(http.MethodPut, http.MethodOptions).Name(SendRouteName)
 
 	v1fedmux.Handle("/invite/{roomID}/{eventID}", MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
@@ -241,7 +252,7 @@ func Setup(
 				httpReq, federation, cfg, rsAPI, fsAPI,
 			)
 		},
-	)).Methods(http.MethodGet)
+	)).Methods(http.MethodGet).Name(QueryDirectoryRouteName)
 
 	v1fedmux.Handle("/query/profile", MakeFedAPI(
 		"federation_query_profile", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
@@ -250,13 +261,13 @@ func Setup(
 				httpReq, userAPI, cfg,
 			)
 		},
-	)).Methods(http.MethodGet)
+	)).Methods(http.MethodGet).Name(QueryProfileRouteName)
 
 	v1fedmux.Handle("/user/devices/{userID}", MakeFedAPI(
 		"federation_user_devices", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
 			return GetUserDevices(
-				httpReq, keyAPI, vars["userID"],
+				httpReq, userAPI, vars["userID"],
 			)
 		},
 	)).Methods(http.MethodGet)
@@ -481,14 +492,14 @@ func Setup(
 	v1fedmux.Handle("/user/keys/claim", MakeFedAPI(
 		"federation_keys_claim", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
-			return ClaimOneTimeKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName)
+			return ClaimOneTimeKeys(httpReq, request, userAPI, cfg.Matrix.ServerName)
 		},
 	)).Methods(http.MethodPost)
 
 	v1fedmux.Handle("/user/keys/query", MakeFedAPI(
 		"federation_keys_query", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
-			return QueryDeviceKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName)
+			return QueryDeviceKeys(httpReq, request, userAPI, cfg.Matrix.ServerName)
 		},
 	)).Methods(http.MethodPost)
 

federationapi/routing/send.go

@@ -17,26 +17,20 @@ package routing
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"sync"
 	"time"
 
-	"github.com/getsentry/sentry-go"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/producers"
-	"github.com/matrix-org/dendrite/federationapi/types"
 	"github.com/matrix-org/dendrite/internal"
-	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	syncTypes "github.com/matrix-org/dendrite/syncapi/types"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
 )
 
 const (
@@ -56,26 +50,6 @@ const (
 	MetricsWorkMissingPrevEvents = "missing_prev_events"
 )
 
-var (
-	pduCountTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "dendrite",
-			Subsystem: "federationapi",
-			Name:      "recv_pdus",
-			Help:      "Number of incoming PDUs from remote servers with labels for success",
-		},
-		[]string{"status"}, // 'success' or 'total'
-	)
-	eduCountTotal = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: "dendrite",
-			Subsystem: "federationapi",
-			Name:      "recv_edus",
-			Help:      "Number of incoming EDUs from remote servers",
-		},
-	)
-)
-
 var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse
 
 // Send implements /_matrix/federation/v1/send/{txnID}
@@ -85,7 +59,7 @@ func Send(
 	txnID gomatrixserverlib.TransactionID,
 	cfg *config.FederationAPI,
 	rsAPI api.FederationRoomserverAPI,
-	keyAPI keyapi.FederationKeyAPI,
+	keyAPI userAPI.FederationUserAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	federation federationAPI.FederationClient,
 	mu *internal.MutexByRoom,
@@ -123,18 +97,6 @@ func Send(
 	defer close(ch)
 	defer inFlightTxnsPerOrigin.Delete(index)
 
-	t := txnReq{
-		rsAPI:                  rsAPI,
-		keys:                   keys,
-		ourServerName:          cfg.Matrix.ServerName,
-		federation:             federation,
-		servers:                servers,
-		keyAPI:                 keyAPI,
-		roomsMu:                mu,
-		producer:               producer,
-		inboundPresenceEnabled: cfg.Matrix.Presence.EnableInbound,
-	}
-
 	var txnEvents struct {
 		PDUs []json.RawMessage       `json:"pdus"`
 		EDUs []gomatrixserverlib.EDU `json:"edus"`
@@ -155,16 +117,23 @@ func Send(
 		}
 	}
 
-	// TODO: Really we should have a function to convert FederationRequest to txnReq
-	t.PDUs = txnEvents.PDUs
-	t.EDUs = txnEvents.EDUs
-	t.Origin = request.Origin()
-	t.TransactionID = txnID
-	t.Destination = cfg.Matrix.ServerName
+	t := internal.NewTxnReq(
+		rsAPI,
+		keyAPI,
+		cfg.Matrix.ServerName,
+		keys,
+		mu,
+		producer,
+		cfg.Matrix.Presence.EnableInbound,
+		txnEvents.PDUs,
+		txnEvents.EDUs,
+		request.Origin(),
+		txnID,
+		cfg.Matrix.ServerName)
 
 	util.GetLogger(httpReq.Context()).Debugf("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs))
 
-	resp, jsonErr := t.processTransaction(httpReq.Context())
+	resp, jsonErr := t.ProcessTransaction(httpReq.Context())
 	if jsonErr != nil {
 		util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed")
 		return *jsonErr
@@ -181,283 +150,3 @@ func Send(
 	ch <- res
 	return res
 }
-
-type txnReq struct {
-	gomatrixserverlib.Transaction
-	rsAPI                  api.FederationRoomserverAPI
-	keyAPI                 keyapi.FederationKeyAPI
-	ourServerName          gomatrixserverlib.ServerName
-	keys                   gomatrixserverlib.JSONVerifier
-	federation             txnFederationClient
-	roomsMu                *internal.MutexByRoom
-	servers                federationAPI.ServersInRoomProvider
-	producer               *producers.SyncAPIProducer
-	inboundPresenceEnabled bool
-}
-
-// A subset of FederationClient functionality that txn requires. Useful for testing.
-type txnFederationClient interface {
-	LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (
-		res gomatrixserverlib.RespState, err error,
-	)
-	LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
-	GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
-	LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents,
-		roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
-}
-
-func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.RespSend, *util.JSONResponse) {
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		t.processEDUs(ctx)
-	}()
-
-	results := make(map[string]gomatrixserverlib.PDUResult)
-	roomVersions := make(map[string]gomatrixserverlib.RoomVersion)
-	getRoomVersion := func(roomID string) gomatrixserverlib.RoomVersion {
-		if v, ok := roomVersions[roomID]; ok {
-			return v
-		}
-		verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
-		verRes := api.QueryRoomVersionForRoomResponse{}
-		if err := t.rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
-			util.GetLogger(ctx).WithError(err).Debug("Transaction: Failed to query room version for room", verReq.RoomID)
-			return ""
-		}
-		roomVersions[roomID] = verRes.RoomVersion
-		return verRes.RoomVersion
-	}
-
-	for _, pdu := range t.PDUs {
-		pduCountTotal.WithLabelValues("total").Inc()
-		var header struct {
-			RoomID string `json:"room_id"`
-		}
-		if err := json.Unmarshal(pdu, &header); err != nil {
-			util.GetLogger(ctx).WithError(err).Debug("Transaction: Failed to extract room ID from event")
-			// We don't know the event ID at this point so we can't return the
-			// failure in the PDU results
-			continue
-		}
-		roomVersion := getRoomVersion(header.RoomID)
-		event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion)
-		if err != nil {
-			if _, ok := err.(gomatrixserverlib.BadJSONError); ok {
-				// Room version 6 states that homeservers should strictly enforce canonical JSON
-				// on PDUs.
-				//
-				// This enforces that the entire transaction is rejected if a single bad PDU is
-				// sent. It is unclear if this is the correct behaviour or not.
-				//
-				// See https://github.com/matrix-org/synapse/issues/7543
-				return nil, &util.JSONResponse{
-					Code: 400,
-					JSON: jsonerror.BadJSON("PDU contains bad JSON"),
-				}
-			}
-			util.GetLogger(ctx).WithError(err).Debugf("Transaction: Failed to parse event JSON of event %s", string(pdu))
-			continue
-		}
-		if event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("") {
-			continue
-		}
-		if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID(), t.Origin) {
-			results[event.EventID()] = gomatrixserverlib.PDUResult{
-				Error: "Forbidden by server ACLs",
-			}
-			continue
-		}
-		if err = event.VerifyEventSignatures(ctx, t.keys); err != nil {
-			util.GetLogger(ctx).WithError(err).Debugf("Transaction: Couldn't validate signature of event %q", event.EventID())
-			results[event.EventID()] = gomatrixserverlib.PDUResult{
-				Error: err.Error(),
-			}
-			continue
-		}
-
-		// pass the event to the roomserver which will do auth checks
-		// If the event fail auth checks, gmsl.NotAllowed error will be returned which we be silently
-		// discarded by the caller of this function
-		if err = api.SendEvents(
-			ctx,
-			t.rsAPI,
-			api.KindNew,
-			[]*gomatrixserverlib.HeaderedEvent{
-				event.Headered(roomVersion),
-			},
-			t.Destination,
-			t.Origin,
-			api.DoNotSendToOtherServers,
-			nil,
-			true,
-		); err != nil {
-			util.GetLogger(ctx).WithError(err).Errorf("Transaction: Couldn't submit event %q to input queue: %s", event.EventID(), err)
-			results[event.EventID()] = gomatrixserverlib.PDUResult{
-				Error: err.Error(),
-			}
-			continue
-		}
-
-		results[event.EventID()] = gomatrixserverlib.PDUResult{}
-		pduCountTotal.WithLabelValues("success").Inc()
-	}
-
-	wg.Wait()
-	return &gomatrixserverlib.RespSend{PDUs: results}, nil
-}
-
-// nolint:gocyclo
-func (t *txnReq) processEDUs(ctx context.Context) {
-	for _, e := range t.EDUs {
-		eduCountTotal.Inc()
-		switch e.Type {
-		case gomatrixserverlib.MTyping:
-			// https://matrix.org/docs/spec/server_server/latest#typing-notifications
-			var typingPayload struct {
-				RoomID string `json:"room_id"`
-				UserID string `json:"user_id"`
-				Typing bool   `json:"typing"`
-			}
-			if err := json.Unmarshal(e.Content, &typingPayload); err != nil {
-				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal typing event")
-				continue
-			}
-			if _, serverName, err := gomatrixserverlib.SplitID('@', typingPayload.UserID); err != nil {
-				continue
-			} else if serverName == t.ourServerName {
-				continue
-			} else if serverName != t.Origin {
-				continue
-			}
-			if err := t.producer.SendTyping(ctx, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil {
-				util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to JetStream")
-			}
-		case gomatrixserverlib.MDirectToDevice:
-			// https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema
-			var directPayload gomatrixserverlib.ToDeviceMessage
-			if err := json.Unmarshal(e.Content, &directPayload); err != nil {
-				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal send-to-device events")
-				continue
-			}
-			if _, serverName, err := gomatrixserverlib.SplitID('@', directPayload.Sender); err != nil {
-				continue
-			} else if serverName == t.ourServerName {
-				continue
-			} else if serverName != t.Origin {
-				continue
-			}
-			for userID, byUser := range directPayload.Messages {
-				for deviceID, message := range byUser {
-					// TODO: check that the user and the device actually exist here
-					if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil {
-						sentry.CaptureException(err)
-						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
-							"sender":    directPayload.Sender,
-							"user_id":   userID,
-							"device_id": deviceID,
-						}).Error("Failed to send send-to-device event to JetStream")
-					}
-				}
-			}
-		case gomatrixserverlib.MDeviceListUpdate:
-			if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil {
-				sentry.CaptureException(err)
-				util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate")
-			}
-		case gomatrixserverlib.MReceipt:
-			// https://matrix.org/docs/spec/server_server/r0.1.4#receipts
-			payload := map[string]types.FederationReceiptMRead{}
-
-			if err := json.Unmarshal(e.Content, &payload); err != nil {
-				util.GetLogger(ctx).WithError(err).Debug("Failed to unmarshal receipt event")
-				continue
-			}
-
-			for roomID, receipt := range payload {
-				for userID, mread := range receipt.User {
-					_, domain, err := gomatrixserverlib.SplitID('@', userID)
-					if err != nil {
-						util.GetLogger(ctx).WithError(err).Debug("Failed to split domain from receipt event sender")
-						continue
-					}
-					if t.Origin != domain {
-						util.GetLogger(ctx).Debugf("Dropping receipt event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin)
-						continue
-					}
-					if err := t.processReceiptEvent(ctx, userID, roomID, "m.read", mread.Data.TS, mread.EventIDs); err != nil {
-						util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
-							"sender":  t.Origin,
-							"user_id": userID,
-							"room_id": roomID,
-							"events":  mread.EventIDs,
-						}).Error("Failed to send receipt event to JetStream")
-						continue
-					}
-				}
-			}
-		case types.MSigningKeyUpdate:
-			if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil {
-				sentry.CaptureException(err)
-				logrus.WithError(err).Errorf("Failed to process signing key update")
-			}
-		case gomatrixserverlib.MPresence:
-			if t.inboundPresenceEnabled {
-				if err := t.processPresence(ctx, e); err != nil {
-					logrus.WithError(err).Errorf("Failed to process presence update")
-				}
-			}
-		default:
-			util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU")
-		}
-	}
-}
-
-// processPresence handles m.receipt events
-func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) error {
-	payload := types.Presence{}
-	if err := json.Unmarshal(e.Content, &payload); err != nil {
-		return err
-	}
-	for _, content := range payload.Push {
-		if _, serverName, err := gomatrixserverlib.SplitID('@', content.UserID); err != nil {
-			continue
-		} else if serverName == t.ourServerName {
-			continue
-		} else if serverName != t.Origin {
-			continue
-		}
-		presence, ok := syncTypes.PresenceFromString(content.Presence)
-		if !ok {
-			continue
-		}
-		if err := t.producer.SendPresence(ctx, content.UserID, presence, content.StatusMsg, content.LastActiveAgo); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// processReceiptEvent sends receipt events to JetStream
-func (t *txnReq) processReceiptEvent(ctx context.Context,
-	userID, roomID, receiptType string,
-	timestamp gomatrixserverlib.Timestamp,
-	eventIDs []string,
-) error {
-	if _, serverName, err := gomatrixserverlib.SplitID('@', userID); err != nil {
-		return nil
-	} else if serverName == t.ourServerName {
-		return nil
-	} else if serverName != t.Origin {
-		return nil
-	}
-	// store every event
-	for _, eventID := range eventIDs {
-		if err := t.producer.SendReceipt(ctx, userID, roomID, eventID, receiptType, timestamp); err != nil {
-			return fmt.Errorf("unable to set receipt event: %w", err)
-		}
-	}
-
-	return nil
-}

federationapi/routing/send_test.go

@@ -1,552 +1,87 @@
-package routing
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing_test
 
 import (
-	"context"
+	"encoding/hex"
 	"encoding/json"
-	"fmt"
+	"net/http/httptest"
 	"testing"
-	"time"
 
-	"github.com/matrix-org/dendrite/internal"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	fedAPI "github.com/matrix-org/dendrite/federationapi"
+	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
+	"github.com/matrix-org/dendrite/federationapi/routing"
+	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ed25519"
 )
 
 const (
-	testOrigin      = gomatrixserverlib.ServerName("kaer.morhen")
-	testDestination = gomatrixserverlib.ServerName("white.orchard")
+	testOrigin = gomatrixserverlib.ServerName("kaer.morhen")
 )
 
-var (
-	testRoomVersion = gomatrixserverlib.RoomVersionV1
-	testData        = []json.RawMessage{
-		[]byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$0ok8ynDp7kjc95e3:kaer.morhen","hashes":{"sha256":"17kPoH+h0Dk4Omn7Sus0qMb6+oGcf+CZFEgDhv7UKWs"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"jP4a04f5/F10Pw95FPpdCyKAO44JOwUQ/MZOOeA/RTU1Dn+AHPMzGSaZnuGjRr/xQuADt+I3ctb5ZQfLKNzHDw"}},"state_key":"","type":"m.room.create"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"content":{"membership":"join"},"depth":1,"event_id":"$LEwEu0kxrtu5fOiS:kaer.morhen","hashes":{"sha256":"B7M88PhXf3vd1LaFtjQutFu4x/w7fHD28XKZ4sAsJTo"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"p2vqmuJn7ZBRImctSaKbXCAxCcBlIjPH9JHte1ouIUGy84gpu4eLipOvSBCLL26hXfC0Zrm4WUto6Hr+ohdrCg"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"join_rule":"public"},"depth":2,"event_id":"$SMHlqUrNhhBBRLeN:kaer.morhen","hashes":{"sha256":"vIuJQvmMjrGxshAkj1SXe0C4RqvMbv4ZADDw9pFCWqQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"hBMsb3Qppo3RaqqAl4JyTgaiWEbW5hlckATky6PrHun+F3YM203TzG7w9clwuQU5F5pZoB1a6nw+to0hN90FAw"}},"state_key":"","type":"m.room.join_rules"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"history_visibility":"shared"},"depth":3,"event_id":"$6F1yGIbO0J7TM93h:kaer.morhen","hashes":{"sha256":"Mr23GKSlZW7UCCYLgOWawI2Sg6KIoMjUWO2TDenuOgw"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$SMHlqUrNhhBBRLeN:kaer.morhen",{"sha256":"SylzE8U02I+6eyEHgL+FlU0L5YdqrVp8OOlxKS9VQW0"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sHLKrFI3hKGrEJfpMVZSDS3LvLasQsy50CTsOwru9XTVxgRsPo6wozNtRVjxo1J3Rk18RC9JppovmQ5VR5EcDw"}},"state_key":"","type":"m.room.history_visibility"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"ban":50,"events":null,"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":null,"users_default":0},"depth":4,"event_id":"$UKNe10XzYzG0TeA9:kaer.morhen","hashes":{"sha256":"ngbP3yja9U5dlckKerUs/fSOhtKxZMCVvsfhPURSS28"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$6F1yGIbO0J7TM93h:kaer.morhen",{"sha256":"A4CucrKSoWX4IaJXhq02mBg1sxIyZEftbC+5p3fZAvk"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"zOmwlP01QL3yFchzuR9WHvogOoBZA3oVtNIF3lM0ZfDnqlSYZB9sns27G/4HVq0k7alaK7ZE3oGoCrVnMkPNCw"}},"state_key":"","type":"m.room.power_levels"}`),
-		// messages
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":6,"event_id":"$MYSbs8m4rEbsCWXD:kaer.morhen","hashes":{"sha256":"kgbYM7v4Ud2YaBsjBTolM4ySg6rHcJNYI6nWhMSdFUA"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$gl2T9l3qm0kUbiIJ:kaer.morhen",{"sha256":"C/rD04h9wGxRdN2G/IBfrgoE1UovzLZ+uskwaKZ37/Q"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"x0UoKh968jj/F5l1/R7Ew0T6CTKuew3PLNHASNxqck/bkNe8yYQiDHXRr+kZxObeqPZZTpaF1+EI+bLU9W8GDQ"}},"type":"m.room.message"}`),
-		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":7,"event_id":"$N5x9WJkl9ClPrAEg:kaer.morhen","hashes":{"sha256":"FWM8oz4yquTunRZ67qlW2gzPDzdWfBP6RPHXhK1I/x8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$MYSbs8m4rEbsCWXD:kaer.morhen",{"sha256":"fatqgW+SE8mb2wFn3UN+drmluoD4UJ/EcSrL6Ur9q1M"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"Y+LX/xcyufoXMOIoqQBNOzy6lZfUGB1ffgXIrSugk6obMiyAsiRejHQN/pciZXsHKxMJLYRFAz4zSJoS/LGPAA"}},"type":"m.room.message"}`),
-	}
-	testEvents      = []*gomatrixserverlib.HeaderedEvent{}
-	testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent)
-)
+type sendContent struct {
+	PDUs []json.RawMessage       `json:"pdus"`
+	EDUs []gomatrixserverlib.EDU `json:"edus"`
+}
 
-func init() {
-	for _, j := range testData {
-		e, err := gomatrixserverlib.NewEventFromTrustedJSON(j, false, testRoomVersion)
+func TestHandleSend(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
+
+		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
+		base.PublicFederationAPIMux = fedMux
+		base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
+		base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
+		fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)
+		serverKeyAPI := &signing.YggdrasilKeys{}
+		keyRing := serverKeyAPI.KeyRing()
+		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+		if !ok {
+			panic("This is a programming error.")
+		}
+		routing.Setup(base, nil, r, keyRing, nil, nil, &base.Cfg.MSCs, nil, nil)
+
+		handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
+		_, sk, _ := ed25519.GenerateKey(nil)
+		keyID := signing.KeyID
+		pk := sk.Public().(ed25519.PublicKey)
+		serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+		req := gomatrixserverlib.NewFederationRequest("PUT", serverName, testOrigin, "/send/1234")
+		content := sendContent{}
+		err := req.SetContent(content)
 		if err != nil {
-			panic("cannot load test data: " + err.Error())
+			t.Fatalf("Error: %s", err.Error())
 		}
-		h := e.Headered(testRoomVersion)
-		testEvents = append(testEvents, h)
-		if e.StateKey() != nil {
-			testStateEvents[gomatrixserverlib.StateKeyTuple{
-				EventType: e.Type(),
-				StateKey:  *e.StateKey(),
-			}] = h
+		req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
+		httpReq, err := req.HTTPRequest()
+		if err != nil {
+			t.Fatalf("Error: %s", err.Error())
 		}
-	}
+		vars := map[string]string{"txnID": "1234"}
+		w := httptest.NewRecorder()
+		httpReq = mux.SetURLVars(httpReq, vars)
+		handler(w, httpReq)
+
+		res := w.Result()
+		assert.Equal(t, 200, res.StatusCode)
+	})
 }
-
-type testRoomserverAPI struct {
-	api.RoomserverInternalAPITrace
-	inputRoomEvents           []api.InputRoomEvent
-	queryStateAfterEvents     func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse
-	queryEventsByID           func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse
-	queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse
-}
-
-func (t *testRoomserverAPI) InputRoomEvents(
-	ctx context.Context,
-	request *api.InputRoomEventsRequest,
-	response *api.InputRoomEventsResponse,
-) error {
-	t.inputRoomEvents = append(t.inputRoomEvents, request.InputRoomEvents...)
-	for _, ire := range request.InputRoomEvents {
-		fmt.Println("InputRoomEvents: ", ire.Event.EventID())
-	}
-	return nil
-}
-
-// Query the latest events and state for a room from the room server.
-func (t *testRoomserverAPI) QueryLatestEventsAndState(
-	ctx context.Context,
-	request *api.QueryLatestEventsAndStateRequest,
-	response *api.QueryLatestEventsAndStateResponse,
-) error {
-	r := t.queryLatestEventsAndState(request)
-	response.RoomExists = r.RoomExists
-	response.RoomVersion = testRoomVersion
-	response.LatestEvents = r.LatestEvents
-	response.StateEvents = r.StateEvents
-	response.Depth = r.Depth
-	return nil
-}
-
-// Query the state after a list of events in a room from the room server.
-func (t *testRoomserverAPI) QueryStateAfterEvents(
-	ctx context.Context,
-	request *api.QueryStateAfterEventsRequest,
-	response *api.QueryStateAfterEventsResponse,
-) error {
-	response.RoomVersion = testRoomVersion
-	res := t.queryStateAfterEvents(request)
-	response.PrevEventsExist = res.PrevEventsExist
-	response.RoomExists = res.RoomExists
-	response.StateEvents = res.StateEvents
-	return nil
-}
-
-// Query a list of events by event ID.
-func (t *testRoomserverAPI) QueryEventsByID(
-	ctx context.Context,
-	request *api.QueryEventsByIDRequest,
-	response *api.QueryEventsByIDResponse,
-) error {
-	res := t.queryEventsByID(request)
-	response.Events = res.Events
-	return nil
-}
-
-// Query if a server is joined to a room
-func (t *testRoomserverAPI) QueryServerJoinedToRoom(
-	ctx context.Context,
-	request *api.QueryServerJoinedToRoomRequest,
-	response *api.QueryServerJoinedToRoomResponse,
-) error {
-	response.RoomExists = true
-	response.IsInRoom = true
-	return nil
-}
-
-// Asks for the room version for a given room.
-func (t *testRoomserverAPI) QueryRoomVersionForRoom(
-	ctx context.Context,
-	request *api.QueryRoomVersionForRoomRequest,
-	response *api.QueryRoomVersionForRoomResponse,
-) error {
-	response.RoomVersion = testRoomVersion
-	return nil
-}
-
-func (t *testRoomserverAPI) QueryServerBannedFromRoom(
-	ctx context.Context, req *api.QueryServerBannedFromRoomRequest, res *api.QueryServerBannedFromRoomResponse,
-) error {
-	res.Banned = false
-	return nil
-}
-
-type txnFedClient struct {
-	state            map[string]gomatrixserverlib.RespState    // event_id to response
-	stateIDs         map[string]gomatrixserverlib.RespStateIDs // event_id to response
-	getEvent         map[string]gomatrixserverlib.Transaction  // event_id to response
-	getMissingEvents func(gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error)
-}
-
-func (c *txnFedClient) LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (
-	res gomatrixserverlib.RespState, err error,
-) {
-	fmt.Println("testFederationClient.LookupState", eventID)
-	r, ok := c.state[eventID]
-	if !ok {
-		err = fmt.Errorf("txnFedClient: no /state for event %s", eventID)
-		return
-	}
-	res = r
-	return
-}
-func (c *txnFedClient) LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) {
-	fmt.Println("testFederationClient.LookupStateIDs", eventID)
-	r, ok := c.stateIDs[eventID]
-	if !ok {
-		err = fmt.Errorf("txnFedClient: no /state_ids for event %s", eventID)
-		return
-	}
-	res = r
-	return
-}
-func (c *txnFedClient) GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) {
-	fmt.Println("testFederationClient.GetEvent", eventID)
-	r, ok := c.getEvent[eventID]
-	if !ok {
-		err = fmt.Errorf("txnFedClient: no /event for event ID %s", eventID)
-		return
-	}
-	res = r
-	return
-}
-func (c *txnFedClient) LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents,
-	roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) {
-	return c.getMissingEvents(missing)
-}
-
-func mustCreateTransaction(rsAPI api.FederationRoomserverAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq {
-	t := &txnReq{
-		rsAPI:      rsAPI,
-		keys:       &test.NopJSONVerifier{},
-		federation: fedClient,
-		roomsMu:    internal.NewMutexByRoom(),
-	}
-	t.PDUs = pdus
-	t.Origin = testOrigin
-	t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
-	t.Destination = testDestination
-	return t
-}
-
-func mustProcessTransaction(t *testing.T, txn *txnReq, pdusWithErrors []string) {
-	res, err := txn.processTransaction(context.Background())
-	if err != nil {
-		t.Errorf("txn.processTransaction returned an error: %v", err)
-		return
-	}
-	if len(res.PDUs) != len(txn.PDUs) {
-		t.Errorf("txn.processTransaction did not return results for all PDUs, got %d want %d", len(res.PDUs), len(txn.PDUs))
-		return
-	}
-NextPDU:
-	for eventID, result := range res.PDUs {
-		if result.Error == "" {
-			continue
-		}
-		for _, eventIDWantError := range pdusWithErrors {
-			if eventID == eventIDWantError {
-				break NextPDU
-			}
-		}
-		t.Errorf("txn.processTransaction PDU %s returned an error %s", eventID, result.Error)
-	}
-}
-
-/*
-func fromStateTuples(tuples []gomatrixserverlib.StateKeyTuple, omitTuples []gomatrixserverlib.StateKeyTuple) (result []*gomatrixserverlib.HeaderedEvent) {
-NextTuple:
-	for _, t := range tuples {
-		for _, o := range omitTuples {
-			if t == o {
-				break NextTuple
-			}
-		}
-		h, ok := testStateEvents[t]
-		if ok {
-			result = append(result, h)
-		}
-	}
-	return
-}
-*/
-
-func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []*gomatrixserverlib.HeaderedEvent) {
-	for _, g := range got {
-		fmt.Println("GOT ", g.Event.EventID())
-	}
-	if len(got) != len(want) {
-		t.Errorf("wrong number of InputRoomEvents: got %d want %d", len(got), len(want))
-		return
-	}
-	for i := range got {
-		if got[i].Event.EventID() != want[i].EventID() {
-			t.Errorf("InputRoomEvents[%d] got %s want %s", i, got[i].Event.EventID(), want[i].EventID())
-		}
-	}
-}
-
-// The purpose of this test is to check that receiving an event over federation for which we have the prev_events works correctly, and passes it on
-// to the roomserver. It's the most basic test possible.
-func TestBasicTransaction(t *testing.T) {
-	rsAPI := &testRoomserverAPI{}
-	pdus := []json.RawMessage{
-		testData[len(testData)-1], // a message event
-	}
-	txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus)
-	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
-}
-
-// The purpose of this test is to check that if the event received fails auth checks the event is still sent to the roomserver
-// as it does the auth check.
-func TestTransactionFailAuthChecks(t *testing.T) {
-	rsAPI := &testRoomserverAPI{}
-	pdus := []json.RawMessage{
-		testData[len(testData)-1], // a message event
-	}
-	txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus)
-	mustProcessTransaction(t, txn, []string{})
-	// expect message to be sent to the roomserver
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]})
-}
-
-// The purpose of this test is to make sure that when an event is received for which we do not know the prev_events,
-// we request them from /get_missing_events. It works by setting PrevEventsExist=false in the roomserver query response,
-// resulting in a call to /get_missing_events which returns the missing prev event. Both events should be processed in
-// topological order and sent to the roomserver.
-/*
-func TestTransactionFetchMissingPrevEvents(t *testing.T) {
-	haveEvent := testEvents[len(testEvents)-3]
-	prevEvent := testEvents[len(testEvents)-2]
-	inputEvent := testEvents[len(testEvents)-1]
-
-	var rsAPI *testRoomserverAPI // ref here so we can refer to inputRoomEvents inside these functions
-	rsAPI = &testRoomserverAPI{
-		queryEventsByID: func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse {
-			res := api.QueryEventsByIDResponse{}
-			for _, ev := range testEvents {
-				for _, id := range req.EventIDs {
-					if ev.EventID() == id {
-						res.Events = append(res.Events, ev)
-					}
-				}
-			}
-			return res
-		},
-		queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse {
-			return api.QueryStateAfterEventsResponse{
-				PrevEventsExist: true,
-				StateEvents:     testEvents[:5],
-			}
-		},
-		queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse {
-			missingPrevEvent := []string{"missing_prev_event"}
-			if len(req.PrevEventIDs) == 1 {
-				switch req.PrevEventIDs[0] {
-				case haveEvent.EventID():
-					missingPrevEvent = []string{}
-				case prevEvent.EventID():
-					// we only have this event if we've been send prevEvent
-					if len(rsAPI.inputRoomEvents) == 1 && rsAPI.inputRoomEvents[0].Event.EventID() == prevEvent.EventID() {
-						missingPrevEvent = []string{}
-					}
-				}
-			}
-
-			return api.QueryMissingAuthPrevEventsResponse{
-				RoomExists:          true,
-				MissingAuthEventIDs: []string{},
-				MissingPrevEventIDs: missingPrevEvent,
-			}
-		},
-		queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse {
-			return api.QueryLatestEventsAndStateResponse{
-				RoomExists: true,
-				Depth:      haveEvent.Depth(),
-				LatestEvents: []gomatrixserverlib.EventReference{
-					haveEvent.EventReference(),
-				},
-				StateEvents: fromStateTuples(req.StateToFetch, nil),
-			}
-		},
-	}
-
-	cli := &txnFedClient{
-		getMissingEvents: func(missing gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error) {
-			if !reflect.DeepEqual(missing.EarliestEvents, []string{haveEvent.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong earliest events: got %v want %v", missing.EarliestEvents, haveEvent.EventID())
-			}
-			if !reflect.DeepEqual(missing.LatestEvents, []string{inputEvent.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong latest events: got %v want %v", missing.LatestEvents, inputEvent.EventID())
-			}
-			return gomatrixserverlib.RespMissingEvents{
-				Events: []*gomatrixserverlib.Event{
-					prevEvent.Unwrap(),
-				},
-			}, nil
-		},
-	}
-
-	pdus := []json.RawMessage{
-		inputEvent.JSON(),
-	}
-	txn := mustCreateTransaction(rsAPI, cli, pdus)
-	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{prevEvent, inputEvent})
-}
-
-// The purpose of this test is to check that when there are missing prev_events and we still haven't been able to fill
-// in the hole with /get_missing_events that the state BEFORE the events we want to persist is fetched via /state_ids
-// and /event. It works by setting PrevEventsExist=false in the roomserver query response, resulting in
-// a call to /get_missing_events which returns 1 out of the 2 events it needs to fill in the gap. Synapse and Dendrite
-// both give up after 1x /get_missing_events call, relying on requesting the state AFTER the missing event in order to
-// continue. The DAG looks something like:
-// FE           GME    TXN
-//  A ---> B ---> C ---> D
-// TXN=event in the txn, GME=response to /get_missing_events, FE=roomserver's forward extremity. Should result in:
-// - /state_ids?event=B is requested, then /event/B to get the state AFTER B. B is a state event.
-// - state resolution is done to check C is allowed.
-// This results in B being sent as an outlier FIRST, then C,D.
-func TestTransactionFetchMissingStateByStateIDs(t *testing.T) {
-	eventA := testEvents[len(testEvents)-5]
-	// this is also len(testEvents)-4
-	eventB := testStateEvents[gomatrixserverlib.StateKeyTuple{
-		EventType: gomatrixserverlib.MRoomPowerLevels,
-		StateKey:  "",
-	}]
-	eventC := testEvents[len(testEvents)-3]
-	eventD := testEvents[len(testEvents)-2]
-	fmt.Println("a:", eventA.EventID())
-	fmt.Println("b:", eventB.EventID())
-	fmt.Println("c:", eventC.EventID())
-	fmt.Println("d:", eventD.EventID())
-	var rsAPI *testRoomserverAPI
-	rsAPI = &testRoomserverAPI{
-		queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse {
-			omitTuples := []gomatrixserverlib.StateKeyTuple{
-				{
-					EventType: gomatrixserverlib.MRoomPowerLevels,
-					StateKey:  "",
-				},
-			}
-			askingForEvent := req.PrevEventIDs[0]
-			haveEventB := false
-			haveEventC := false
-			for _, ev := range rsAPI.inputRoomEvents {
-				switch ev.Event.EventID() {
-				case eventB.EventID():
-					haveEventB = true
-					omitTuples = nil // include event B now
-				case eventC.EventID():
-					haveEventC = true
-				}
-			}
-			prevEventExists := false
-			if askingForEvent == eventC.EventID() {
-				prevEventExists = haveEventC
-			} else if askingForEvent == eventB.EventID() {
-				prevEventExists = haveEventB
-			}
-			var stateEvents []*gomatrixserverlib.HeaderedEvent
-			if prevEventExists {
-				stateEvents = fromStateTuples(req.StateToFetch, omitTuples)
-			}
-			return api.QueryStateAfterEventsResponse{
-				PrevEventsExist: prevEventExists,
-				RoomExists:      true,
-				StateEvents:     stateEvents,
-			}
-		},
-
-		queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse {
-			askingForEvent := req.PrevEventIDs[0]
-			haveEventB := false
-			haveEventC := false
-			for _, ev := range rsAPI.inputRoomEvents {
-				switch ev.Event.EventID() {
-				case eventB.EventID():
-					haveEventB = true
-				case eventC.EventID():
-					haveEventC = true
-				}
-			}
-			prevEventExists := false
-			if askingForEvent == eventC.EventID() {
-				prevEventExists = haveEventC
-			} else if askingForEvent == eventB.EventID() {
-				prevEventExists = haveEventB
-			}
-
-			var missingPrevEvent []string
-			if !prevEventExists {
-				missingPrevEvent = []string{"test"}
-			}
-
-			return api.QueryMissingAuthPrevEventsResponse{
-				RoomExists:          true,
-				MissingAuthEventIDs: []string{},
-				MissingPrevEventIDs: missingPrevEvent,
-			}
-		},
-
-		queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse {
-			omitTuples := []gomatrixserverlib.StateKeyTuple{
-				{EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""},
-			}
-			return api.QueryLatestEventsAndStateResponse{
-				RoomExists: true,
-				Depth:      eventA.Depth(),
-				LatestEvents: []gomatrixserverlib.EventReference{
-					eventA.EventReference(),
-				},
-				StateEvents: fromStateTuples(req.StateToFetch, omitTuples),
-			}
-		},
-		queryEventsByID: func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse {
-			var res api.QueryEventsByIDResponse
-			fmt.Println("queryEventsByID ", req.EventIDs)
-			for _, wantEventID := range req.EventIDs {
-				for _, ev := range testStateEvents {
-					// roomserver is missing the power levels event unless it's been sent to us recently as an outlier
-					if wantEventID == eventB.EventID() {
-						fmt.Println("Asked for pl event")
-						for _, inEv := range rsAPI.inputRoomEvents {
-							fmt.Println("recv ", inEv.Event.EventID())
-							if inEv.Event.EventID() == wantEventID {
-								res.Events = append(res.Events, inEv.Event)
-								break
-							}
-						}
-						continue
-					}
-					if ev.EventID() == wantEventID {
-						res.Events = append(res.Events, ev)
-					}
-				}
-			}
-			return res
-		},
-	}
-	// /state_ids for event B returns every state event but B (it's the state before)
-	var authEventIDs []string
-	var stateEventIDs []string
-	for _, ev := range testStateEvents {
-		if ev.EventID() == eventB.EventID() {
-			continue
-		}
-		// state res checks what auth events you give it, and this isn't a valid auth event
-		if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility {
-			authEventIDs = append(authEventIDs, ev.EventID())
-		}
-		stateEventIDs = append(stateEventIDs, ev.EventID())
-	}
-	cli := &txnFedClient{
-		stateIDs: map[string]gomatrixserverlib.RespStateIDs{
-			eventB.EventID(): {
-				StateEventIDs: stateEventIDs,
-				AuthEventIDs:  authEventIDs,
-			},
-		},
-		// /event for event B returns it
-		getEvent: map[string]gomatrixserverlib.Transaction{
-			eventB.EventID(): {
-				PDUs: []json.RawMessage{
-					eventB.JSON(),
-				},
-			},
-		},
-		// /get_missing_events should be done exactly once
-		getMissingEvents: func(missing gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error) {
-			if !reflect.DeepEqual(missing.EarliestEvents, []string{eventA.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong earliest events: got %v want %v", missing.EarliestEvents, eventA.EventID())
-			}
-			if !reflect.DeepEqual(missing.LatestEvents, []string{eventD.EventID()}) {
-				t.Errorf("call to /get_missing_events wrong latest events: got %v want %v", missing.LatestEvents, eventD.EventID())
-			}
-			// just return event C, not event B so /state_ids logic kicks in as there will STILL be missing prev_events
-			return gomatrixserverlib.RespMissingEvents{
-				Events: []*gomatrixserverlib.Event{
-					eventC.Unwrap(),
-				},
-			}, nil
-		},
-	}
-
-	pdus := []json.RawMessage{
-		eventD.JSON(),
-	}
-	txn := mustCreateTransaction(rsAPI, cli, pdus)
-	mustProcessTransaction(t, txn, nil)
-	assertInputRoomEvents(t, rsAPI.inputRoomEvents, []*gomatrixserverlib.HeaderedEvent{eventB, eventC, eventD})
-}
-*/

federationapi/statistics/statistics.go

@@ -1,6 +1,7 @@
 package statistics
 
 import (
+	"context"
 	"math"
 	"math/rand"
 	"sync"
@@ -28,25 +29,30 @@ type Statistics struct {
 	// just blacklist the host altogether? The backoff is exponential,
 	// so the max time here to attempt is 2**failures seconds.
 	FailuresUntilBlacklist uint32
+
+	// How many times should we tolerate consecutive failures before we
+	// mark the destination as offline. At this point we should attempt
+	// to send messages to the user's async relay servers if we know them.
+	FailuresUntilAssumedOffline uint32
 }
 
-func NewStatistics(db storage.Database, failuresUntilBlacklist uint32) Statistics {
+func NewStatistics(
+	db storage.Database,
+	failuresUntilBlacklist uint32,
+	failuresUntilAssumedOffline uint32,
+) Statistics {
 	return Statistics{
-		DB:                     db,
-		FailuresUntilBlacklist: failuresUntilBlacklist,
-		backoffTimers:          make(map[gomatrixserverlib.ServerName]*time.Timer),
+		DB:                          db,
+		FailuresUntilBlacklist:      failuresUntilBlacklist,
+		FailuresUntilAssumedOffline: failuresUntilAssumedOffline,
+		backoffTimers:               make(map[gomatrixserverlib.ServerName]*time.Timer),
+		servers:                     make(map[gomatrixserverlib.ServerName]*ServerStatistics),
 	}
 }
 
 // ForServer returns server statistics for the given server name. If it
 // does not exist, it will create empty statistics and return those.
 func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerStatistics {
-	// If the map hasn't been initialised yet then do that.
-	if s.servers == nil {
-		s.mutex.Lock()
-		s.servers = make(map[gomatrixserverlib.ServerName]*ServerStatistics)
-		s.mutex.Unlock()
-	}
 	// Look up if we have statistics for this server already.
 	s.mutex.RLock()
 	server, found := s.servers[serverName]
@@ -55,8 +61,9 @@ func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerS
 	if !found {
 		s.mutex.Lock()
 		server = &ServerStatistics{
-			statistics: s,
-			serverName: serverName,
+			statistics:        s,
+			serverName:        serverName,
+			knownRelayServers: []gomatrixserverlib.ServerName{},
 		}
 		s.servers[serverName] = server
 		s.mutex.Unlock()
@@ -66,24 +73,49 @@ func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerS
 		} else {
 			server.blacklisted.Store(blacklisted)
 		}
+		assumedOffline, err := s.DB.IsServerAssumedOffline(context.Background(), serverName)
+		if err != nil {
+			logrus.WithError(err).Errorf("Failed to get assumed offline entry %q", serverName)
+		} else {
+			server.assumedOffline.Store(assumedOffline)
+		}
+
+		knownRelayServers, err := s.DB.P2PGetRelayServersForServer(context.Background(), serverName)
+		if err != nil {
+			logrus.WithError(err).Errorf("Failed to get relay server list for %q", serverName)
+		} else {
+			server.relayMutex.Lock()
+			server.knownRelayServers = knownRelayServers
+			server.relayMutex.Unlock()
+		}
 	}
 	return server
 }
 
+type SendMethod uint8
+
+const (
+	SendDirect SendMethod = iota
+	SendViaRelay
+)
+
 // ServerStatistics contains information about our interactions with a
 // remote federated host, e.g. how many times we were successful, how
 // many times we failed etc. It also manages the backoff time and black-
 // listing a remote host if it remains uncooperative.
 type ServerStatistics struct {
-	statistics      *Statistics                  //
-	serverName      gomatrixserverlib.ServerName //
-	blacklisted     atomic.Bool                  // is the node blacklisted
-	backoffStarted  atomic.Bool                  // is the backoff started
-	backoffUntil    atomic.Value                 // time.Time until this backoff interval ends
-	backoffCount    atomic.Uint32                // number of times BackoffDuration has been called
-	successCounter  atomic.Uint32                // how many times have we succeeded?
-	backoffNotifier func()                       // notifies destination queue when backoff completes
-	notifierMutex   sync.Mutex
+	statistics        *Statistics                  //
+	serverName        gomatrixserverlib.ServerName //
+	blacklisted       atomic.Bool                  // is the node blacklisted
+	assumedOffline    atomic.Bool                  // is the node assumed to be offline
+	backoffStarted    atomic.Bool                  // is the backoff started
+	backoffUntil      atomic.Value                 // time.Time until this backoff interval ends
+	backoffCount      atomic.Uint32                // number of times BackoffDuration has been called
+	successCounter    atomic.Uint32                // how many times have we succeeded?
+	backoffNotifier   func()                       // notifies destination queue when backoff completes
+	notifierMutex     sync.Mutex
+	knownRelayServers []gomatrixserverlib.ServerName
+	relayMutex        sync.Mutex
 }
 
 const maxJitterMultiplier = 1.4
@@ -118,14 +150,22 @@ func (s *ServerStatistics) AssignBackoffNotifier(notifier func()) {
 // attempt, which increases the sent counter and resets the idle and
 // failure counters. If a host was blacklisted at this point then
 // we will unblacklist it.
-func (s *ServerStatistics) Success() {
+// `relay` specifies whether the success was to the actual destination
+// or one of their relay servers.
+func (s *ServerStatistics) Success(method SendMethod) {
 	s.cancel()
 	s.backoffCount.Store(0)
-	s.successCounter.Inc()
-	if s.statistics.DB != nil {
-		if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil {
-			logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName)
+	// NOTE : Sending to the final destination vs. a relay server has
+	// slightly different semantics.
+	if method == SendDirect {
+		s.successCounter.Inc()
+		if s.blacklisted.Load() && s.statistics.DB != nil {
+			if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil {
+				logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName)
+			}
 		}
+
+		s.removeAssumedOffline()
 	}
 }
 
@@ -144,7 +184,18 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 	// start a goroutine which will wait out the backoff and
 	// unset the backoffStarted flag when done.
 	if s.backoffStarted.CompareAndSwap(false, true) {
-		if s.backoffCount.Inc() >= s.statistics.FailuresUntilBlacklist {
+		backoffCount := s.backoffCount.Inc()
+
+		if backoffCount >= s.statistics.FailuresUntilAssumedOffline {
+			s.assumedOffline.CompareAndSwap(false, true)
+			if s.statistics.DB != nil {
+				if err := s.statistics.DB.SetServerAssumedOffline(context.Background(), s.serverName); err != nil {
+					logrus.WithError(err).Errorf("Failed to set %q as assumed offline", s.serverName)
+				}
+			}
+		}
+
+		if backoffCount >= s.statistics.FailuresUntilBlacklist {
 			s.blacklisted.Store(true)
 			if s.statistics.DB != nil {
 				if err := s.statistics.DB.AddServerToBlacklist(s.serverName); err != nil {
@@ -162,13 +213,21 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 		s.backoffUntil.Store(until)
 
 		s.statistics.backoffMutex.Lock()
-		defer s.statistics.backoffMutex.Unlock()
 		s.statistics.backoffTimers[s.serverName] = time.AfterFunc(time.Until(until), s.backoffFinished)
+		s.statistics.backoffMutex.Unlock()
 	}
 
 	return s.backoffUntil.Load().(time.Time), false
 }
 
+// MarkServerAlive removes the assumed offline and blacklisted statuses from this server.
+// Returns whether the server was blacklisted before this point.
+func (s *ServerStatistics) MarkServerAlive() bool {
+	s.removeAssumedOffline()
+	wasBlacklisted := s.removeBlacklist()
+	return wasBlacklisted
+}
+
 // ClearBackoff stops the backoff timer for this destination if it is running
 // and removes the timer from the backoffTimers map.
 func (s *ServerStatistics) ClearBackoff() {
@@ -196,13 +255,13 @@ func (s *ServerStatistics) backoffFinished() {
 }
 
 // BackoffInfo returns information about the current or previous backoff.
-// Returns the last backoffUntil time and whether the server is currently blacklisted or not.
-func (s *ServerStatistics) BackoffInfo() (*time.Time, bool) {
+// Returns the last backoffUntil time.
+func (s *ServerStatistics) BackoffInfo() *time.Time {
 	until, ok := s.backoffUntil.Load().(time.Time)
 	if ok {
-		return &until, s.blacklisted.Load()
+		return &until
 	}
-	return nil, s.blacklisted.Load()
+	return nil
 }
 
 // Blacklisted returns true if the server is blacklisted and false
@@ -211,10 +270,33 @@ func (s *ServerStatistics) Blacklisted() bool {
 	return s.blacklisted.Load()
 }
 
-// RemoveBlacklist removes the blacklisted status from the server.
-func (s *ServerStatistics) RemoveBlacklist() {
+// AssumedOffline returns true if the server is assumed offline and false
+// otherwise.
+func (s *ServerStatistics) AssumedOffline() bool {
+	return s.assumedOffline.Load()
+}
+
+// removeBlacklist removes the blacklisted status from the server.
+// Returns whether the server was blacklisted.
+func (s *ServerStatistics) removeBlacklist() bool {
+	var wasBlacklisted bool
+
+	if s.Blacklisted() {
+		wasBlacklisted = true
+		_ = s.statistics.DB.RemoveServerFromBlacklist(s.serverName)
+	}
 	s.cancel()
 	s.backoffCount.Store(0)
+
+	return wasBlacklisted
+}
+
+// removeAssumedOffline removes the assumed offline status from the server.
+func (s *ServerStatistics) removeAssumedOffline() {
+	if s.AssumedOffline() {
+		_ = s.statistics.DB.RemoveServerAssumedOffline(context.Background(), s.serverName)
+	}
+	s.assumedOffline.Store(false)
 }
 
 // SuccessCount returns the number of successful requests. This is
@@ -222,3 +304,46 @@ func (s *ServerStatistics) RemoveBlacklist() {
 func (s *ServerStatistics) SuccessCount() uint32 {
 	return s.successCounter.Load()
 }
+
+// KnownRelayServers returns the list of relay servers associated with this
+// server.
+func (s *ServerStatistics) KnownRelayServers() []gomatrixserverlib.ServerName {
+	s.relayMutex.Lock()
+	defer s.relayMutex.Unlock()
+	return s.knownRelayServers
+}
+
+func (s *ServerStatistics) AddRelayServers(relayServers []gomatrixserverlib.ServerName) {
+	seenSet := make(map[gomatrixserverlib.ServerName]bool)
+	uniqueList := []gomatrixserverlib.ServerName{}
+	for _, srv := range relayServers {
+		if seenSet[srv] {
+			continue
+		}
+		seenSet[srv] = true
+		uniqueList = append(uniqueList, srv)
+	}
+
+	err := s.statistics.DB.P2PAddRelayServersForServer(context.Background(), s.serverName, uniqueList)
+	if err != nil {
+		logrus.WithError(err).Errorf("Failed to add relay servers for %q. Servers: %v", s.serverName, uniqueList)
+		return
+	}
+
+	for _, newServer := range uniqueList {
+		alreadyKnown := false
+		knownRelayServers := s.KnownRelayServers()
+		for _, srv := range knownRelayServers {
+			if srv == newServer {
+				alreadyKnown = true
+			}
+		}
+		if !alreadyKnown {
+			{
+				s.relayMutex.Lock()
+				s.knownRelayServers = append(s.knownRelayServers, newServer)
+				s.relayMutex.Unlock()
+			}
+		}
+	}
+}

federationapi/statistics/statistics_test.go

@@ -4,17 +4,26 @@ import (
 	"math"
 	"testing"
 	"time"
+
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	FailuresUntilAssumedOffline = 3
+	FailuresUntilBlacklist      = 8
 )
 
 func TestBackoff(t *testing.T) {
-	stats := NewStatistics(nil, 7)
+	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
 	server := ServerStatistics{
 		statistics: &stats,
 		serverName: "test.com",
 	}
 
 	// Start by checking that counting successes works.
-	server.Success()
+	server.Success(SendDirect)
 	if successes := server.SuccessCount(); successes != 1 {
 		t.Fatalf("Expected success count 1, got %d", successes)
 	}
@@ -31,9 +40,8 @@ func TestBackoff(t *testing.T) {
 		// side effects since a backoff is already in progress. If it does
 		// then we'll fail.
 		until, blacklisted := server.Failure()
-
-		// Get the duration.
-		_, blacklist := server.BackoffInfo()
+		blacklist := server.Blacklisted()
+		assumedOffline := server.AssumedOffline()
 		duration := time.Until(until)
 
 		// Unset the backoff, or otherwise our next call will think that
@@ -41,16 +49,43 @@ func TestBackoff(t *testing.T) {
 		server.cancel()
 		server.backoffStarted.Store(false)
 
+		if i >= stats.FailuresUntilAssumedOffline {
+			if !assumedOffline {
+				t.Fatalf("Backoff %d should have resulted in assuming the destination was offline but didn't", i)
+			}
+		}
+
+		// Check if we should be assumed offline by now.
+		if i >= stats.FailuresUntilAssumedOffline {
+			if !assumedOffline {
+				t.Fatalf("Backoff %d should have resulted in assumed offline but didn't", i)
+			} else {
+				t.Logf("Backoff %d is assumed offline as expected", i)
+			}
+		} else {
+			if assumedOffline {
+				t.Fatalf("Backoff %d should not have resulted in assumed offline but did", i)
+			} else {
+				t.Logf("Backoff %d is not assumed offline as expected", i)
+			}
+		}
+
 		// Check if we should be blacklisted by now.
 		if i >= stats.FailuresUntilBlacklist {
 			if !blacklist {
 				t.Fatalf("Backoff %d should have resulted in blacklist but didn't", i)
 			} else if blacklist != blacklisted {
-				t.Fatalf("BackoffInfo and Failure returned different blacklist values")
+				t.Fatalf("Blacklisted and Failure returned different blacklist values")
 			} else {
 				t.Logf("Backoff %d is blacklisted as expected", i)
 				continue
 			}
+		} else {
+			if blacklist {
+				t.Fatalf("Backoff %d should not have resulted in blacklist but did", i)
+			} else {
+				t.Logf("Backoff %d is not blacklisted as expected", i)
+			}
 		}
 
 		// Check if the duration is what we expect.
@@ -69,3 +104,14 @@ func TestBackoff(t *testing.T) {
 		}
 	}
 }
+
+func TestRelayServersListing(t *testing.T) {
+	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	server := ServerStatistics{statistics: &stats}
+	server.AddRelayServers([]gomatrixserverlib.ServerName{"relay1", "relay1", "relay2"})
+	relayServers := server.KnownRelayServers()
+	assert.Equal(t, []gomatrixserverlib.ServerName{"relay1", "relay2"}, relayServers)
+	server.AddRelayServers([]gomatrixserverlib.ServerName{"relay1", "relay1", "relay2"})
+	relayServers = server.KnownRelayServers()
+	assert.Equal(t, []gomatrixserverlib.ServerName{"relay1", "relay2"}, relayServers)
+}

federationapi/storage/interface.go

@@ -20,11 +20,12 @@ import (
 
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/federationapi/storage/shared"
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
 	"github.com/matrix-org/dendrite/federationapi/types"
 )
 
 type Database interface {
+	P2PDatabase
 	gomatrixserverlib.KeyDatabase
 
 	UpdateRoom(ctx context.Context, roomID string, addHosts []types.JoinedHost, removeHosts []string, purgeRoomFirst bool) (joinedHosts []types.JoinedHost, err error)
@@ -34,16 +35,16 @@ type Database interface {
 	// GetJoinedHostsForRooms returns the complete set of servers in the rooms given.
 	GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, excludeSelf, excludeBlacklisted bool) ([]gomatrixserverlib.ServerName, error)
 
-	StoreJSON(ctx context.Context, js string) (*shared.Receipt, error)
+	StoreJSON(ctx context.Context, js string) (*receipt.Receipt, error)
 
-	GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error)
-	GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error)
+	GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*receipt.Receipt]*gomatrixserverlib.HeaderedEvent, err error)
+	GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*receipt.Receipt]*gomatrixserverlib.EDU, err error)
 
-	AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt) error
-	AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error
+	AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, dbReceipt *receipt.Receipt) error
+	AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, dbReceipt *receipt.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error
 
-	CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
-	CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error
+	CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*receipt.Receipt) error
+	CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*receipt.Receipt) error
 
 	GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
 	GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error)
@@ -54,6 +55,18 @@ type Database interface {
 	RemoveAllServersFromBlacklist() error
 	IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error)
 
+	// Adds the server to the list of assumed offline servers.
+	// If the server already exists in the table, nothing happens and returns success.
+	SetServerAssumedOffline(ctx context.Context, serverName gomatrixserverlib.ServerName) error
+	// Removes the server from the list of assumed offline servers.
+	// If the server doesn't exist in the table, nothing happens and returns success.
+	RemoveServerAssumedOffline(ctx context.Context, serverName gomatrixserverlib.ServerName) error
+	// Purges all entries from the assumed offline table.
+	RemoveAllServersAssumedOffline(ctx context.Context) error
+	// Gets whether the provided server is present in the table.
+	// If it is present, returns true. If not, returns false.
+	IsServerAssumedOffline(ctx context.Context, serverName gomatrixserverlib.ServerName) (bool, error)
+
 	AddOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error
 	RenewOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error
 	GetOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string) (*types.OutboundPeek, error)
@@ -71,4 +84,24 @@ type Database interface {
 	GetNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, optKeyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error)
 	// DeleteExpiredEDUs cleans up expired EDUs
 	DeleteExpiredEDUs(ctx context.Context) error
+
+	PurgeRoom(ctx context.Context, roomID string) error
+}
+
+type P2PDatabase interface {
+	// Stores the given list of servers as relay servers for the provided destination server.
+	// Providing duplicates will only lead to a single entry and won't lead to an error.
+	P2PAddRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName, relayServers []gomatrixserverlib.ServerName) error
+
+	// Get the list of relay servers associated with the provided destination server.
+	// If no entry exists in the table, an empty list is returned and does not result in an error.
+	P2PGetRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName) ([]gomatrixserverlib.ServerName, error)
+
+	// Deletes any entries for the provided destination server that match the provided relayServers list.
+	// If any of the provided servers don't match an entry, nothing happens and no error is returned.
+	P2PRemoveRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName, relayServers []gomatrixserverlib.ServerName) error
+
+	// Deletes all entries for the provided destination server.
+	// If the destination server doesn't exist in the table, nothing happens and no error is returned.
+	P2PRemoveAllRelayServersForServer(ctx context.Context, serverName gomatrixserverlib.ServerName) error
 }

federationapi/storage/postgres/assumed_offline_table.go

@@ -0,0 +1,107 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const assumedOfflineSchema = `
+CREATE TABLE IF NOT EXISTS federationsender_assumed_offline(
+    -- The assumed offline server name
+	server_name TEXT PRIMARY KEY NOT NULL
+);
+`
+
+const insertAssumedOfflineSQL = "" +
+	"INSERT INTO federationsender_assumed_offline (server_name) VALUES ($1)" +
+	" ON CONFLICT DO NOTHING"
+
+const selectAssumedOfflineSQL = "" +
+	"SELECT server_name FROM federationsender_assumed_offline WHERE server_name = $1"
+
+const deleteAssumedOfflineSQL = "" +
+	"DELETE FROM federationsender_assumed_offline WHERE server_name = $1"
+
+const deleteAllAssumedOfflineSQL = "" +
+	"TRUNCATE federationsender_assumed_offline"
+
+type assumedOfflineStatements struct {
+	db                          *sql.DB
+	insertAssumedOfflineStmt    *sql.Stmt
+	selectAssumedOfflineStmt    *sql.Stmt
+	deleteAssumedOfflineStmt    *sql.Stmt
+	deleteAllAssumedOfflineStmt *sql.Stmt
+}
+
+func NewPostgresAssumedOfflineTable(db *sql.DB) (s *assumedOfflineStatements, err error) {
+	s = &assumedOfflineStatements{
+		db: db,
+	}
+	_, err = db.Exec(assumedOfflineSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertAssumedOfflineStmt, insertAssumedOfflineSQL},
+		{&s.selectAssumedOfflineStmt, selectAssumedOfflineSQL},
+		{&s.deleteAssumedOfflineStmt, deleteAssumedOfflineSQL},
+		{&s.deleteAllAssumedOfflineStmt, deleteAllAssumedOfflineSQL},
+	}.Prepare(db)
+}
+
+func (s *assumedOfflineStatements) InsertAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.insertAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx, serverName)
+	return err
+}
+
+func (s *assumedOfflineStatements) SelectAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) (bool, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectAssumedOfflineStmt)
+	res, err := stmt.QueryContext(ctx, serverName)
+	if err != nil {
+		return false, err
+	}
+	defer res.Close() // nolint:errcheck
+	// The query will return the server name if the server is assume offline, and
+	// will return no rows if not. By calling Next, we find out if a row was
+	// returned or not - we don't care about the value itself.
+	return res.Next(), nil
+}
+
+func (s *assumedOfflineStatements) DeleteAssumedOffline(
+	ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx, serverName)
+	return err
+}
+
+func (s *assumedOfflineStatements) DeleteAllAssumedOffline(
+	ctx context.Context, txn *sql.Tx,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteAllAssumedOfflineStmt)
+	_, err := stmt.ExecContext(ctx)
+	return err
+}