Forgejo does not support build cache

https://github.com/matrix-org/sytest/pull/1383 updates some sytests in
line with MSC3967. Dendrite does not support MSC3967, so these tests
fail.

.forgejo/workflows/docker.yml

@@ -0,0 +1,59 @@
+on:
+  push:
+    tags:
+      - 'v*'
+
+env:
+  GHCR_NAMESPACE: sigb.us
+  PLATFORMS: linux/amd64
+  FORGEJO_USER: signaryk
+
+jobs:
+  monolith:
+    name: Monolith image
+    runs-on: docker
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get release tag & build flags
+        if: github.event_name == 'release' # Only for GitHub releases
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to sigb.us container registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.sigb.us
+          username: ${{ env.FORGEJO_USER }}
+          password: ${{ secrets.FORGEJO_TOKEN }}
+
+      - name: Build main monolith image
+        id: docker_build_monolith
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ github.ref_name }}
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:devel
+
+      - name: Build release monolith image
+        if: github.event_name == 'release' # Only for GitHub releases
+        id: docker_build_monolith_release
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: |
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:stable
+            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ env.RELEASE_VERSION }}

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include
 those (please be careful to remove any personal or private data). Please surround them with
 ``` (three backticks, on a line on their own), so that they are formatted legibly.
 
-Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org
+Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host
 with a link to the respective Github issue, thanks!
 -->

.github/codecov.yaml

@@ -7,7 +7,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0%
+        threshold: 0.1%
         base: auto
         flags:
           - unittests

.github/workflows/dendrite.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ false }} # disable for now
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
@@ -41,7 +41,7 @@ jobs:
         with:
           node-version: 14
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -66,11 +66,11 @@ jobs:
     name: Linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: golangci-lint
@@ -102,14 +102,14 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
         with:
           path: |
@@ -141,12 +141,12 @@ jobs:
         goos: ["linux"]
         goarch: ["amd64", "386"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -174,12 +174,12 @@ jobs:
         goos: ["windows"]
         goarch: ["amd64"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -235,11 +235,11 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
       - name: Set up gotestfmt
@@ -247,7 +247,7 @@ jobs:
         with:
           # Optional: pass GITHUB_TOKEN to avoid rate limiting.
           token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -262,10 +262,11 @@ jobs:
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: dendrite
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           flags: unittests
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run database upgrade tests
   upgrade_test:
@@ -274,12 +275,20 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-test-
       - name: Docker version
         run: docker version
       - name: Build upgrade-tests
@@ -296,12 +305,20 @@ jobs:
     needs: initial-tests-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "stable"
           cache: true
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-upgrade-direct-test-
       - name: Docker version
         run: docker version
       - name: Build upgrade-tests
@@ -340,8 +357,8 @@ jobs:
         SYTEST_BRANCH: ${{ github.head_ref }}
         CGO_ENABLED: ${{ matrix.cgo && 1 }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -364,7 +381,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
@@ -404,8 +421,8 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v4 for dendrite
+        uses: actions/checkout@v4
         with:
           path: dendrite
 
@@ -440,7 +457,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt -hide all
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:

.github/workflows/docker.yml

@@ -27,26 +27,22 @@ jobs:
       security-events: write # To upload Trivy sarif files
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -60,7 +56,6 @@ jobs:
           cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache
           cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -75,7 +70,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}
           push: true
           tags: |
@@ -104,26 +98,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -137,7 +127,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -153,7 +142,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-pinecone
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -171,26 +159,22 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get release tag & build flags
         if: github.event_name == 'release' # Only for GitHub releases
         run: |
           echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
-          BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
-          [ ${BRANCH} == "main" ] && BRANCH=""
-          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Login to GitHub Containers
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -204,7 +188,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -220,7 +203,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: .
-          build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           file: ./build/docker/Dockerfile.demo-yggdrasil
           platforms: ${{ env.PLATFORMS }}
           push: true

.github/workflows/gh-pages.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Build with Jekyll

.github/workflows/helm.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/k8s.yml

@@ -17,7 +17,7 @@ jobs:
     outputs:
       changed: ${{ steps.list-changed.outputs.changed }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: azure/setup-helm@v3
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ inputs.checkoutCommit }}
@@ -66,7 +66,7 @@ jobs:
       - name: Create k3d cluster
         uses: nolar/setup-k3d-k3s@v1
         with:
-          version: v1.21
+          version: v1.28
       - name: Remove node taints
         run: |
           kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true

.github/workflows/schedules.yaml

@@ -10,8 +10,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
+    runs-on: ubuntu-latest
+    name: Check latest commit
+    outputs:
+      should_run: ${{ steps.should_run.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: print latest_commit
+        run: echo ${{ github.sha }}
+
+      - id: should_run
+        continue-on-error: true
+        name: check latest commit is less than a day
+        if: ${{ github.event_name == 'schedule' }}
+        run: test -z $(git rev-list  --after="24 hours"  ${{ github.sha }}) && echo "::set-output name=should_run::false"
+
   # run Sytest in different variations
   sytest:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     timeout-minutes: 60
     name: "Sytest (${{ matrix.label }})"
     runs-on: ubuntu-latest
@@ -38,8 +56,8 @@ jobs:
         RACE_DETECTION: 1
         COVER: 1
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -62,7 +80,7 @@ jobs:
         run: /src/are-we-synapse-yet.py /logs/results.tap -v
         continue-on-error: true # not fatal
       - name: Upload Sytest logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -75,31 +93,34 @@ jobs:
     timeout-minutes: 5
     name: "Sytest Coverage"
     runs-on: ubuntu-latest
-    needs: sytest # only run once Sytest is done
-    if: ${{ always() }}
+    needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
           go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: sytest
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # run Complement
   complement:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
     name: "Complement (${{ matrix.label }})"
     timeout-minutes: 60
     runs-on: ubuntu-latest
@@ -129,8 +150,8 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-      - name: Run actions/checkout@v3 for dendrite
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v4 for dendrite
+        uses: actions/checkout@v4
         with:
           path: dendrite
 
@@ -174,7 +195,7 @@ jobs:
       # Run Complement
       - run: |
           set -o pipefail &&
-          go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
+          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
         shell: bash
         name: Run Complement Tests
         env:
@@ -185,7 +206,7 @@ jobs:
         working-directory: complement
 
       - name: Upload Complement logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
@@ -196,30 +217,32 @@ jobs:
     timeout-minutes: 5
     name: "Complement Coverage"
     runs-on: ubuntu-latest
-    needs: complement # only run once Complement is done
-    if: ${{ always() }}
+    needs: [ complement, check_date ] # only run once Complements is done and there was a commit
+    if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 'stable'
           cache: true
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Collect coverage
         run: |
           go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
           grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
           go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./final.cov
           flags: complement
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }} # required
 
   element-web:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -228,7 +251,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3
@@ -259,6 +282,7 @@ jobs:
           TMPDIR: ${{ runner.temp }}
 
   element-web-pinecone:
+    if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
     timeout-minutes: 120
     runs-on: ubuntu-latest
     steps:
@@ -267,7 +291,7 @@ jobs:
           # Our test suite includes some screenshot tests with unusual diacritics, which are
           # supposed to be covered by STIXGeneral.
           tools: fonts-stix
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           repository: matrix-org/matrix-react-sdk
       - uses: actions/setup-node@v3

.gitignore

@@ -5,6 +5,7 @@
 
 # Allow GitHub config
 !.github
+!.forgejo
 
 # Downloads
 /.downloads
@@ -78,3 +79,6 @@ build
 
 # golang workspaces
 go.work*
+
+# helm chart
+helm/dendrite/charts/

.golangci.yml

@@ -6,7 +6,7 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 30m
+  timeout: 5m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
@@ -18,24 +18,6 @@ run:
   #build-tags:
   #  - mytag
 
-  # which dirs to skip: they won't be analyzed;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but next dirs are always skipped independently
-  # from this option's value:
-  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
-    - bin
-    - docs
-
-  # which files to skip: they will be analyzed, but issues from them
-  # won't be reported. Default value is empty list, but there is
-  # no need to include all autogenerated files, we confidently recognize
-  # autogenerated files. If it's not please let us know.
-  skip-files:
-    - ".*\\.md$"
-    - ".*\\.sh$"
-    - "^cmd/syncserver-integration-tests/testdata.go$"
-
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit
   # automatic updating of go.mod described above. Instead, it fails when any changes
@@ -50,7 +32,8 @@ run:
 # output configuration options
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
-  format: colored-line-number
+  formats:
+    - format: colored-line-number
 
   # print lines of code with issue, default is true
   print-issued-lines: true
@@ -79,9 +62,8 @@ linters-settings:
     # see https://github.com/kisielk/errcheck#excluding-functions for details
     #exclude: /path/to/file.txt
   govet:
-    # report about shadowed variables
-    check-shadowing: true
-
+    enable:
+      - shadow
     # settings per analyzer
     settings:
       printf: # analyzer name, run `go tool vet help` to see all analyzers
@@ -180,7 +162,6 @@ linters-settings:
 linters:
   enable:
     - errcheck
-    - goconst
     - gocyclo
     - goimports # Does everything gofmt does
     - gosimple
@@ -211,12 +192,31 @@ linters:
     - stylecheck
     - typecheck # Should turn back on soon
     - unconvert # Should turn back on soon
+    - goconst # Slightly annoying, as it reports "issues" in SQL statements
   disable-all: false
   presets:
   fast: false
 
 
 issues:
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  exclude-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  exclude-dirs:
+    - bin
+    - docs
+
   # List of regexps of issue texts to exclude, empty list by default.
   # But independently from this option we use default exclude patterns,
   # it can be disabled by `exclude-use-default: false`. To list all

CHANGES.md

@@ -1,5 +1,85 @@
 # Changelog
 
+## Dendrite 0.13.7 (2024-04-09)
+
+### Fixes
+
+- Fixed an issue where the displayname/avatar of an invited user was replaced with the inviter's details
+- Improved server startup performance by avoiding unnecessary room ACL queries
+  - This change reduces memory footprint as it caches ACL regex patterns once instead of for each room
+  - Unnecessary Relay related queries have been removed. **Note**: To use relays, you now need to explicitly enable them using the `federation_api.enable_relays` config
+- Fixed space summaries over federation
+- Improved usage of external NATS JetStream by reusing existing connections instead of opening new ones unnecessarily
+
+### Features
+
+- Modernized Appservices (contributed by [tulir](https://github.com/tulir))
+- Added event reporting with Synapse Admin endpoints for querying them
+- Updated dependencies
+
+## Dendrite 0.13.6 (2024-01-26)
+
+Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
+
+### Fixes
+
+- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS
+- A rare panic when assigning a state key NID has been fixed
+- A rare panic when checking powerlevels has been fixed
+- Notary keys requests for all keys now work correctly
+- Spec compliance:
+  - Return `M_INVALID_PARAM` when querying room aliases
+  - Handle empty `from` parameter when requesting `/messages`
+  - Add CORP headers on media endpoints
+  - Remove `aliases` from `/publicRooms` responses
+  - Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss))
+- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms
+- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht))
+- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again
+- Backfilling from other servers now only uses at max 100 events instead of potentially thousands
+
+## Dendrite 0.13.5 (2023-12-12)
+
+Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in
+our CanonicalJSON implementation.
+
+### Fixes
+
+- Convert unicode escapes to lowercase (gomatrixserverlib)
+- Fix canonical json utf-16 surrogate pair detection logic (gomatrixserverlib)
+- Handle negative zero and exponential numbers in Canonical JSON verification (gomatrixserverlib)
+- Avoid logging unnecessary messages when unable to fetch server keys if multiple fetchers are used (gomatrixserverlib)
+- Issues around the device list updater have been fixed, which should ensure that there are always
+  workers available to process incoming device list updates.
+- A panic in the `/hierarchy` endpoints used for spaces has been fixed (client-server and server-server API)
+- Fixes around the way we handle database transactions (including a potential connection leak)
+- ACLs are now updated when received as outliers
+- A race condition, which could lead to bridges instantly leaving a room after joining it, between the SyncAPI and
+  Appservices has been fixed
+
+### Features
+
+- **Appservice login is now supported!**
+- Users can now kick themselves (used by some bridges)
+
+## Dendrite 0.13.4 (2023-10-25)
+
+Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution
+algorithm.
+
+### Fixes:
+
+- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also 
+  avoids spamming the logs when shutting down.)
+- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset"
+  out of rooms and other missing state events due to calculating the wrong state.
+- A bug when setting notifications from Element Android has been fixed by implementing MSC3987
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.19 to v2.9.23
+
 ## Dendrite 0.13.3 (2023-09-28)
 
 ### Fixes:

Dockerfile

@@ -3,7 +3,8 @@
 #
 # base installs required dependencies and runs go mod download to cache dependencies
 #
-FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine3.18 AS base
 RUN apk --update --no-cache add bash build-base curl git
 
 #
@@ -13,7 +14,6 @@ FROM --platform=${BUILDPLATFORM} base AS build
 WORKDIR /src
 ARG TARGETOS
 ARG TARGETARCH
-ARG FLAGS
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
@@ -21,7 +21,7 @@ RUN --mount=target=. \
     GOARCH="$TARGETARCH" \
     GOOS="linux" \
     CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
-    go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
+    go build -v -trimpath -o /out/ ./cmd/...
 
 
 #

appservice/api/query.go

@@ -82,9 +82,17 @@ type UserIDExistsResponse struct {
 }
 
 const (
-	ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
-	ASUserPath     = "/_matrix/app/unstable/thirdparty/user"
-	ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
+	ASProtocolLegacyPath        = "/_matrix/app/unstable/thirdparty/protocol/"
+	ASUserLegacyPath            = "/_matrix/app/unstable/thirdparty/user"
+	ASLocationLegacyPath        = "/_matrix/app/unstable/thirdparty/location"
+	ASRoomAliasExistsLegacyPath = "/rooms/"
+	ASUserExistsLegacyPath      = "/users/"
+
+	ASProtocolPath        = "/_matrix/app/v1/thirdparty/protocol/"
+	ASUserPath            = "/_matrix/app/v1/thirdparty/user"
+	ASLocationPath        = "/_matrix/app/v1/thirdparty/location"
+	ASRoomAliasExistsPath = "/_matrix/app/v1/rooms/"
+	ASUserExistsPath      = "/_matrix/app/v1/users/"
 )
 
 type ProtocolRequest struct {

appservice/appservice_test.go

@@ -14,7 +14,18 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/dendrite/syncapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/appservice/api"
@@ -32,6 +43,10 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestAppserviceInternalAPI(t *testing.T) {
 
 	// Set expected results
@@ -144,7 +159,7 @@ func TestAppserviceInternalAPI(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+		usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 		runCases(t, asAPI)
@@ -239,7 +254,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) {
 	cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
 	rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
-	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil)
+	usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI)
 
 	t.Run("UserIDExists", func(t *testing.T) {
@@ -378,7 +393,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		// Create required internal APIs
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// start the consumer
 		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
 
@@ -402,3 +417,190 @@ func TestRoomserverConsumerOneInvite(t *testing.T) {
 		close(evChan)
 	})
 }
+
+// Note: If this test panics, it is because we timed out waiting for the
+// join event to come through to the appservice and we close the DB/shutdown Dendrite. This makes the
+// syncAPI unhappy, as it is unable to write to the database.
+func TestOutputAppserviceEvent(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
+		defer closeDB()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		natsInstance := &jetstream.NATSInstance{}
+
+		evChan := make(chan struct{})
+
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		// Create required internal APIs
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+
+		// Create the router, so we can hit `/joined_members`
+		routers := httputil.NewRouters()
+
+		accessTokens := map[*test.User]userDevice{
+			bob: {},
+		}
+
+		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+		clientapi.AddPublicRoutes(processCtx, routers, cfg, natsInstance, nil, rsAPI, nil, nil, nil, usrAPI, nil, nil, caching.DisableMetrics)
+		createAccessTokens(t, accessTokens, usrAPI, processCtx.Context(), routers)
+
+		room := test.NewRoom(t, alice)
+
+		// Invite Bob
+		room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+			"membership": "invite",
+		}, test.WithStateKey(bob.ID))
+
+		// create a dummy AS url, handling the events
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var txn consumers.ApplicationServiceTransaction
+			err := json.NewDecoder(r.Body).Decode(&txn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, ev := range txn.Events {
+				if ev.Type != spec.MRoomMember {
+					continue
+				}
+				if ev.StateKey != nil && *ev.StateKey == bob.ID {
+					membership := gjson.GetBytes(ev.Content, "membership").Str
+					t.Logf("Processing membership: %s", membership)
+					switch membership {
+					case spec.Invite:
+						// Accept the invite
+						joinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+							"membership": "join",
+						}, test.WithStateKey(bob.ID))
+
+						if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, []*types.HeaderedEvent{joinEv}, "test", "test", "test", nil, false); err != nil {
+							t.Fatalf("failed to send events: %v", err)
+						}
+					case spec.Join: // the AS has received the join event, now hit `/joined_members` to validate that
+						rec := httptest.NewRecorder()
+						req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/rooms/"+room.ID+"/joined_members", nil)
+						req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+						routers.Client.ServeHTTP(rec, req)
+						if rec.Code != http.StatusOK {
+							t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
+						}
+
+						// Both Alice and Bob should be joined. If not, we have a race condition
+						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+alice.ID).Exists() {
+							t.Errorf("Alice is not joined to the room") // in theory should not happen
+						}
+						if !gjson.GetBytes(rec.Body.Bytes(), "joined."+bob.ID).Exists() {
+							t.Errorf("Bob is not joined to the room")
+						}
+						evChan <- struct{}{}
+					default:
+						t.Fatalf("Unexpected membership: %s", membership)
+					}
+				}
+			}
+		}))
+		defer srv.Close()
+
+		as := &config.ApplicationService{
+			ID:              "someID",
+			URL:             srv.URL,
+			ASToken:         "",
+			HSToken:         "",
+			SenderLocalpart: "senderLocalPart",
+			NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+				"users":   {{RegexpObject: regexp.MustCompile(bob.ID)}},
+				"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
+			},
+		}
+		as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation)
+
+		// Create a dummy application service
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		// Prepare AS Streams on the old topic to validate that they get deleted
+		jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+
+		token := jetstream.Tokenise(as.ID)
+		if err := jetstream.JetStreamConsumer(
+			processCtx.Context(), jsCtx, cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
+			cfg.Global.JetStream.Durable("Appservice_"+token),
+			50, // maximum number of events to send in a single transaction
+			func(ctx context.Context, msgs []*nats.Msg) bool {
+				return true
+			},
+		); err != nil {
+			t.Fatal(err)
+		}
+
+		// Start the syncAPI to have `/joined_members` available
+		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, natsInstance, usrAPI, rsAPI, caches, caching.DisableMetrics)
+
+		// start the consumer
+		appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
+
+		// At this point, the old JetStream consumers should be deleted
+		for consumer := range jsCtx.Consumers(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) {
+			if consumer.Name == cfg.Global.JetStream.Durable("Appservice_"+token)+"Pull" {
+				t.Fatalf("Consumer still exists")
+			}
+		}
+
+		// Create the room, this triggers the AS to receive an invite for Bob.
+		if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		select {
+		// Pretty generous timeout duration...
+		case <-time.After(time.Millisecond * 1000): // wait for the AS to process the events
+			t.Errorf("Timed out waiting for join event")
+		case <-evChan:
+		}
+		close(evChan)
+	})
+}
+
+type userDevice struct {
+	accessToken string
+	deviceID    string
+	password    string
+}
+
+func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) {
+	t.Helper()
+	for u := range accessTokens {
+		localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+		userRes := &uapi.PerformAccountCreationResponse{}
+		password := util.RandomString(8)
+		if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+			AccountType: u.AccountType,
+			Localpart:   localpart,
+			ServerName:  serverName,
+			Password:    password,
+		}, userRes); err != nil {
+			t.Errorf("failed to create account: %s", err)
+		}
+		req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+			"type": authtypes.LoginTypePassword,
+			"identifier": map[string]interface{}{
+				"type": "m.id.user",
+				"user": u.ID,
+			},
+			"password": password,
+		}))
+		rec := httptest.NewRecorder()
+		routers.Client.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("failed to login: %s", rec.Body.String())
+		}
+		accessTokens[u] = userDevice{
+			accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(),
+			deviceID:    gjson.GetBytes(rec.Body.Bytes(), "device_id").String(),
+			password:    password,
+		}
+	}
+}

appservice/consumers/roomserver.go

@@ -71,13 +71,14 @@ func NewOutputRoomEventConsumer(
 		ctx:       process.Context(),
 		cfg:       cfg,
 		jetstream: js,
-		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
+		topic:     cfg.Matrix.JetStream.Prefixed(jetstream.OutputAppserviceEvent),
 		rsAPI:     rsAPI,
 	}
 }
 
 // Start consuming from room servers
 func (s *OutputRoomEventConsumer) Start() error {
+	durableNames := make([]string, 0, len(s.cfg.Derived.ApplicationServices))
 	for _, as := range s.cfg.Derived.ApplicationServices {
 		appsvc := as
 		state := &appserviceState{
@@ -95,6 +96,15 @@ func (s *OutputRoomEventConsumer) Start() error {
 		); err != nil {
 			return fmt.Errorf("failed to create %q consumer: %w", token, err)
 		}
+		durableNames = append(durableNames, s.cfg.Matrix.JetStream.Durable("Appservice_"+token))
+	}
+	// Cleanup any consumers still existing on the OutputRoomEvent stream
+	// to avoid messages not being deleted
+	for _, consumerName := range durableNames {
+		err := s.jetstream.DeleteConsumer(s.cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), consumerName+"Pull")
+		if err != nil && err != nats.ErrConsumerNotFound {
+			return err
+		}
 	}
 	return nil
 }
@@ -196,13 +206,21 @@ func (s *OutputRoomEventConsumer) sendEvents(
 	}
 
 	// Send the transaction to the appservice.
-	// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
-	address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken))
+	// https://spec.matrix.org/v1.9/application-service-api/#pushing-events
+	path := "_matrix/app/v1/transactions"
+	if s.cfg.LegacyPaths {
+		path = "transactions"
+	}
+	address := fmt.Sprintf("%s/%s/%s", state.RequestUrl(), path, txnID)
+	if s.cfg.LegacyAuth {
+		address += "?access_token=" + url.QueryEscape(state.HSToken)
+	}
 	req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", state.HSToken))
 	resp, err := state.HTTPClient.Do(req)
 	if err != nil {
 		return state.backoffAndPause(err)

appservice/query/query.go

@@ -19,10 +19,10 @@ package query
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"strings"
 	"sync"
 
 	log "github.com/sirupsen/logrus"
@@ -32,9 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 )
 
-const roomAliasExistsPath = "/rooms/"
-const userIDExistsPath = "/users/"
-
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
 	Cfg           *config.AppServiceAPI
@@ -55,14 +52,23 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 	// Determine which application service should handle this request
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
+			path := api.ASRoomAliasExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASRoomAliasExistsLegacyPath
+			}
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath)
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 
 			URL.Path += request.Alias
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the room, immediately return.
@@ -70,6 +76,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			req = req.WithContext(ctx)
 
 			resp, err := appservice.HTTPClient.Do(req)
@@ -123,12 +130,21 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	for _, appservice := range a.Cfg.Derived.ApplicationServices {
 		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
 			// The full path to the rooms API, includes hs token
-			URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath)
+			path := api.ASUserExistsPath
+			if a.Cfg.LegacyPaths {
+				path = api.ASUserExistsLegacyPath
+			}
+			URL, err := url.Parse(appservice.RequestUrl() + path)
 			if err != nil {
 				return err
 			}
 			URL.Path += request.UserID
-			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+			if a.Cfg.LegacyAuth {
+				q := URL.Query()
+				q.Set("access_token", appservice.HSToken)
+				URL.RawQuery = q.Encode()
+			}
+			apiURL := URL.String()
 
 			// Send a request to each application service. If one responds that it has
 			// created the user, immediately return.
@@ -136,6 +152,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
 			resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
@@ -176,25 +193,22 @@ type thirdpartyResponses interface {
 	api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
 }
 
-func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
-	origURL := url
-	// try v1 and unstable appservice endpoints
-	for _, version := range []string{"v1", "unstable"} {
-		var resp *http.Response
-		var body []byte
-		asURL := strings.Replace(origURL, "unstable", version, 1)
-		resp, err = client.Get(asURL)
+func requestDo[T thirdpartyResponses](as *config.ApplicationService, url string, response *T) error {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
-			continue
+		return err
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", as.HSToken))
+	resp, err := as.HTTPClient.Do(req)
+	if err != nil {
+		return err
 	}
 	defer resp.Body.Close() // nolint: errcheck
-		body, err = io.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-			continue
+		return err
 	}
 	return json.Unmarshal(body, &response)
-	}
-	return err
 }
 
 func (a *AppServiceQueryAPI) Locations(
@@ -207,16 +221,22 @@ func (a *AppServiceQueryAPI) Locations(
 		return err
 	}
 
+	path := api.ASLocationPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASLocationLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asLocations []api.ASLocationResponse
+		if a.Cfg.LegacyAuth {
 			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASLocationPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+		if err := requestDo[[]api.ASLocationResponse](&as, url+"?"+params.Encode(), &asLocations); err != nil {
 			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
 			continue
 		}
@@ -242,16 +262,22 @@ func (a *AppServiceQueryAPI) User(
 		return err
 	}
 
+	path := api.ASUserPath
+	if a.Cfg.LegacyPaths {
+		path = api.ASUserLegacyPath
+	}
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		var asUsers []api.ASUserResponse
+		if a.Cfg.LegacyAuth {
 			params.Set("access_token", as.HSToken)
+		}
 
-		url := as.RequestUrl() + api.ASUserPath
+		url := as.RequestUrl() + path
 		if req.Protocol != "" {
 			url += "/" + req.Protocol
 		}
 
-		if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+		if err := requestDo[[]api.ASUserResponse](&as, url+"?"+params.Encode(), &asUsers); err != nil {
 			log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
 			continue
 		}
@@ -272,6 +298,10 @@ func (a *AppServiceQueryAPI) Protocols(
 	req *api.ProtocolRequest,
 	resp *api.ProtocolResponse,
 ) error {
+	protocolPath := api.ASProtocolPath
+	if a.Cfg.LegacyPaths {
+		protocolPath = api.ASProtocolLegacyPath
+	}
 
 	// get a single protocol response
 	if req.Protocol != "" {
@@ -289,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols(
 		response := api.ASProtocolResponse{}
 		for _, as := range a.Cfg.Derived.ApplicationServices {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+req.Protocol, &proto); err != nil {
 				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}
@@ -319,7 +349,7 @@ func (a *AppServiceQueryAPI) Protocols(
 	for _, as := range a.Cfg.Derived.ApplicationServices {
 		for _, p := range as.Protocols {
 			var proto api.ASProtocolResponse
-			if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
+			if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+p, &proto); err != nil {
 				log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
 				continue
 			}

build/dendritejs-pinecone/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -190,13 +191,13 @@ func startup() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff)
 
 	asQuery := appservice.NewInternalAPI(
 		processCtx, cfg, &natsInstance, userAPI, rsAPI,
 	)
 	rsAPI.SetAppserviceAPI(asQuery)
-	fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true)
 	rsAPI.SetFederationAPI(fedSenderAPI, keyRing)
 
 	monolith := setup.Monolith{

build/docker/Dockerfile.demo-pinecone

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.21-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/docker/Dockerfile.demo-yggdrasil

@@ -1,4 +1,5 @@
-FROM docker.io/golang:1.19-alpine AS base
+# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
+FROM docker.io/golang:1.21-alpine3.18 AS base
 
 #
 # Needs to be separate from the main Dockerfile for OpenShift,

build/gobind-yggdrasil/monolith.go

@@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() {
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 	rsAPI.SetAppserviceAPI(asAPI)

build/scripts/complement.sh

@@ -15,5 +15,5 @@ tar -xzf master.tar.gz
 
 # Run the tests!
 cd complement-master
-COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests
+COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi
 

clientapi/admin_test.go

@@ -2,10 +2,12 @@ package clientapi
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -45,7 +47,7 @@ func TestAdminCreateToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -196,7 +198,7 @@ func TestAdminListRegistrationTokens(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -314,7 +316,7 @@ func TestAdminGetRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -415,7 +417,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -509,7 +511,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 		accessTokens := map[*test.User]userDevice{
 			aliceAdmin: {},
@@ -693,7 +695,7 @@ func TestAdminResetPassword(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for changing the password/login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -791,7 +793,7 @@ func TestPurgeRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics)
 
 		// Create the room
@@ -863,7 +865,7 @@ func TestAdminEvacuateRoom(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
@@ -964,7 +966,7 @@ func TestAdminEvacuateUser(t *testing.T) {
 		fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
 		rsAPI.SetFederationAPI(fsAPI, nil)
 
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// Create the room
 		if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
@@ -1055,7 +1057,7 @@ func TestAdminMarkAsStale(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1092,3 +1094,382 @@ func TestAdminMarkAsStale(t *testing.T) {
 		}
 	})
 }
+
+func TestAdminQueryEventReports(t *testing.T) {
+	alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+	room2 := test.NewRoom(t, alice)
+
+	// room2 has a name and canonical alias
+	room2.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": "Testing"}, test.WithStateKey(""))
+	room2.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": "#testing"}, test.WithStateKey(""))
+
+	// Join the rooms with Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+	room2.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	// Create a few events to report
+	eventsToReportPerRoom := make(map[string][]string)
+	for i := 0; i < 10; i++ {
+		ev1 := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+		ev2 := room2.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+		eventsToReportPerRoom[room.ID] = append(eventsToReportPerRoom[room.ID], ev1.EventID())
+		eventsToReportPerRoom[room2.ID] = append(eventsToReportPerRoom[room2.ID], ev2.EventID())
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		/*if dbType == test.DBTypeSQLite {
+			t.Skip()
+		}*/
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room2.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		// Report all events
+		for roomID, eventIDs := range eventsToReportPerRoom {
+			for _, eventID := range eventIDs {
+				req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", roomID, eventID), strings.NewReader(string(body)))
+				req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+				routers.Client.ServeHTTP(w, req)
+
+				if w.Code != http.StatusOK {
+					t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+				}
+			}
+		}
+
+		type response struct {
+			EventReports []api.QueryAdminEventReportsResponse `json:"event_reports"`
+			Total        int64                                `json:"total"`
+			NextToken    *int64                               `json:"next_token,omitempty"`
+		}
+
+		t.Run("Can query all reports", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 20
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can filter on room", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 10
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can filter on user_id", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?user_id=%s", "@doesnotexist:test"), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+
+			// The user does not exist, so we expect no results
+			wantCount := 0
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+		})
+
+		t.Run("Can set direction=f", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&dir=f", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			var resp response
+			if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			wantCount := 10
+			// Only validating the count
+			if len(resp.EventReports) != wantCount {
+				t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+			}
+			if resp.Total != int64(wantCount) {
+				t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+			}
+			// we now should have the first reported event
+			wantEventID := eventsToReportPerRoom[room.ID][0]
+			gotEventID := resp.EventReports[0].EventID
+			if gotEventID != wantEventID {
+				t.Fatalf("expected eventID to be %v, got %v", wantEventID, gotEventID)
+			}
+		})
+
+		t.Run("Can limit and paginate", func(t *testing.T) {
+			var from int64 = 0
+			var limit int64 = 5
+			var wantTotal int64 = 10 // We expect there to be 10 events in total
+			var resp response
+			for from+limit <= wantTotal {
+				resp = response{}
+				t.Logf("Getting reports starting from %d", from)
+				w = httptest.NewRecorder()
+				req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&limit=%d&from=%d", room2.ID, limit, from), strings.NewReader(string(body)))
+				req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+				routers.SynapseAdmin.ServeHTTP(w, req)
+
+				if w.Code != http.StatusOK {
+					t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+				}
+
+				if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+					t.Fatal(err)
+				}
+
+				wantCount := 5 // we are limited to 5
+				if len(resp.EventReports) != wantCount {
+					t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports))
+				}
+				if resp.Total != int64(wantTotal) {
+					t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total)
+				}
+
+				// We've reached the end
+				if (from + int64(len(resp.EventReports))) == wantTotal {
+					return
+				}
+
+				// The next_token should be set
+				if resp.NextToken == nil {
+					t.Fatal("expected nextToken to be set")
+				}
+				from = *resp.NextToken
+			}
+		})
+	})
+}
+
+func TestEventReportsGetDelete(t *testing.T) {
+	alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bob := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	// Add a name and alias
+	roomName := "Testing"
+	alias := "#testing"
+	room.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": roomName}, test.WithStateKey(""))
+	room.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": alias}, test.WithStateKey(""))
+
+	// Join the rooms with Bob
+	room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(bob.ID))
+
+	// Create a few events to report
+
+	eventIDToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		// Report the event
+		req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventIDToReport.EventID()), strings.NewReader(string(body)))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+		routers.Client.ServeHTTP(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String())
+		}
+
+		t.Run("Can not query with invalid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/abc", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusBadRequest {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Can query with valid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+			resp := api.QueryAdminEventReportResponse{}
+			if err = json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+			// test a few things
+			if resp.EventID != eventIDToReport.EventID() {
+				t.Fatalf("expected eventID to be %s, got %s instead", eventIDToReport.EventID(), resp.EventID)
+			}
+			if resp.RoomName != roomName {
+				t.Fatalf("expected roomName to be %s, got %s instead", roomName, resp.RoomName)
+			}
+			if resp.CanonicalAlias != alias {
+				t.Fatalf("expected alias to be %s, got %s instead", alias, resp.CanonicalAlias)
+			}
+			if reflect.DeepEqual(resp.EventJSON, eventIDToReport.JSON()) {
+				t.Fatal("mismatching eventJSON")
+			}
+		})
+
+		t.Run("Can delete with a valid ID", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodDelete, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Can not query deleted report", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+			routers.SynapseAdmin.ServeHTTP(w, req)
+
+			if w.Code == http.StatusOK {
+				t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+	})
+}

clientapi/auth/login.go

@@ -15,7 +15,6 @@
 package auth
 
 import (
-	"context"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -32,8 +31,13 @@ import (
 // called after authorization has completed, with the result of the authorization.
 // If the final return value is non-nil, an error occurred and the cleanup function
 // is nil.
-func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
-	reqBytes, err := io.ReadAll(r)
+func LoginFromJSONReader(
+	req *http.Request,
+	useraccountAPI uapi.UserLoginAPI,
+	userAPI UserInternalAPIForLogin,
+	cfg *config.ClientAPI,
+) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	reqBytes, err := io.ReadAll(req.Body)
 	if err != nil {
 		err := &util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -57,7 +61,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 	switch header.Type {
 	case authtypes.LoginTypePassword:
 		typ = &LoginTypePassword{
-			GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
+			UserAPI: useraccountAPI,
 			Config:  cfg,
 		}
 	case authtypes.LoginTypeToken:
@@ -65,6 +69,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 			UserAPI: userAPI,
 			Config:  cfg,
 		}
+	case authtypes.LoginTypeApplicationService:
+		token, err := ExtractAccessToken(req)
+		if err != nil {
+			err := &util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.MissingToken(err.Error()),
+			}
+			return nil, nil, err
+		}
+
+		typ = &LoginTypeApplicationService{
+			Config: cfg,
+			Token:  token,
+		}
 	default:
 		err := util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -73,7 +91,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
 		return nil, nil, &err
 	}
 
-	return typ.LoginFromJSON(ctx, reqBytes)
+	return typ.LoginFromJSON(req.Context(), reqBytes)
 }
 
 // UserInternalAPIForLogin contains the aspects of UserAPI required for logging in.

clientapi/auth/login_application_service.go

@@ -0,0 +1,55 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/util"
+)
+
+// LoginTypeApplicationService describes how to authenticate as an
+// application service
+type LoginTypeApplicationService struct {
+	Config *config.ClientAPI
+	Token  string
+}
+
+// Name implements Type
+func (t *LoginTypeApplicationService) Name() string {
+	return authtypes.LoginTypeApplicationService
+}
+
+// LoginFromJSON implements Type
+func (t *LoginTypeApplicationService) LoginFromJSON(
+	ctx context.Context, reqBytes []byte,
+) (*Login, LoginCleanupFunc, *util.JSONResponse) {
+	var r Login
+	if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
+		return nil, nil, err
+	}
+
+	_, err := internal.ValidateApplicationServiceRequest(t.Config, r.Identifier.User, t.Token)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cleanup := func(ctx context.Context, j *util.JSONResponse) {}
+	return &r, cleanup, nil
+}

clientapi/auth/login_test.go

@@ -17,7 +17,9 @@ package auth
 import (
 	"context"
 	"net/http"
+	"net/http/httptest"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -35,6 +37,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 	tsts := []struct {
 		Name  string
 		Body  string
+		Token string
 
 		WantUsername      string
 		WantDeviceID      string
@@ -62,6 +65,30 @@ func TestLoginFromJSONReader(t *testing.T) {
 			WantDeviceID:      "adevice",
 			WantDeletedTokens: []string{"atoken"},
 		},
+		{
+			Name: "appServiceWorksUserID",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			Token: "astoken",
+
+			WantUsername: "@alice:example.com",
+			WantDeviceID: "adevice",
+		},
+		{
+			Name: "appServiceWorksLocalpart",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "alice" },
+				"device_id": "adevice"
+			}`,
+			Token: "astoken",
+
+			WantUsername: "alice",
+			WantDeviceID: "adevice",
+		},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -72,11 +99,35 @@ func TestLoginFromJSONReader(t *testing.T) {
 						ServerName: serverName,
 					},
 				},
+				Derived: &config.Derived{
+					ApplicationServices: []config.ApplicationService{
+						{
+							ID:      "anapplicationservice",
+							ASToken: "astoken",
+							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+								"users": {
+									{
+										Exclusive:    true,
+										Regex:        "@alice:example.com",
+										RegexpObject: regexp.MustCompile("@alice:example.com"),
+									},
+								},
+							},
+						},
+					},
+				},
 			}
-			login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
-			if err != nil {
-				t.Fatalf("LoginFromJSONReader failed: %+v", err)
+
+			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
+			if tst.Token != "" {
+				req.Header.Add("Authorization", "Bearer "+tst.Token)
 			}
+
+			login, cleanup, jsonErr := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
+			if jsonErr != nil {
+				t.Fatalf("LoginFromJSONReader failed: %+v", jsonErr)
+			}
+
 			cleanup(ctx, &util.JSONResponse{Code: http.StatusOK})
 
 			if login.Username() != tst.WantUsername {
@@ -106,6 +157,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 	tsts := []struct {
 		Name  string
 		Body  string
+		Token string
 
 		WantErrCode spec.MatrixErrorCode
 	}{
@@ -142,6 +194,45 @@ func TestBadLoginFromJSONReader(t *testing.T) {
             }`,
 			WantErrCode: spec.ErrorInvalidParam,
 		},
+		{
+			Name: "noASToken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_MISSING_TOKEN",
+		},
+		{
+			Name:  "badASToken",
+			Token: "badastoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_UNKNOWN_TOKEN",
+		},
+		{
+			Name:  "badASNamespace",
+			Token: "astoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@bob:example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_EXCLUSIVE",
+		},
+		{
+			Name:  "badASUserID",
+			Token: "astoken",
+			Body: `{
+				"type": "m.login.application_service",
+				"identifier": { "type": "m.id.user", "user": "@alice:wrong.example.com" },
+				"device_id": "adevice"
+			}`,
+			WantErrCode: "M_INVALID_USERNAME",
+		},
 	}
 	for _, tst := range tsts {
 		t.Run(tst.Name, func(t *testing.T) {
@@ -152,8 +243,30 @@ func TestBadLoginFromJSONReader(t *testing.T) {
 						ServerName: serverName,
 					},
 				},
+				Derived: &config.Derived{
+					ApplicationServices: []config.ApplicationService{
+						{
+							ID:      "anapplicationservice",
+							ASToken: "astoken",
+							NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+								"users": {
+									{
+										Exclusive:    true,
+										Regex:        "@alice:example.com",
+										RegexpObject: regexp.MustCompile("@alice:example.com"),
+									},
+								},
+							},
+						},
+					},
+				},
 			}
-			_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
+			req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body))
+			if tst.Token != "" {
+				req.Header.Add("Authorization", "Bearer "+tst.Token)
+			}
+
+			_, cleanup, errRes := LoginFromJSONReader(req, &userAPI, &userAPI, cfg)
 			if errRes == nil {
 				cleanup(ctx, nil)
 				t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
@@ -179,6 +292,14 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 	return nil
 }
 
+func (ua *fakeUserInternalAPI) QueryAccountByLocalpart(ctx context.Context, req *uapi.QueryAccountByLocalpartRequest, res *uapi.QueryAccountByLocalpartResponse) error {
+	return nil
+}
+
+func (ua *fakeUserInternalAPI) PerformAccountCreation(ctx context.Context, req *uapi.PerformAccountCreationRequest, res *uapi.PerformAccountCreationResponse) error {
+	return nil
+}
+
 func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, req *uapi.PerformLoginTokenDeletionRequest, res *uapi.PerformLoginTokenDeletionResponse) error {
 	ua.DeletedTokens = append(ua.DeletedTokens, req.Token)
 	return nil

clientapi/auth/password.go

@@ -16,6 +16,9 @@ package auth
 
 import (
 	"context"
+	"database/sql"
+	"github.com/go-ldap/ldap/v3"
+	"github.com/google/uuid"
 	"net/http"
 	"strings"
 
@@ -28,8 +31,6 @@ import (
 	"github.com/matrix-org/util"
 )
 
-type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error
-
 type PasswordRequest struct {
 	Login
 	Password string `json:"password"`
@@ -37,8 +38,8 @@ type PasswordRequest struct {
 
 // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
 type LoginTypePassword struct {
-	GetAccountByPassword GetAccountByPassword
 	Config  *config.ClientAPI
+	UserAPI api.UserLoginAPI
 }
 
 func (t *LoginTypePassword) Name() string {
@@ -59,22 +60,21 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
 	return login, func(context.Context, *util.JSONResponse) {}, nil
 }
 
-func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
-	r := req.(*PasswordRequest)
-	username := r.Username()
-	if username == "" {
+func (t *LoginTypePassword) Login(ctx context.Context, request *PasswordRequest) (*Login, *util.JSONResponse) {
+	fullUsername := request.Username()
+	if fullUsername == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
 			JSON: spec.BadJSON("A username must be supplied."),
 		}
 	}
-	if len(r.Password) == 0 {
+	if len(request.Password) == 0 {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
 			JSON: spec.BadJSON("A password must be supplied."),
 		}
 	}
-	localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
+	username, domain, err := userutil.ParseUsernameParam(fullUsername, t.Config.Matrix)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
@@ -87,12 +87,38 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			JSON: spec.InvalidUsername("The server name is not known."),
 		}
 	}
-	// Squash username to all lowercase letters
+
+	var account *api.Account
+	if t.Config.Ldap.Enabled {
+		isAdmin, err := t.authenticateLdap(username, request.Password)
+		if err != nil {
+			return nil, err
+		}
+		acc, err := t.getOrCreateAccount(ctx, username, domain, isAdmin)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	} else {
+		acc, err := t.authenticateDb(ctx, username, domain, request.Password)
+		if err != nil {
+			return nil, err
+		}
+		account = acc
+	}
+
+	// Set the user, so login.Username() can do the right thing
+	request.Identifier.User = account.UserID
+	request.User = account.UserID
+	return &request.Login, nil
+}
+
+func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string, domain spec.ServerName, password string) (*api.Account, *util.JSONResponse) {
 	res := &api.QueryAccountByPasswordResponse{}
-	err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-		Localpart:         strings.ToLower(localpart),
+	err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		Localpart:         strings.ToLower(username),
 		ServerName:        domain,
-		PlaintextPassword: r.Password,
+		PlaintextPassword: password,
 	}, res)
 	if err != nil {
 		return nil, &util.JSONResponse{
@@ -101,13 +127,11 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		}
 	}
 
-	// If we couldn't find the user by the lower cased localpart, try the provided
-	// localpart as is.
 	if !res.Exists {
-		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-			Localpart:         localpart,
+		err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+			Localpart:         username,
 			ServerName:        domain,
-			PlaintextPassword: r.Password,
+			PlaintextPassword: password,
 		}, res)
 		if err != nil {
 			return nil, &util.JSONResponse{
@@ -115,8 +139,6 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 				JSON: spec.Unknown("Unable to fetch account by password."),
 			}
 		}
-		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
-		// but that would leak the existence of the user.
 		if !res.Exists {
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
@@ -124,8 +146,141 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			}
 		}
 	}
-	// Set the user, so login.Username() can do the right thing
-	r.Identifier.User = res.Account.UserID
-	r.User = res.Account.UserID
-	return &r.Login, nil
+	return res.Account, nil
+}
+func (t *LoginTypePassword) authenticateLdap(username, password string) (bool, *util.JSONResponse) {
+	var conn *ldap.Conn
+	conn, err := ldap.DialURL(t.Config.Ldap.Uri)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("unable to connect to ldap: " + err.Error()),
+		}
+	}
+	defer conn.Close()
+
+	if t.Config.Ldap.AdminBindEnabled {
+		err = conn.Bind(t.Config.Ldap.AdminBindDn, t.Config.Ldap.AdminBindPassword)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("unable to bind to ldap: " + err.Error()),
+			}
+		}
+		filter := strings.ReplaceAll(t.Config.Ldap.SearchFilter, "{username}", username)
+		searchRequest := ldap.NewSearchRequest(
+			t.Config.Ldap.BaseDn, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases,
+			0, 0, false, filter, []string{t.Config.Ldap.SearchAttribute}, nil,
+		)
+		result, err := conn.Search(searchRequest)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("unable to bind to search ldap: " + err.Error()),
+			}
+		}
+		if len(result.Entries) > 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.BadJSON("'user' must be duplicated."),
+			}
+		}
+		if len(result.Entries) < 1 {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.BadJSON("'user' not found."),
+			}
+		}
+
+		userDN := result.Entries[0].DN
+		err = conn.Bind(userDN, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.InvalidUsername(err.Error()),
+			}
+		}
+	} else {
+		bindDn := strings.ReplaceAll(t.Config.Ldap.UserBindDn, "{username}", username)
+		err = conn.Bind(bindDn, password)
+		if err != nil {
+			return false, &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: spec.InvalidUsername(err.Error()),
+			}
+		}
+	}
+
+	isAdmin, err := t.isLdapAdmin(conn, username)
+	if err != nil {
+		return false, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.InvalidUsername(err.Error()),
+		}
+	}
+	return isAdmin, nil
+}
+
+func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, error) {
+	searchRequest := ldap.NewSearchRequest(
+		t.Config.Ldap.AdminGroupDn,
+		ldap.ScopeWholeSubtree, ldap.DerefAlways, 0, 0, false,
+		strings.ReplaceAll(t.Config.Ldap.AdminGroupFilter, "{username}", username),
+		[]string{t.Config.Ldap.AdminGroupAttribute},
+		nil)
+
+	sr, err := conn.Search(searchRequest)
+	if err != nil {
+		return false, err
+	}
+
+	if len(sr.Entries) < 1 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain spec.ServerName, admin bool) (*api.Account, *util.JSONResponse) {
+	var existing api.QueryAccountByLocalpartResponse
+	err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{
+		Localpart:  username,
+		ServerName: domain,
+	}, &existing)
+
+	if err == nil {
+		return existing.Account, nil
+	}
+	if err != sql.ErrNoRows {
+		return nil, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: spec.InvalidUsername(err.Error()),
+		}
+	}
+
+	accountType := api.AccountTypeUser
+	if admin {
+		accountType = api.AccountTypeAdmin
+	}
+	var created api.PerformAccountCreationResponse
+	err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{
+		AppServiceID: "ldap",
+		Localpart:    username,
+		Password:     uuid.New().String(),
+		AccountType:  accountType,
+		OnConflict:   api.ConflictAbort,
+	}, &created)
+
+	if err != nil {
+		if _, ok := err.(*api.ErrorConflict); ok {
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.UserInUse("Desired user ID is already taken."),
+			}
+		}
+		return nil, &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown("failed to create account: " + err.Error()),
+		}
+	}
+	return created.Account, nil
 }

clientapi/auth/user_interactive.go

@@ -55,7 +55,7 @@ type LoginCleanupFunc func(context.Context, *util.JSONResponse)
 // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types
 type LoginIdentifier struct {
 	Type string `json:"type"`
-	// when type = m.id.user
+	// when type = m.id.user or m.id.application_service
 	User string `json:"user"`
 	// when type = m.id.thirdparty
 	Medium  string `json:"medium"`
@@ -113,7 +113,7 @@ type UserInteractive struct {
 
 func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
+		UserAPI: userAccountAPI,
 		Config:  cfg,
 	}
 	return &UserInteractive{

clientapi/auth/user_interactive_test.go

@@ -45,6 +45,14 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
 	return nil
 }
 
+func (d *fakeAccountDatabase) QueryAccountByLocalpart(ctx context.Context, req *api.QueryAccountByLocalpartRequest, res *api.QueryAccountByLocalpartResponse) error {
+	return nil
+}
+
+func (d *fakeAccountDatabase) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error {
+	return nil
+}
+
 func setup() *UserInteractive {
 	cfg := &config.ClientAPI{
 		Matrix: &config.Global{

clientapi/clientapi_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	"github.com/matrix-org/dendrite/clientapi/threepid"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/pushrules"
@@ -49,6 +50,10 @@ type userDevice struct {
 	password    string
 }
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestGetPutDevices(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -121,7 +126,7 @@ func TestGetPutDevices(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -170,7 +175,7 @@ func TestDeleteDevice(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -275,7 +280,7 @@ func TestDeleteDevices(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -442,7 +447,7 @@ func TestSetDisplayname(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -554,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) {
 
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI)
 
 		AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -632,7 +637,7 @@ func TestTyping(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -716,7 +721,7 @@ func TestMembership(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed to create accounts
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		rsAPI.SetUserAPI(userAPI)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -953,9 +958,10 @@ func TestCapabilities(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1000,9 +1006,10 @@ func TestTurnserver(t *testing.T) {
 	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 	// Needed to create accounts
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+	caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 	rsAPI.SetFederationAPI(nil, nil)
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 	//rsAPI.SetUserAPI(userAPI)
 	// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 	AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1098,9 +1105,10 @@ func Test3PID(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 
 		// Needed to create accounts
-		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
 
@@ -1276,7 +1284,7 @@ func TestPushRules(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -1418,7 +1426,7 @@ func TestPushRules(t *testing.T) {
 				validateFunc: func(t *testing.T, respBody *bytes.Buffer) {
 					actions := gjson.GetBytes(respBody.Bytes(), "actions").Array()
 					// only a basic check
-					assert.Equal(t, 1, len(actions))
+					assert.Equal(t, 0, len(actions))
 				},
 			},
 			{
@@ -1663,7 +1671,7 @@ func TestKeys(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -2125,7 +2133,7 @@ func TestKeyBackup(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
 		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
@@ -2146,3 +2154,284 @@ func TestKeyBackup(t *testing.T) {
 		}
 	})
 }
+
+func TestGetMembership(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	testCases := []struct {
+		name             string
+		roomID           string
+		user             *test.User
+		additionalEvents func(t *testing.T, room *test.Room)
+		request          func(t *testing.T, room *test.Room, accessToken string) *http.Request
+		wantOK           bool
+		wantMemberCount  int
+	}{
+
+		{
+			name: "/joined_members - Bob never joined",
+			user: bob,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Alice joined",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			wantOK:          true,
+			wantMemberCount: 1,
+		},
+		{
+			name: "/joined_members - Alice leaves, shouldn't be able to see members ",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
+					"membership": "leave",
+				}, test.WithStateKey(alice.ID))
+			},
+			wantOK: false,
+		},
+		{
+			name: "/joined_members - Bob joins, Alice sees two members",
+			user: alice,
+			request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
+				return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
+					"access_token": accessToken,
+				}))
+			},
+			additionalEvents: func(t *testing.T, room *test.Room) {
+				room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
+					"membership": "join",
+				}, test.WithStateKey(bob.ID))
+			},
+			wantOK:          true,
+			wantMemberCount: 2,
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+			bob:   {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				room := test.NewRoom(t, alice)
+				t.Cleanup(func() {
+					t.Logf("running cleanup for %s", tc.name)
+				})
+				// inject additional events
+				if tc.additionalEvents != nil {
+					tc.additionalEvents(t, room)
+				}
+				if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+					t.Fatalf("failed to send events: %v", err)
+				}
+
+				w := httptest.NewRecorder()
+				routers.Client.ServeHTTP(w, tc.request(t, room, accessTokens[tc.user].accessToken))
+				if w.Code != 200 && tc.wantOK {
+					t.Logf("%s", w.Body.String())
+					t.Fatalf("got HTTP %d want %d", w.Code, 200)
+				}
+				t.Logf("[%s] Resp: %s", tc.name, w.Body.String())
+
+				// check we got the expected events
+				if tc.wantOK {
+					memberCount := len(gjson.GetBytes(w.Body.Bytes(), "joined").Map())
+					if memberCount != tc.wantMemberCount {
+						t.Fatalf("expected %d members, got %d", tc.wantMemberCount, memberCount)
+					}
+				}
+			})
+		}
+	})
+}
+
+func TestCreateRoomInvite(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"invite": []string{bob.ID},
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/createRoom", strings.NewReader(string(body)))
+		req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
+
+		routers.Client.ServeHTTP(w, req)
+
+		if w.Code != http.StatusOK {
+			t.Fatalf("expected room creation to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+		}
+
+		roomID := gjson.GetBytes(w.Body.Bytes(), "room_id").Str
+		validRoomID, _ := spec.NewRoomID(roomID)
+		// Now ask the roomserver about the membership event of Bob
+		ev, err := rsAPI.CurrentStateEvent(context.Background(), *validRoomID, spec.MRoomMember, bob.ID)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if ev == nil {
+			t.Fatal("Membership event for Bob does not exist")
+		}
+
+		// Validate that there is NO displayname in content
+		if gjson.GetBytes(ev.Content(), "displayname").Exists() {
+			t.Fatal("Found displayname in invite")
+		}
+	})
+}
+
+func TestReportEvent(t *testing.T) {
+	alice := test.NewUser(t)
+	bob := test.NewUser(t)
+	charlie := test.NewUser(t)
+	room := test.NewRoom(t, alice)
+
+	room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, test.WithStateKey(charlie.ID))
+	eventToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"})
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		routers := httputil.NewRouters()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		defer close()
+		natsInstance := jetstream.NATSInstance{}
+		jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
+		defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
+
+		// Use an actual roomserver for this
+		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
+
+		if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
+			t.Fatalf("failed to send events: %v", err)
+		}
+
+		// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
+		AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
+
+		accessTokens := map[*test.User]userDevice{
+			alice:   {},
+			bob:     {},
+			charlie: {},
+		}
+		createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
+
+		reqBody := map[string]any{
+			"reason": "baaad",
+			"score":  -100,
+		}
+		body, err := json.Marshal(reqBody)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		w := httptest.NewRecorder()
+
+		var req *http.Request
+		t.Run("Bob is not joined and should not be able to report the event", func(t *testing.T) {
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined but the event does not exist", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/$doesNotExist", room.ID), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusNotFound {
+				t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+
+		t.Run("Charlie is joined and allowed to report the event", func(t *testing.T) {
+			w = httptest.NewRecorder()
+			req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body)))
+			req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken)
+
+			routers.Client.ServeHTTP(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("expected report to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
+			}
+		})
+	})
+}

clientapi/routing/admin.go

@@ -495,3 +495,93 @@ func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverA
 		JSON: struct{}{},
 	}
 }
+
+// GetEventReports returns reported events for a given user/room.
+func GetEventReports(
+	req *http.Request,
+	rsAPI roomserverAPI.ClientRoomserverAPI,
+	from, limit uint64,
+	backwards bool,
+	userID, roomID string,
+) util.JSONResponse {
+
+	eventReports, count, err := rsAPI.QueryAdminEventReports(req.Context(), from, limit, backwards, userID, roomID)
+	if err != nil {
+		logrus.WithError(err).Error("failed to query event reports")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	resp := map[string]any{
+		"event_reports": eventReports,
+		"total":         count,
+	}
+
+	// Add a next_token if there are still reports
+	if int64(from+limit) < count {
+		resp["next_token"] = int(from) + len(eventReports)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp,
+	}
+}
+
+func GetEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	report, err := rsAPI.QueryAdminEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: report,
+	}
+}
+
+func DeleteEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse {
+	parsedReportID, err := strconv.ParseUint(reportID, 10, 64)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			// Given this is an admin endpoint, let them know what didn't work.
+			JSON: spec.InvalidParam(err.Error()),
+		}
+	}
+
+	err = rsAPI.PerformAdminDeleteEventReport(req.Context(), parsedReportID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.Unknown(err.Error()),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func parseUint64OrDefault(input string, defaultValue uint64) uint64 {
+	v, err := strconv.ParseUint(input, 10, 64)
+	if err != nil {
+		return defaultValue
+	}
+	return v
+}

clientapi/routing/createroom.go

@@ -48,7 +48,6 @@ type createRoomRequest struct {
 	RoomVersion               gomatrixserverlib.RoomVersion      `json:"room_version"`
 	PowerLevelContentOverride json.RawMessage                    `json:"power_level_content_override"`
 	IsDirect                  bool                               `json:"is_direct"`
-	CryptoID                  string                             `json:"cryptoid"`
 }
 
 func (r createRoomRequest) Validate() *util.JSONResponse {
@@ -108,27 +107,12 @@ type createRoomResponse struct {
 	RoomAlias string `json:"room_alias,omitempty"` // in synapse not spec
 }
 
-type createRoomCryptoIDsResponse struct {
-	RoomID  string            `json:"room_id"`
-	Version string            `json:"room_version"`
-	PDUs    []json.RawMessage `json:"pdus"`
-}
-
-func ToProtoEvents(ctx context.Context, events []gomatrixserverlib.PDU, rsAPI roomserverAPI.ClientRoomserverAPI) []json.RawMessage {
-	result := make([]json.RawMessage, len(events))
-	for i, event := range events {
-		result[i] = json.RawMessage(event.JSON())
-	}
-	return result
-}
-
 // CreateRoom implements /createRoom
 func CreateRoom(
 	req *http.Request, device *api.Device,
 	cfg *config.ClientAPI,
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
-	cryptoIDs bool,
 ) util.JSONResponse {
 	var createRequest createRoomRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &createRequest)
@@ -145,9 +129,10 @@ func CreateRoom(
 			JSON: spec.InvalidParam(err.Error()),
 		}
 	}
-	return createRoom(req.Context(), createRequest, device, cfg, profileAPI, rsAPI, asAPI, evTime, cryptoIDs)
+	return createRoom(req.Context(), createRequest, device, cfg, profileAPI, rsAPI, asAPI, evTime)
 }
 
+// createRoom implements /createRoom
 func createRoom(
 	ctx context.Context,
 	createRequest createRoomRequest, device *api.Device,
@@ -155,7 +140,6 @@ func createRoom(
 	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI,
 	evTime time.Time,
-	cryptoIDs bool,
 ) util.JSONResponse {
 	userID, err := spec.NewUserID(device.UserID, true)
 	if err != nil {
@@ -241,7 +225,6 @@ func createRoom(
 		EventTime:       evTime,
 	}
 
-	if !cryptoIDs {
 	roomAlias, createRes := rsAPI.PerformCreateRoom(ctx, *userID, *roomID, &req)
 	if createRes != nil {
 		return *createRes
@@ -256,27 +239,4 @@ func createRoom(
 		Code: 200,
 		JSON: response,
 	}
-	} else {
-		req.SenderID = createRequest.CryptoID
-		createEvents, err := rsAPI.PerformCreateRoomCryptoIDs(ctx, *userID, *roomID, &req)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("MakeCreateRoomEvents failed")
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{Err: err.Error()},
-			}
-		}
-
-		response := createRoomCryptoIDsResponse{
-			RoomID:  roomID.String(),
-			Version: string(roomVersion),
-			PDUs:    ToProtoEvents(ctx, createEvents, rsAPI),
-		}
-
-		return util.JSONResponse{
-			Code: 200,
-			JSON: response,
-		}
-	}
-
 }

clientapi/routing/directory.go

@@ -55,7 +55,7 @@ func DirectoryRoom(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 
@@ -134,7 +134,7 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
+			JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
 		}
 	}
 

clientapi/routing/joinroom.go

@@ -27,7 +27,6 @@ import (
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
-	"github.com/sirupsen/logrus"
 )
 
 func JoinRoomByIDOrAlias(
@@ -145,139 +144,3 @@ func JoinRoomByIDOrAlias(
 		return result
 	}
 }
-
-type joinRoomCryptoIDsResponse struct {
-	RoomID    string          `json:"room_id"`
-	Version   string          `json:"room_version"`
-	ViaServer string          `json:"via_server"`
-	PDU       json.RawMessage `json:"pdu"`
-}
-
-func JoinRoomByIDOrAliasCryptoIDs(
-	req *http.Request,
-	device *api.Device,
-	rsAPI roomserverAPI.ClientRoomserverAPI,
-	profileAPI api.ClientUserAPI,
-	roomIDOrAlias string,
-) util.JSONResponse {
-	// Prepare to ask the roomserver to perform the room join.
-	joinReq := roomserverAPI.PerformJoinRequest{
-		RoomIDOrAlias: roomIDOrAlias,
-		UserID:        device.UserID,
-		IsGuest:       device.AccountType == api.AccountTypeGuest,
-		Content:       map[string]interface{}{},
-	}
-
-	// Check to see if any ?server_name= query parameters were
-	// given in the request.
-	if serverNames, ok := req.URL.Query()["server_name"]; ok {
-		for _, serverName := range serverNames {
-			joinReq.ServerNames = append(
-				joinReq.ServerNames,
-				spec.ServerName(serverName),
-			)
-		}
-	}
-
-	// If content was provided in the request then include that
-	// in the request. It'll get used as a part of the membership
-	// event content.
-	_ = httputil.UnmarshalJSONRequest(req, &joinReq.Content)
-
-	if senderid, ok := joinReq.Content["cryptoid"]; ok {
-		logrus.Errorf("CryptoID: %s", senderid.(string))
-		joinReq.SenderID = spec.SenderID(senderid.(string))
-	} else {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.Unknown("Missing cryptoid in request body"),
-		}
-	}
-	delete(joinReq.Content, "cryptoid")
-
-	// Work out our localpart for the client profile request.
-
-	// Request our profile content to populate the request content with.
-	profile, err := profileAPI.QueryProfile(req.Context(), device.UserID)
-
-	switch err {
-	case nil:
-		joinReq.Content["displayname"] = profile.DisplayName
-		joinReq.Content["avatar_url"] = profile.AvatarURL
-	case appserviceAPI.ErrProfileNotExists:
-		util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("Unable to query user profile, no profile found."),
-		}
-	default:
-	}
-
-	// Ask the roomserver to perform the join.
-	done := make(chan util.JSONResponse, 1)
-	go func() {
-		defer close(done)
-		joinEvent, roomID, version, serverName, err := rsAPI.PerformJoinCryptoIDs(req.Context(), &joinReq)
-		var response util.JSONResponse
-
-		switch e := err.(type) {
-		case nil: // success case
-			response = util.JSONResponse{
-				Code: http.StatusOK,
-				JSON: joinRoomCryptoIDsResponse{
-					RoomID:    roomID,
-					Version:   string(version),
-					ViaServer: string(serverName),
-					PDU:       json.RawMessage(joinEvent.JSON()),
-				},
-			}
-		case roomserverAPI.ErrInvalidID:
-			response = util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.Unknown(e.Error()),
-			}
-		case roomserverAPI.ErrNotAllowed:
-			jsonErr := spec.Forbidden(e.Error())
-			if device.AccountType == api.AccountTypeGuest {
-				jsonErr = spec.GuestAccessForbidden(e.Error())
-			}
-			response = util.JSONResponse{
-				Code: http.StatusForbidden,
-				JSON: jsonErr,
-			}
-		case *gomatrix.HTTPError: // this ensures we proxy responses over federation to the client
-			response = util.JSONResponse{
-				Code: e.Code,
-				JSON: json.RawMessage(e.Message),
-			}
-		case eventutil.ErrRoomNoExists:
-			response = util.JSONResponse{
-				Code: http.StatusNotFound,
-				JSON: spec.NotFound(e.Error()),
-			}
-		default:
-			response = util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{},
-			}
-		}
-		done <- response
-	}()
-
-	// Wait either for the join to finish, or for us to hit a reasonable
-	// timeout, at which point we'll just return a 200 to placate clients.
-	timer := time.NewTimer(time.Second * 20)
-	select {
-	case <-timer.C:
-		return util.JSONResponse{
-			Code: http.StatusRequestTimeout,
-			JSON: spec.Unknown("Failed creating join event with the remote server."),
-		}
-	case result := <-done:
-		// Stop and drain the timer
-		if !timer.Stop() {
-			<-timer.C
-		}
-		return result
-	}
-}

clientapi/routing/joinroom_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/jetstream"
@@ -21,6 +22,10 @@ import (
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 )
 
+var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) {
+	return &statistics.ServerStatistics{}, nil
+}
+
 func TestJoinRoomByIDOrAlias(t *testing.T) {
 	alice := test.NewUser(t)
 	bob := test.NewUser(t)
@@ -36,7 +41,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 		// Create the users in the userapi
@@ -67,7 +72,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			Preset:        spec.PresetPublicChat,
 			RoomAliasName: "alias",
 			Invite:        []string{bob.ID},
-		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now(), false)
+		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
 		crResp, ok := resp.JSON.(createRoomResponse)
 		if !ok {
 			t.Fatalf("response is not a createRoomResponse: %+v", resp)
@@ -81,7 +86,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 			Visibility: "public",
 			Preset:     spec.PresetPublicChat,
 			Invite:     []string{charlie.ID},
-		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now(), false)
+		}, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
 		crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse)
 		if !ok {
 			t.Fatalf("response is not a createRoomResponse: %+v", resp)

clientapi/routing/key_crosssigning.go

@@ -32,7 +32,7 @@ type crossSigningRequest struct {
 }
 
 func UploadCrossSigningDeviceKeys(
-	req *http.Request, userInteractiveAuth *auth.UserInteractive,
+	req *http.Request,
 	keyserverAPI api.ClientKeyAPI, device *api.Device,
 	accountAPI api.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
@@ -62,7 +62,7 @@ func UploadCrossSigningDeviceKeys(
 		}
 	}
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: accountAPI.QueryAccountByPassword,
+		UserAPI: accountAPI,
 		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {

clientapi/routing/keys.go

@@ -93,96 +93,9 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device)
 
 type queryKeysRequest struct {
 	Timeout    int                 `json:"timeout"`
-	Token      string              `json:"token"`
 	DeviceKeys map[string][]string `json:"device_keys"`
 }
 
-type uploadKeysCryptoIDsRequest struct {
-	DeviceKeys       json.RawMessage            `json:"device_keys"`
-	OneTimeKeys      map[string]json.RawMessage `json:"one_time_keys"`
-	OneTimeCryptoIDs map[string]json.RawMessage `json:"one_time_cryptoids"`
-}
-
-func UploadKeysCryptoIDs(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse {
-	var r uploadKeysCryptoIDsRequest
-	resErr := httputil.UnmarshalJSONRequest(req, &r)
-	if resErr != nil {
-		return *resErr
-	}
-
-	uploadReq := &api.PerformUploadKeysRequest{
-		DeviceID: device.ID,
-		UserID:   device.UserID,
-	}
-	if r.DeviceKeys != nil {
-		uploadReq.DeviceKeys = []api.DeviceKeys{
-			{
-				DeviceID: device.ID,
-				UserID:   device.UserID,
-				KeyJSON:  r.DeviceKeys,
-			},
-		}
-	}
-	if r.OneTimeKeys != nil {
-		uploadReq.OneTimeKeys = []api.OneTimeKeys{
-			{
-				DeviceID: device.ID,
-				UserID:   device.UserID,
-				KeyJSON:  r.OneTimeKeys,
-			},
-		}
-	}
-	if r.OneTimeCryptoIDs != nil {
-		uploadReq.OneTimeCryptoIDs = []api.OneTimeCryptoIDs{
-			{
-				UserID:  device.UserID,
-				KeyJSON: r.OneTimeCryptoIDs,
-			},
-		}
-	}
-
-	util.GetLogger(req.Context()).
-		WithField("device keys", r.DeviceKeys).
-		WithField("one-time keys", r.OneTimeKeys).
-		WithField("one-time cryptoids", r.OneTimeCryptoIDs).
-		Info("Uploading keys")
-
-	var uploadRes api.PerformUploadKeysResponse
-	if err := keyAPI.PerformUploadKeys(req.Context(), uploadReq, &uploadRes); err != nil {
-		return util.ErrorResponse(err)
-	}
-	if uploadRes.Error != nil {
-		util.GetLogger(req.Context()).WithError(uploadRes.Error).Error("Failed to PerformUploadKeys")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}
-	}
-	if len(uploadRes.KeyErrors) > 0 {
-		util.GetLogger(req.Context()).WithField("key_errors", uploadRes.KeyErrors).Error("Failed to upload one or more keys")
-		return util.JSONResponse{
-			Code: 400,
-			JSON: uploadRes.KeyErrors,
-		}
-	}
-
-	keyCount := make(map[string]int)
-	if len(uploadRes.OneTimeKeyCounts) > 0 {
-		keyCount = uploadRes.OneTimeKeyCounts[0].KeyCount
-	}
-	cryptoIDCount := make(map[string]int)
-	if len(uploadRes.OneTimeCryptoIDCounts) > 0 {
-		cryptoIDCount = uploadRes.OneTimeCryptoIDCounts[0].KeyCount
-	}
-	return util.JSONResponse{
-		Code: 200,
-		JSON: struct {
-			OTKCounts  interface{} `json:"one_time_key_counts"`
-			OTIDCounts interface{} `json:"one_time_cryptoid_counts"`
-		}{keyCount, cryptoIDCount},
-	}
-}
-
 func (r *queryKeysRequest) GetTimeout() time.Duration {
 	if r.Timeout == 0 {
 		return 10 * time.Second
@@ -205,7 +118,6 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) u
 		UserID:        device.UserID,
 		UserToDevices: r.DeviceKeys,
 		Timeout:       r.GetTimeout(),
-		// TODO: Token?
 	}, &queryRes)
 	return util.JSONResponse{
 		Code: 200,

clientapi/routing/leaveroom.go

@@ -15,7 +15,6 @@
 package routing
 
 import (
-	"encoding/json"
 	"net/http"
 
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@@ -24,16 +23,11 @@ import (
 	"github.com/matrix-org/util"
 )
 
-type leaveRoomCryptoIDsResponse struct {
-	PDU json.RawMessage `json:"pdu"`
-}
-
 func LeaveRoomByID(
 	req *http.Request,
 	device *api.Device,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	roomID string,
-	cryptoIDs bool,
 ) util.JSONResponse {
 	userID, err := spec.NewUserID(device.UserID, true)
 	if err != nil {
@@ -51,8 +45,7 @@ func LeaveRoomByID(
 	leaveRes := roomserverAPI.PerformLeaveResponse{}
 
 	// Ask the roomserver to perform the leave.
-	leaveEvent, err := rsAPI.PerformLeave(req.Context(), &leaveReq, &leaveRes, cryptoIDs)
-	if err != nil {
+	if err := rsAPI.PerformLeave(req.Context(), &leaveReq, &leaveRes); err != nil {
 		if leaveRes.Code != 0 {
 			return util.JSONResponse{
 				Code: leaveRes.Code,
@@ -65,15 +58,8 @@ func LeaveRoomByID(
 		}
 	}
 
-	if cryptoIDs {
-		return util.JSONResponse{
-			Code: http.StatusOK,
-			JSON: leaveRoomCryptoIDsResponse{
-				PDU: json.RawMessage(leaveEvent.JSON()),
-			},
-		}
-	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
+		JSON: struct{}{},
 	}
 }

clientapi/routing/login.go

@@ -19,6 +19,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -40,28 +41,25 @@ type flow struct {
 	Type string `json:"type"`
 }
 
-func passwordLogin() flows {
-	f := flows{}
-	s := flow{
-		Type: "m.login.password",
-	}
-	f.Flows = append(f.Flows, s)
-	return f
-}
-
 // Login implements GET and POST /login
 func Login(
 	req *http.Request, userAPI userapi.ClientUserAPI,
 	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
-		// TODO: support other forms of login other than password, depending on config options
+		loginFlows := []flow{{Type: authtypes.LoginTypePassword}}
+		if len(cfg.Derived.ApplicationServices) > 0 {
+			loginFlows = append(loginFlows, flow{Type: authtypes.LoginTypeApplicationService})
+		}
+		// TODO: support other forms of login, depending on config options
 		return util.JSONResponse{
 			Code: http.StatusOK,
-			JSON: passwordLogin(),
+			JSON: flows{
+				Flows: loginFlows,
+			},
 		}
 	} else if req.Method == http.MethodPost {
-		login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg)
+		login, cleanup, authErr := auth.LoginFromJSONReader(req, userAPI, userAPI, cfg)
 		if authErr != nil {
 			return *authErr
 		}

clientapi/routing/login_test.go

@@ -49,7 +49,7 @@ func TestLogin(t *testing.T) {
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for /login
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
 		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
@@ -114,6 +114,44 @@ func TestLogin(t *testing.T) {
 
 		ctx := context.Background()
 
+		// Inject a dummy application service, so we have a "m.login.application_service"
+		// in the login flows
+		as := &config.ApplicationService{}
+		cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as}
+
+		t.Run("Supported log-in flows are returned", func(t *testing.T) {
+			req := test.NewRequest(t, http.MethodGet, "/_matrix/client/v3/login")
+			rec := httptest.NewRecorder()
+			routers.Client.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("failed to get log-in flows: %s", rec.Body.String())
+			}
+
+			t.Logf("response: %s", rec.Body.String())
+			resp := flows{}
+			if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+				t.Fatal(err)
+			}
+
+			appServiceFound := false
+			passwordFound := false
+			for _, flow := range resp.Flows {
+				if flow.Type == "m.login.password" {
+					passwordFound = true
+				} else if flow.Type == "m.login.application_service" {
+					appServiceFound = true
+				} else {
+					t.Fatalf("got unknown login flow: %s", flow.Type)
+				}
+			}
+			if !appServiceFound {
+				t.Fatal("m.login.application_service missing from login flows")
+			}
+			if !passwordFound {
+				t.Fatal("m.login.password missing from login flows")
+			}
+		})
+
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
 				req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{

clientapi/routing/membership.go

@@ -17,7 +17,6 @@ package routing
 import (
 	"context"
 	"crypto/ed25519"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
@@ -40,15 +39,10 @@ import (
 	"github.com/matrix-org/util"
 )
 
-type membershipCryptoIDsResponse struct {
-	PDU json.RawMessage `json:"pdu"`
-}
-
 func SendBan(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
-	cryptoIDs bool,
 ) util.JSONResponse {
 	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
@@ -101,14 +95,13 @@ func SendBan(
 		}
 	}
 
-	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Ban, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI, cryptoIDs)
+	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Ban, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
 }
 
 func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
-	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, cryptoIDs bool) util.JSONResponse {
+	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI) util.JSONResponse {
 
-	// TODO: cryptoIDs - what about when we don't know the senderID for a user?
 	event, err := buildMembershipEvent(
 		ctx, targetUserID, reason, profileAPI, device, membership,
 		roomID, false, cfg, evTime, rsAPI, asAPI,
@@ -121,8 +114,6 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
 		}
 	}
 
-	var json interface{}
-	if !cryptoIDs {
 	serverName := device.UserDomain()
 	if err = roomserverAPI.SendEvents(
 		ctx, rsAPI,
@@ -140,14 +131,10 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
 			JSON: spec.InternalServerError{},
 		}
 	}
-		json = struct{}{}
-	} else {
-		json = membershipCryptoIDsResponse{PDU: event.JSON()}
-	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: json,
+		JSON: struct{}{},
 	}
 }
 
@@ -155,7 +142,6 @@ func SendKick(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
-	cryptoIDs bool,
 ) util.JSONResponse {
 	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
@@ -195,18 +181,6 @@ func SendKick(
 		return *errRes
 	}
 
-	pl, errRes := getPowerlevels(req, rsAPI, roomID)
-	if errRes != nil {
-		return *errRes
-	}
-	allowedToKick := pl.UserLevel(*senderID) >= pl.Kick
-	if !allowedToKick {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."),
-		}
-	}
-
 	bodyUserID, err := spec.NewUserID(body.UserID, true)
 	if err != nil {
 		return util.JSONResponse{
@@ -214,6 +188,19 @@ func SendKick(
 			JSON: spec.BadJSON("body userID is invalid"),
 		}
 	}
+
+	pl, errRes := getPowerlevels(req, rsAPI, roomID)
+	if errRes != nil {
+		return *errRes
+	}
+	allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String()
+	if !allowedToKick {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."),
+		}
+	}
+
 	var queryRes roomserverAPI.QueryMembershipForUserResponse
 	err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
 		RoomID: roomID,
@@ -230,14 +217,13 @@ func SendKick(
 		}
 	}
 	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI, cryptoIDs)
+	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
 }
 
 func SendUnban(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
-	cryptoIDs bool,
 ) util.JSONResponse {
 	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
@@ -287,14 +273,13 @@ func SendUnban(
 		}
 	}
 	// TODO: should we be using SendLeave instead?
-	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI, cryptoIDs)
+	return sendMembership(req.Context(), profileAPI, device, roomID, spec.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI)
 }
 
 func SendInvite(
 	req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
 	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
-	cryptoIDs bool,
 ) util.JSONResponse {
 	body, evTime, reqErr := extractRequestData(req)
 	if reqErr != nil {
@@ -339,20 +324,18 @@ func SendInvite(
 	}
 
 	// We already received the return value, so no need to check for an error here.
-	response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime, cryptoIDs)
+	response, _ := sendInvite(req.Context(), device, roomID, body.UserID, body.Reason, cfg, rsAPI, evTime)
 	return response
 }
 
 // sendInvite sends an invitation to a user. Returns a JSONResponse and an error
 func sendInvite(
 	ctx context.Context,
-	profileAPI userapi.ClientUserAPI,
 	device *userapi.Device,
 	roomID, userID, reason string,
 	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
-	cryptoIDs bool,
+	evTime time.Time,
 ) (util.JSONResponse, error) {
 	validRoomID, err := spec.NewRoomID(roomID)
 	if err != nil {
@@ -375,13 +358,7 @@ func sendInvite(
 			JSON: spec.InvalidParam("UserID is invalid"),
 		}, err
 	}
-	profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{},
-		}, err
-	}
+
 	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
 	if err != nil {
 		return util.JSONResponse{
@@ -389,13 +366,11 @@ func sendInvite(
 			JSON: spec.InternalServerError{},
 		}, err
 	}
-	inviteEvent, err := rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
+	err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
 		InviteInput: roomserverAPI.InviteInput{
 			RoomID:     *validRoomID,
 			Inviter:    *inviter,
 			Invitee:    *invitee,
-			DisplayName: profile.DisplayName,
-			AvatarURL:   profile.AvatarURL,
 			Reason:     reason,
 			IsDirect:   false,
 			KeyID:      identity.KeyID,
@@ -404,7 +379,7 @@ func sendInvite(
 		},
 		InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
 		SendAsServer:    string(device.UserDomain()),
-	}, cryptoIDs)
+	})
 
 	switch e := err.(type) {
 	case roomserverAPI.ErrInvalidID:
@@ -427,22 +402,10 @@ func sendInvite(
 		}, err
 	}
 
-	response := util.JSONResponse{
+	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
-	}
-
-	type inviteCryptoIDResponse struct {
-		PDU json.RawMessage `json:"pdu"`
-	}
-
-	if inviteEvent != nil {
-		response.JSON = inviteCryptoIDResponse{
-			PDU: json.RawMessage(inviteEvent.JSON()),
-		}
-	}
-
-	return response, nil
+	}, nil
 }
 
 func buildMembershipEventDirect(

clientapi/routing/memberships.go

@@ -0,0 +1,139 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
+type getJoinedMembersResponse struct {
+	Joined map[string]joinedMember `json:"joined"`
+}
+
+type joinedMember struct {
+	DisplayName string `json:"display_name"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// The database stores 'displayname' without an underscore.
+// Deserialize into this and then change to the actual API response
+type databaseJoinedMember struct {
+	DisplayName string `json:"displayname"`
+	AvatarURL   string `json:"avatar_url"`
+}
+
+// GetJoinedMembers implements
+//
+//	GET /rooms/{roomId}/joined_members
+func GetJoinedMembers(
+	req *http.Request, device *userapi.Device, roomID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	// Validate the userID
+	userID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("Device UserID is invalid"),
+		}
+	}
+
+	// Validate the roomID
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}
+	}
+
+	// Get the current memberships for the requesting user to determine
+	// if they are allowed to query this endpoint.
+	queryReq := api.QueryMembershipForUserRequest{
+		RoomID: validRoomID.String(),
+		UserID: *userID,
+	}
+
+	var queryRes api.QueryMembershipForUserResponse
+	if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil {
+		util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	if !queryRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	if !queryRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	// Get the current membership events
+	var membershipsForRoomResp api.QueryMembershipsForRoomResponse
+	if err = rsAPI.QueryMembershipsForRoom(req.Context(), &api.QueryMembershipsForRoomRequest{
+		JoinedOnly: true,
+		RoomID:     validRoomID.String(),
+	}, &membershipsForRoomResp); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed")
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}
+	}
+
+	var res getJoinedMembersResponse
+	res.Joined = make(map[string]joinedMember)
+	for _, ev := range membershipsForRoomResp.JoinEvents {
+		var content databaseJoinedMember
+		if err := json.Unmarshal(ev.Content, &content); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(ev.Sender))
+		if err != nil || userID == nil {
+			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{},
+			}
+		}
+
+		res.Joined[userID.String()] = joinedMember(content)
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}

clientapi/routing/password.go

@@ -73,7 +73,7 @@ func Password(
 
 	// Check if the existing password is correct.
 	typePassword := auth.LoginTypePassword{
-		GetAccountByPassword: userAPI.QueryAccountByPassword,
+		UserAPI: userAPI,
 		Config:  cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {

clientapi/routing/profile.go

@@ -300,7 +300,7 @@ func updateProfile(
 		}, e
 	}
 
-	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil {
+	if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil {
 		util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,

clientapi/routing/receipt.go

@@ -23,13 +23,12 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 
-	"github.com/matrix-org/dendrite/userapi/api"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
-func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
+func SetReceipt(req *http.Request, userAPI userapi.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
 	timestamp := spec.AsTimestamp(time.Now())
 	logrus.WithFields(logrus.Fields{
 		"roomID":      roomID,
@@ -54,13 +53,13 @@ func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *prod
 			}
 		}
 
-		dataReq := api.InputAccountDataRequest{
+		dataReq := userapi.InputAccountDataRequest{
 			UserID:      device.UserID,
 			DataType:    "m.fully_read",
 			RoomID:      roomID,
 			AccountData: data,
 		}
-		dataRes := api.InputAccountDataResponse{}
+		dataRes := userapi.InputAccountDataResponse{}
 		if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
 			util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
 			return util.ErrorResponse(err)

clientapi/routing/register.go

@@ -236,7 +236,7 @@ type authDict struct {
 	// TODO: Lots of custom keys depending on the type
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
+// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api
 type userInteractiveResponse struct {
 	Flows     []authtypes.Flow       `json:"flows"`
 	Completed []authtypes.LoginType  `json:"completed"`
@@ -256,7 +256,7 @@ func newUserInteractiveResponse(
 	}
 }
 
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 type registerResponse struct {
 	UserID      string `json:"user_id"`
 	AccessToken string `json:"access_token,omitempty"`
@@ -462,7 +462,7 @@ func validateApplicationService(
 }
 
 // Register processes a /register request.
-// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register
 func Register(
 	req *http.Request,
 	userAPI userapi.ClientUserAPI,
@@ -630,6 +630,7 @@ func handleGuestRegistration(
 		AccessToken:       token,
 		IPAddr:            req.RemoteAddr,
 		UserAgent:         req.UserAgent(),
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{
@@ -647,6 +648,16 @@ func handleGuestRegistration(
 	}
 }
 
+// localpartMatchesExclusiveNamespaces will check if a given username matches any
+// application service's exclusive users namespace
+func localpartMatchesExclusiveNamespaces(
+	cfg *config.ClientAPI,
+	localpart string,
+) bool {
+	userID := userutil.MakeUserID(localpart, cfg.Matrix.ServerName)
+	return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID)
+}
+
 // handleRegistrationFlow will direct and complete registration flow stages
 // that the client has requested.
 // nolint: gocyclo
@@ -695,7 +706,7 @@ func handleRegistrationFlow(
 	// If an access token is provided, ignore this check this is an appservice
 	// request and we will validate in validateApplicationService
 	if len(cfg.Derived.ApplicationServices) != 0 &&
-		UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
+		localpartMatchesExclusiveNamespaces(cfg, r.Username) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: spec.ASExclusive("This username is reserved by an application service."),
@@ -772,7 +783,7 @@ func handleApplicationServiceRegistration(
 
 	// Check application service register user request is valid.
 	// The application service's ID is returned if so.
-	appserviceID, err := validateApplicationService(
+	appserviceID, err := internal.ValidateApplicationServiceRequest(
 		cfg, r.Username, accessToken,
 	)
 	if err != nil {
@@ -909,6 +920,7 @@ func completeRegistration(
 		DeviceID:          deviceID,
 		IPAddr:            ipAddr,
 		UserAgent:         userAgent,
+		FromRegistration:  true,
 	}, &devRes)
 	if err != nil {
 		return util.JSONResponse{

clientapi/routing/register_test.go

@@ -298,13 +298,16 @@ func Test_register(t *testing.T) {
 		guestsDisabled       bool
 		enableRecaptcha      bool
 		captchaBody          string
-		wantResponse         util.JSONResponse
+		// in case of an error, the expected response
+		wantErrorResponse util.JSONResponse
+		// in case of success, the expected username assigned
+		wantUsername string
 	}{
 		{
 			name:           "disallow guests",
 			kind:           "guest",
 			guestsDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Forbidden(`Guest registration is disabled on "test"`),
 			},
@@ -312,11 +315,12 @@ func Test_register(t *testing.T) {
 		{
 			name:         "allow guests",
 			kind:         "guest",
+			wantUsername: "1",
 		},
 		{
 			name:      "unknown login type",
 			loginType: "im.not.known",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusNotImplemented,
 				JSON: spec.Unknown("unknown/unimplemented auth type"),
 			},
@@ -324,7 +328,7 @@ func Test_register(t *testing.T) {
 		{
 			name:                 "disabled registration",
 			registrationDisabled: true,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Forbidden(`Registration is disabled on "test"`),
 			},
@@ -334,15 +338,23 @@ func Test_register(t *testing.T) {
 			username:     "",
 			password:     "someRandomPassword",
 			forceEmpty:   true,
+			wantUsername: "2",
 		},
 		{
 			name:     "successful registration",
 			username: "success",
 		},
+		{
+			name:         "successful registration, sequential numeric ID",
+			username:     "",
+			password:     "someRandomPassword",
+			forceEmpty:   true,
+			wantUsername: "3",
+		},
 		{
 			name:     "failing registration - user already exists",
 			username: "success",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.UserInUse("Desired user ID is already taken."),
 			},
@@ -354,12 +366,12 @@ func Test_register(t *testing.T) {
 		{
 			name:              "invalid username",
 			username:          "#totalyNotValid",
-			wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
+			wantErrorResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
 		},
 		{
 			name:     "numeric username is forbidden",
 			username: "1337",
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.InvalidUsername("Numeric user IDs are reserved"),
 			},
@@ -367,7 +379,7 @@ func Test_register(t *testing.T) {
 		{
 			name:      "disabled recaptcha login",
 			loginType: authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: spec.Unknown(ErrCaptchaDisabled.Error()),
 			},
@@ -376,7 +388,7 @@ func Test_register(t *testing.T) {
 			name:            "enabled recaptcha, no response defined",
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: spec.BadJSON(ErrMissingResponse.Error()),
 			},
@@ -386,7 +398,7 @@ func Test_register(t *testing.T) {
 			enableRecaptcha: true,
 			loginType:       authtypes.LoginTypeRecaptcha,
 			captchaBody:     `notvalid`,
-			wantResponse: util.JSONResponse{
+			wantErrorResponse: util.JSONResponse{
 				Code: http.StatusUnauthorized,
 				JSON: spec.BadJSON(ErrInvalidCaptcha.Error()),
 			},
@@ -402,7 +414,7 @@ func Test_register(t *testing.T) {
 			enableRecaptcha:   true,
 			loginType:         authtypes.LoginTypeRecaptcha,
 			captchaBody:       `i should fail for other reasons`,
-			wantResponse:    util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}},
+			wantErrorResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}},
 		},
 	}
 
@@ -416,7 +428,7 @@ func Test_register(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
@@ -486,8 +498,8 @@ func Test_register(t *testing.T) {
 						t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows)
 					}
 				case spec.MatrixError:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse)
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
+						t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantErrorResponse)
 					}
 					return
 				case registerResponse:
@@ -505,6 +517,13 @@ func Test_register(t *testing.T) {
 					if r.DeviceID == "" {
 						t.Fatalf("missing deviceID in response")
 					}
+					// if an expected username is provided, assert that it is a match
+					if tc.wantUsername != "" {
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
+						if wantUserID != r.UserID {
+							t.Fatalf("unexpected userID: %s, want %s", r.UserID, wantUserID)
+						}
+					}
 					return
 				default:
 					t.Logf("Got response: %T", resp.JSON)
@@ -541,45 +560,30 @@ func Test_register(t *testing.T) {
 
 				resp = Register(req, userAPI, &cfg.ClientAPI)
 
-				switch resp.JSON.(type) {
-				case spec.InternalServerError:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
+				switch rr := resp.JSON.(type) {
+				case spec.InternalServerError, spec.MatrixError, util.JSONResponse:
+					if !reflect.DeepEqual(tc.wantErrorResponse, resp) {
+						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantErrorResponse)
 					}
 					return
-				case spec.MatrixError:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
-					}
-					return
-				case util.JSONResponse:
-					if !reflect.DeepEqual(tc.wantResponse, resp) {
-						t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
-					}
-					return
-				}
-
-				rr, ok := resp.JSON.(registerResponse)
-				if !ok {
-					t.Fatalf("expected a registerresponse, got %T", resp.JSON)
-				}
-
+				case registerResponse:
 					// validate the response
-				if tc.forceEmpty {
-					// when not supplying a username, one will be generated. Given this _SHOULD_ be
-					// the second user, set the username accordingly
-					reg.Username = "2"
-				}
-				wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test"))
+					if tc.wantUsername != "" {
+						// if an expected username is provided, assert that it is a match
+						wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test"))
 						if wantUserID != rr.UserID {
 							t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
 						}
+					}
 					if rr.DeviceID != *reg.DeviceID {
 						t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
 					}
 					if rr.AccessToken == "" {
 						t.Fatalf("missing accessToken in response")
 					}
+				default:
+					t.Fatalf("expected one of internalservererror, matrixerror, jsonresponse, registerresponse, got %T", resp.JSON)
+				}
 			})
 		}
 	})
@@ -596,7 +600,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
 		response := completeRegistration(
@@ -637,7 +641,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
 		rsAPI.SetFederationAPI(nil, nil)
-		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
+		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		expectedDisplayName := "rabbit"
 		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)

clientapi/routing/report_event.go

@@ -0,0 +1,93 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userAPI "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+)
+
+type reportEventRequest struct {
+	Reason string `json:"reason"`
+	Score  int64  `json:"score"`
+}
+
+func ReportEvent(
+	req *http.Request,
+	device *userAPI.Device,
+	roomID, eventID string,
+	rsAPI api.ClientRoomserverAPI,
+) util.JSONResponse {
+	defer req.Body.Close() // nolint: errcheck
+
+	deviceUserID, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: spec.NotFound("You don't have permission to report this event, bad userID"),
+		}
+	}
+	// The requesting user must be a member of the room
+	errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID)
+	if errRes != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound, // Spec demands this...
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	// Parse the request
+	report := reportEventRequest{}
+	if resErr := httputil.UnmarshalJSONRequest(req, &report); resErr != nil {
+		return *resErr
+	}
+
+	queryRes := &api.QueryEventsByIDResponse{}
+	if err = rsAPI.QueryEventsByID(req.Context(), &api.QueryEventsByIDRequest{
+		RoomID:   roomID,
+		EventIDs: []string{eventID},
+	}, queryRes); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	// No event was found or it was already redacted
+	if len(queryRes.Events) == 0 || queryRes.Events[0].Redacted() {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.NotFound("The event was not found or you are not joined to the room."),
+		}
+	}
+
+	_, err = rsAPI.InsertReportedEvent(req.Context(), roomID, eventID, device.UserID, report.Reason, report.Score)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{Err: err.Error()},
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/room_hierarchy.go

@@ -138,7 +138,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str
 		walker = *cachedWalker
 	}
 
-	discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
+	discoveredRooms, _, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
 
 	if err != nil {
 		switch err.(type) {

clientapi/routing/routing.go

@@ -255,7 +255,7 @@ func Setup(
 		logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
 		serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
 		if err != nil {
-			logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
+			logrus.WithError(err).Fatal("unable to get account for sending server notices")
 		}
 
 		synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
@@ -309,33 +309,9 @@ func Setup(
 
 	v3mux.Handle("/createRoom",
 		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return CreateRoom(req, device, cfg, userAPI, rsAPI, asAPI, false)
+			return CreateRoom(req, device, cfg, userAPI, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/createRoom",
-		httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/createRoom")
-			return CreateRoom(req, device, cfg, userAPI, rsAPI, asAPI, true)
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/send_pdus/{txnID}",
-		httputil.MakeAuthAPI("send_pdus", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/send_pdus")
-			if r := rateLimits.Limit(req, device); r != nil {
-				return *r
-			}
-
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			txnID := vars["txnID"]
-
-			// NOTE: when making events such as for create_room, multiple PDUs will need to be passed between the client & server.
-			return SendPDUs(req, device, cfg, userAPI, rsAPI, asAPI, &txnID)
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-
 	v3mux.Handle("/join/{roomIDOrAlias}",
 		httputil.MakeAuthAPI(spec.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
@@ -358,32 +334,8 @@ func Setup(
 			return resp.(util.JSONResponse)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/join/{roomIDOrAlias}",
-		httputil.MakeAuthAPI(spec.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/join/{roomIDOrAlias}")
-			if r := rateLimits.Limit(req, device); r != nil {
-				return *r
-			}
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			// Only execute a join for roomIDOrAlias and UserID once. If there is a join in progress
-			// it waits for it to complete and returns that result for subsequent requests.
-			resp, _, _ := sf.Do(vars["roomIDOrAlias"]+device.UserID, func() (any, error) {
-				return JoinRoomByIDOrAliasCryptoIDs(
-					req, device, rsAPI, userAPI, vars["roomIDOrAlias"],
-				), nil
-			})
-			// once all joins are processed, drop them from the cache. Further requests
-			// will be processed as usual.
-			sf.Forget(vars["roomIDOrAlias"] + device.UserID)
-			return resp.(util.JSONResponse)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPost, http.MethodOptions)
 
 	if mscCfg.Enabled("msc2753") {
-		// TODO: update for cryptoIDs
 		v3mux.Handle("/peek/{roomIDOrAlias}",
 			httputil.MakeAuthAPI(spec.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 				if r := rateLimits.Limit(req, device); r != nil {
@@ -426,29 +378,6 @@ func Setup(
 			return resp.(util.JSONResponse)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/join",
-		httputil.MakeAuthAPI(spec.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/join")
-			if r := rateLimits.Limit(req, device); r != nil {
-				return *r
-			}
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			// Only execute a join for roomID and UserID once. If there is a join in progress
-			// it waits for it to complete and returns that result for subsequent requests.
-			resp, _, _ := sf.Do(vars["roomID"]+device.UserID, func() (any, error) {
-				return JoinRoomByIDOrAliasCryptoIDs(
-					req, device, rsAPI, userAPI, vars["roomID"],
-				), nil
-			})
-			// once all joins are processed, drop them from the cache. Further requests
-			// will be processed as usual.
-			sf.Forget(vars["roomID"] + device.UserID)
-			return resp.(util.JSONResponse)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/leave",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
@@ -459,26 +388,10 @@ func Setup(
 				return util.ErrorResponse(err)
 			}
 			return LeaveRoomByID(
-				req, device, rsAPI, vars["roomID"], false,
+				req, device, rsAPI, vars["roomID"],
 			)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/leave",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/leave")
-			if r := rateLimits.Limit(req, device); r != nil {
-				return *r
-			}
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return LeaveRoomByID(
-				req, device, rsAPI, vars["roomID"], true,
-			)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPost, http.MethodOptions)
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/rooms/{roomID}/unpeek",
 		httputil.MakeAuthAPI("unpeek", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -496,17 +409,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendBan(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, false)
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/ban",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/ban")
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return SendBan(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, true)
+			return SendBan(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/invite",
@@ -518,20 +421,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendInvite(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, false)
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/invite",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/invite")
-			if r := rateLimits.Limit(req, device); r != nil {
-				return *r
-			}
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return SendInvite(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, true)
+			return SendInvite(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/kick",
@@ -540,17 +430,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendKick(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, false)
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/kick",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/kick")
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return SendKick(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, true)
+			return SendKick(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/unban",
@@ -559,17 +439,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return SendUnban(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, false)
-		}),
-	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/unban",
-		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/unban")
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return SendUnban(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI, true)
+			return SendUnban(req, userAPI, device, vars["roomID"], cfg, rsAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/send/{eventType}",
@@ -581,16 +451,6 @@ func Setup(
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, nil)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/send/{eventType}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/send/{eventType}")
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			return SendEventCryptoIDs(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, nil)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
 		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -602,18 +462,6 @@ func Setup(
 				nil, cfg, rsAPI, transactionsCache)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPut, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/send/{eventType}/{txnID}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/send/{eventType}/{txnID}")
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			txnID := vars["txnID"]
-			return SendEventCryptoIDs(req, device, vars["roomID"], vars["eventType"], &txnID,
-				nil, cfg, rsAPI, transactionsCache)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPut, http.MethodOptions)
 
 	v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -662,18 +510,6 @@ func Setup(
 			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPut, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/state/{eventType:[^/]+/?}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/state/{eventType:[^/]+/?}")
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			emptyString := ""
-			eventType := strings.TrimSuffix(vars["eventType"], "/")
-			return SendEventCryptoIDs(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPut, http.MethodOptions)
 
 	v3mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
 		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -685,17 +521,6 @@ func Setup(
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, nil)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPut, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/rooms/{roomID}/state/{eventType}/{stateKey}",
-		httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/rooms/{roomID}/state/{eventType}/{stateKey}")
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			if err != nil {
-				return util.ErrorResponse(err)
-			}
-			stateKey := vars["stateKey"]
-			return SendEventCryptoIDs(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, nil)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPut, http.MethodOptions)
 
 	// Defined outside of handler to persist between calls
 	// TODO: clear based on some criteria
@@ -734,7 +559,6 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/directory/room/{roomAlias}",
 		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -745,7 +569,6 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/directory/room/{roomAlias}",
 		httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -813,7 +636,6 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/rooms/{roomID}/typing/{userID}",
 		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
@@ -826,7 +648,6 @@ func Setup(
 			return SendTyping(req, device, vars["roomID"], vars["userID"], rsAPI, syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/rooms/{roomID}/redact/{eventID}",
 		httputil.MakeAuthAPI("rooms_redact", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -836,7 +657,6 @@ func Setup(
 			return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, nil, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/rooms/{roomID}/redact/{eventID}/{txnId}",
 		httputil.MakeAuthAPI("rooms_redact", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -848,7 +668,6 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/sendToDevice/{eventType}/{txnID}",
 		httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -1299,7 +1118,6 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/rooms/{roomID}/read_markers",
 		httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			if r := rateLimits.Limit(req, device); r != nil {
@@ -1326,7 +1144,6 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/rooms/{roomID}/upgrade",
 		httputil.MakeAuthAPI("rooms_upgrade", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -1631,7 +1448,7 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg)
+		return UploadCrossSigningDeviceKeys(req, userAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@@ -1655,12 +1472,6 @@ func Setup(
 			return UploadKeys(req, userAPI, device)
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPost, http.MethodOptions)
-	unstableMux.Handle("/org.matrix.msc4080/keys/upload",
-		httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			logrus.Info("Processing request to /org.matrix.msc4080/keys/upload")
-			return UploadKeysCryptoIDs(req, userAPI, device)
-		}, httputil.WithAllowGuests()),
-	).Methods(http.MethodPost, http.MethodOptions)
 	v3mux.Handle("/keys/query",
 		httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			return QueryKeys(req, userAPI, device)
@@ -1684,7 +1495,6 @@ func Setup(
 			return SetReceipt(req, userAPI, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
-	// TODO: update for cryptoIDs
 	v3mux.Handle("/presence/{userId}/status",
 		httputil.MakeAuthAPI("set_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -1703,4 +1513,58 @@ func Setup(
 			return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/joined_members",
+		httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetJoinedMembers(req, device, vars["roomID"], rsAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	v3mux.Handle("/rooms/{roomID}/report/{eventID}",
+		httputil.MakeAuthAPI("report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return ReportEvent(req, device, vars["roomID"], vars["eventID"], rsAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports",
+		httputil.MakeAdminAPI("admin_report_events", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			from := parseUint64OrDefault(req.URL.Query().Get("from"), 0)
+			limit := parseUint64OrDefault(req.URL.Query().Get("limit"), 100)
+			dir := req.URL.Query().Get("dir")
+			userID := req.URL.Query().Get("user_id")
+			roomID := req.URL.Query().Get("room_id")
+
+			// Go backwards if direction is empty or "b"
+			backwards := dir == "" || dir == "b"
+			return GetEventReports(req, rsAPI, from, limit, backwards, userID, roomID)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}",
+		httputil.MakeAdminAPI("admin_report_event_delete", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return DeleteEventReport(req, rsAPI, vars["reportID"])
+		}),
+	).Methods(http.MethodDelete, http.MethodOptions)
 }

clientapi/routing/send_pdus.go

@@ -1,272 +0,0 @@
-// Copyright 2023 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package routing
-
-import (
-	"encoding/json"
-	"net/http"
-	"sync"
-	"time"
-
-	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
-	"github.com/matrix-org/dendrite/clientapi/httputil"
-	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/types"
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-type PDUInfo struct {
-	Version   string          `json:"room_version"`
-	ViaServer string          `json:"via_server,omitempty"`
-	PDU       json.RawMessage `json:"pdu"`
-}
-
-type sendPDUsRequest struct {
-	PDUs []PDUInfo `json:"pdus"`
-}
-
-// SendPDUs implements /sendPDUs
-// nolint:gocyclo
-func SendPDUs(
-	req *http.Request, device *api.Device,
-	cfg *config.ClientAPI,
-	profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
-	asAPI appserviceAPI.AppServiceInternalAPI,
-	txnID *string,
-) util.JSONResponse {
-	// TODO: cryptoIDs - should this include an "eventType"?
-	// if it's a bulk send endpoint, I don't think that makes any sense since there are multiple event types
-	// In that case, how do I know how to treat the events?
-	// I could sort them all by roomID?
-	// Then filter them down based on event type? (how do I collect groups of events such as for room creation?)
-	// Possibly based on event hash tracking that I know were sent to the client?
-	// For createRoom, I know what the possible list of events are, so try to find those and collect them to send to room creation.
-	// Could also sort by depth... but that seems dangerous and depth may not be a field forever
-	// Does it matter at all?
-	// Can't I just forward all the events to the roomserver?
-	// Do I need to do any specific processing on them?
-
-	var pdus sendPDUsRequest
-	resErr := httputil.UnmarshalJSONRequest(req, &pdus)
-	if resErr != nil {
-		return *resErr
-	}
-
-	userID, err := spec.NewUserID(device.UserID, true)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam(err.Error()),
-		}
-	}
-
-	// create a mutex for the specific user in the specific room
-	// this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order
-	// TODO: cryptoIDs - where to get roomID from?
-	mutex, _ := userRoomSendMutexes.LoadOrStore("roomID"+userID.String(), &sync.Mutex{})
-	mutex.(*sync.Mutex).Lock()
-	defer mutex.(*sync.Mutex).Unlock()
-
-	inputs := make([]roomserverAPI.InputRoomEvent, 0, len(pdus.PDUs))
-	for _, event := range pdus.PDUs {
-		// TODO: cryptoIDs - event hash check?
-		verImpl, err := gomatrixserverlib.GetRoomVersion(gomatrixserverlib.RoomVersion(event.Version))
-		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{Err: err.Error()},
-			}
-		}
-		//eventJSON, err := json.Marshal(event)
-		//if err != nil {
-		//	return util.JSONResponse{
-		//		Code: http.StatusInternalServerError,
-		//		JSON: spec.InternalServerError{Err: err.Error()},
-		//	}
-		//}
-		// TODO: cryptoIDs - how should we be converting to a PDU here?
-		// if the hash matches an event we sent to the client, then the JSON should be good.
-		// But how do we know how to fill out if the event is redacted if we use the trustedJSON function?
-		// Also - untrusted JSON seems better - except it strips off the unsigned field?
-		// Also - gmsl events don't store the `hashes` field... problem?
-
-		pdu, err := verImpl.NewEventFromUntrustedJSON(event.PDU)
-		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{Err: err.Error()},
-			}
-		}
-
-		util.GetLogger(req.Context()).Infof("Processing %s event (%s): %s", pdu.Type(), pdu.EventID(), pdu.JSON())
-
-		// Check that the event is signed by the server sending the request.
-		redacted, err := verImpl.RedactEventJSON(pdu.JSON())
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("RedactEventJSON failed")
-			continue
-		}
-
-		verifier := gomatrixserverlib.JSONVerifierSelf{}
-		verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
-			ServerName:           spec.ServerName(pdu.SenderID()),
-			Message:              redacted,
-			AtTS:                 pdu.OriginServerTS(),
-			ValidityCheckingFunc: gomatrixserverlib.StrictValiditySignatureCheck,
-		}}
-		verifyResults, err := verifier.VerifyJSONs(req.Context(), verifyRequests)
-		if err != nil {
-			util.GetLogger(req.Context()).WithError(err).Error("keys.VerifyJSONs failed")
-			continue
-		}
-		if verifyResults[0].Error != nil {
-			util.GetLogger(req.Context()).WithError(verifyResults[0].Error).Error("Signature check failed: ")
-			continue
-		}
-
-		switch pdu.Type() {
-		case spec.MRoomCreate:
-		case spec.MRoomMember:
-			var membership gomatrixserverlib.MemberContent
-			err = json.Unmarshal(pdu.Content(), &membership)
-			switch {
-			case err != nil:
-				util.GetLogger(req.Context()).Errorf("m.room.member event (%s) content invalid: %v", pdu.EventID(), pdu.Content())
-				continue
-			case membership.Membership == spec.Join:
-				deviceUserID, err := spec.NewUserID(device.UserID, true)
-				if err != nil {
-					return util.JSONResponse{
-						Code: http.StatusForbidden,
-						JSON: spec.Forbidden("userID doesn't have power level to change visibility"),
-					}
-				}
-				if !cfg.Matrix.IsLocalServerName(pdu.RoomID().Domain()) {
-					queryReq := roomserverAPI.QueryMembershipForUserRequest{
-						RoomID: pdu.RoomID().String(),
-						UserID: *deviceUserID,
-					}
-					var queryRes roomserverAPI.QueryMembershipForUserResponse
-					if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil {
-						util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
-						return util.JSONResponse{
-							Code: http.StatusInternalServerError,
-							JSON: spec.InternalServerError{},
-						}
-					}
-					if !queryRes.IsInRoom {
-						// This is a join event to a remote room
-						// TODO: cryptoIDs - figure out how to obtain unsigned contents for outstanding federated invites
-						joinReq := roomserverAPI.PerformJoinRequestCryptoIDs{
-							RoomID:      pdu.RoomID().String(),
-							UserID:      device.UserID,
-							IsGuest:     device.AccountType == api.AccountTypeGuest,
-							ServerNames: []spec.ServerName{spec.ServerName(event.ViaServer)},
-							JoinEvent:   pdu,
-						}
-						err := rsAPI.PerformSendJoinCryptoIDs(req.Context(), &joinReq)
-						if err != nil {
-							util.GetLogger(req.Context()).Errorf("Failed processing %s event (%s): %v", pdu.Type(), pdu.EventID(), err)
-						}
-						continue // NOTE: don't send this event on to the roomserver
-					}
-				}
-			case membership.Membership == spec.Invite:
-				stateKey := pdu.StateKey()
-				if stateKey == nil {
-					return util.JSONResponse{
-						Code: http.StatusForbidden,
-						JSON: spec.Forbidden("invalid state_key for membership event"),
-					}
-				}
-				invitedUserID, err := rsAPI.QueryUserIDForSender(req.Context(), pdu.RoomID(), spec.SenderID(*stateKey))
-				if err != nil || invitedUserID == nil {
-					return util.JSONResponse{
-						Code: http.StatusNotFound,
-						JSON: spec.NotFound("cannot find userID for invite event"),
-					}
-				}
-				if !cfg.Matrix.IsLocalServerName(spec.ServerName(invitedUserID.Domain())) {
-					inviteReq := roomserverAPI.PerformInviteRequestCryptoIDs{
-						RoomID:      pdu.RoomID().String(),
-						UserID:      *invitedUserID,
-						InviteEvent: pdu,
-					}
-					err := rsAPI.PerformSendInviteCryptoIDs(req.Context(), &inviteReq)
-					if err != nil {
-						util.GetLogger(req.Context()).Errorf("Failed processing %s event (%s): %v", pdu.Type(), pdu.EventID(), err)
-					}
-					continue // NOTE: don't send this event on to the roomserver
-				}
-			}
-		}
-
-		// TODO: cryptoIDs - does it matter which order these are added?
-		// yes - if the events are for room creation.
-		// Could make this a client requirement? ie. events are processed based on the order they appear
-		// We need to check event validity after processing each event.
-		// ie. what if the client changes power levels that disallow further events they sent?
-		// We should be doing this already as part of `SendInputRoomEvents`, but how should we pass this
-		// failure back to the client?
-
-		var transactionID *roomserverAPI.TransactionID
-		if txnID != nil {
-			transactionID = &roomserverAPI.TransactionID{
-				SessionID: device.SessionID, TransactionID: *txnID,
-			}
-		}
-
-		inputs = append(inputs, roomserverAPI.InputRoomEvent{
-			Kind:   roomserverAPI.KindNew,
-			Event:  &types.HeaderedEvent{PDU: pdu},
-			Origin: userID.Domain(),
-			// TODO: cryptoIDs - what to do with this field?
-			// should probably generate this based on the event type being sent?
-			//SendAsServer: roomserverAPI.DoNotSendToOtherServers,
-			TransactionID: transactionID,
-		})
-	}
-
-	startedSubmittingEvents := time.Now()
-	// send the events to the roomserver
-	if err := roomserverAPI.SendInputRoomEvents(req.Context(), rsAPI, userID.Domain(), inputs, false); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed")
-		return util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.InternalServerError{Err: err.Error()},
-		}
-	}
-	timeToSubmitEvents := time.Since(startedSubmittingEvents)
-	sendEventDuration.With(prometheus.Labels{"action": "submit"}).Observe(float64(timeToSubmitEvents.Milliseconds()))
-
-	// Add response to transactionsCache
-	if txnID != nil {
-		// TODO : cryptoIDs - fix this
-		//res := util.JSONResponse{
-		//	Code: http.StatusOK,
-		//	JSON: sendEventResponse{e.EventID()},
-		//}
-		//txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res)
-	}
-
-	return util.JSONResponse{
-		Code: http.StatusOK,
-	}
-}

clientapi/routing/sendevent.go

@@ -32,7 +32,6 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/synctypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -45,11 +44,6 @@ type sendEventResponse struct {
 	EventID string `json:"event_id"`
 }
 
-type sendEventResponseCryptoIDs struct {
-	EventID string          `json:"event_id"`
-	PDU     json.RawMessage `json:"pdu"`
-}
-
 var (
 	userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents
 )
@@ -100,7 +94,7 @@ func SendEvent(
 	}
 
 	// Translate user ID state keys to room keys in pseudo ID rooms
-	if (roomVersion == gomatrixserverlib.RoomVersionPseudoIDs || roomVersion == gomatrixserverlib.RoomVersionCryptoIDs) && stateKey != nil {
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && stateKey != nil {
 		parsedRoomID, innerErr := spec.NewRoomID(roomID)
 		if innerErr != nil {
 			return util.JSONResponse{
@@ -154,7 +148,7 @@ func SendEvent(
 	}
 
 	// for power level events we need to replace the userID with the pseudoID
-	if (roomVersion == gomatrixserverlib.RoomVersionPseudoIDs || roomVersion == gomatrixserverlib.RoomVersionCryptoIDs) && eventType == spec.MRoomPowerLevels {
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && eventType == spec.MRoomPowerLevels {
 		err = updatePowerLevels(req, r, roomID, rsAPI)
 		if err != nil {
 			return util.JSONResponse{
@@ -172,7 +166,7 @@ func SendEvent(
 		}
 	}
 
-	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, rsAPI, evTime, false)
+	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, rsAPI, evTime)
 	if resErr != nil {
 		return *resErr
 	}
@@ -230,7 +224,7 @@ func SendEvent(
 		req.Context(), rsAPI,
 		api.KindNew,
 		[]*types.HeaderedEvent{
-			&types.HeaderedEvent{PDU: e},
+			{PDU: e},
 		},
 		device.UserDomain(),
 		domain,
@@ -268,156 +262,6 @@ func SendEvent(
 	return res
 }
 
-// SendEventCryptoIDs implements:
-//
-//	/rooms/{roomID}/send/{eventType}
-//	/rooms/{roomID}/send/{eventType}/{txnID}
-//	/rooms/{roomID}/state/{eventType}/{stateKey}
-//
-// nolint: gocyclo
-func SendEventCryptoIDs(
-	req *http.Request,
-	device *userapi.Device,
-	roomID, eventType string, txnID, stateKey *string,
-	cfg *config.ClientAPI,
-	rsAPI api.ClientRoomserverAPI,
-	txnCache *transactions.Cache,
-) util.JSONResponse {
-	roomVersion, err := rsAPI.QueryRoomVersionForRoom(req.Context(), roomID)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.UnsupportedRoomVersion(err.Error()),
-		}
-	}
-
-	if txnID != nil {
-		// Try to fetch response from transactionsCache
-		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
-			return *res
-		}
-	}
-
-	// Translate user ID state keys to room keys in pseudo ID rooms
-	if (roomVersion == gomatrixserverlib.RoomVersionPseudoIDs || roomVersion == gomatrixserverlib.RoomVersionCryptoIDs) && stateKey != nil {
-		parsedRoomID, innerErr := spec.NewRoomID(roomID)
-		if innerErr != nil {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.InvalidParam("invalid room ID"),
-			}
-		}
-
-		newStateKey, innerErr := synctypes.FromClientStateKey(*parsedRoomID, *stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
-			return rsAPI.QuerySenderIDForUser(req.Context(), roomID, userID)
-		})
-		if innerErr != nil {
-			// TODO: work out better logic for failure cases (e.g. sender ID not found)
-			util.GetLogger(req.Context()).WithError(innerErr).Error("synctypes.FromClientStateKey failed")
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.Unknown("internal server error"),
-			}
-		}
-		stateKey = newStateKey
-	}
-
-	var r map[string]interface{} // must be a JSON object
-	resErr := httputil.UnmarshalJSONRequest(req, &r)
-	if resErr != nil {
-		return *resErr
-	}
-
-	if stateKey != nil {
-		// If the existing/new state content are equal, return the existing event_id, making the request idempotent.
-		if resp := stateEqual(req.Context(), rsAPI, eventType, *stateKey, roomID, r); resp != nil {
-			return *resp
-		}
-	}
-
-	startedGeneratingEvent := time.Now()
-
-	// If we're sending a membership update, make sure to strip the authorised
-	// via key if it is present, otherwise other servers won't be able to auth
-	// the event if the room is set to the "restricted" join rule.
-	if eventType == spec.MRoomMember {
-		delete(r, "join_authorised_via_users_server")
-	}
-
-	// for power level events we need to replace the userID with the pseudoID
-	if (roomVersion == gomatrixserverlib.RoomVersionPseudoIDs || roomVersion == gomatrixserverlib.RoomVersionCryptoIDs) && eventType == spec.MRoomPowerLevels {
-		err = updatePowerLevels(req, r, roomID, rsAPI)
-		if err != nil {
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{Err: err.Error()},
-			}
-		}
-	}
-
-	evTime, err := httputil.ParseTSParam(req)
-	if err != nil {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.InvalidParam(err.Error()),
-		}
-	}
-
-	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, rsAPI, evTime, true)
-	if resErr != nil {
-		return *resErr
-	}
-	timeToGenerateEvent := time.Since(startedGeneratingEvent)
-
-	// validate that the aliases exists
-	if eventType == spec.MRoomCanonicalAlias && stateKey != nil && *stateKey == "" {
-		aliasReq := api.AliasEvent{}
-		if err = json.Unmarshal(e.Content(), &aliasReq); err != nil {
-			return util.ErrorResponse(fmt.Errorf("unable to parse alias event: %w", err))
-		}
-		if !aliasReq.Valid() {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.InvalidParam("Request contains invalid aliases."),
-			}
-		}
-		aliasRes := &api.GetAliasesForRoomIDResponse{}
-		if err = rsAPI.GetAliasesForRoomID(req.Context(), &api.GetAliasesForRoomIDRequest{RoomID: roomID}, aliasRes); err != nil {
-			return util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{},
-			}
-		}
-		var found int
-		requestAliases := append(aliasReq.AltAliases, aliasReq.Alias)
-		for _, alias := range aliasRes.Aliases {
-			for _, altAlias := range requestAliases {
-				if altAlias == alias {
-					found++
-				}
-			}
-		}
-		// check that we found at least the same amount of existing aliases as are in the request
-		if aliasReq.Alias != "" && found < len(requestAliases) {
-			return util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.BadAlias("No matching alias found."),
-			}
-		}
-	}
-	sendEventDuration.With(prometheus.Labels{"action": "build"}).Observe(float64(timeToGenerateEvent.Milliseconds()))
-
-	res := util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: sendEventResponseCryptoIDs{
-			EventID: e.EventID(),
-			PDU:     json.RawMessage(e.JSON()),
-		},
-	}
-
-	return res
-}
-
 func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error {
 	users, ok := r["users"]
 	if !ok {
@@ -485,7 +329,6 @@ func generateSendEvent(
 	roomID, eventType string, stateKey *string,
 	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
-	cryptoIDs bool,
 ) (gomatrixserverlib.PDU, *util.JSONResponse) {
 	// parse the incoming http request
 	fullUserID, err := spec.NewUserID(device.UserID, true)
@@ -533,19 +376,13 @@ func generateSendEvent(
 		}
 	}
 
-	var identity fclient.SigningIdentity
-	if !cryptoIDs {
-		id, idErr := rsAPI.SigningIdentityFor(ctx, *validRoomID, *fullUserID)
-		if idErr != nil {
+	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *fullUserID)
+	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: spec.InternalServerError{},
 		}
 	}
-		identity = id
-	} else {
-		identity.ServerName = spec.ServerName(*senderID)
-	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
 	e, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, &queryRes)

clientapi/routing/server_notices.go

@@ -169,7 +169,7 @@ func SendServerNotice(
 			PowerLevelContentOverride: pl,
 		}
 
-		roomRes := createRoom(ctx, crReq, senderDevice, cfgClient, userAPI, rsAPI, asAPI, time.Now(), false)
+		roomRes := createRoom(ctx, crReq, senderDevice, cfgClient, userAPI, rsAPI, asAPI, time.Now())
 
 		switch data := roomRes.JSON.(type) {
 		case createRoomResponse:
@@ -215,7 +215,7 @@ func SendServerNotice(
 		}
 		if !membershipRes.IsInRoom {
 			// re-invite the user
-			res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now(), false)
+			res, err := sendInvite(ctx, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, time.Now())
 			if err != nil {
 				return res
 			}
@@ -228,7 +228,7 @@ func SendServerNotice(
 		"body":    r.Content.Body,
 		"msgtype": r.Content.MsgType,
 	}
-	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, rsAPI, time.Now(), false)
+	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, rsAPI, time.Now())
 	if resErr != nil {
 		logrus.Errorf("failed to send message: %+v", resErr)
 		return *resErr

clientapi/routing/state.go

@@ -198,7 +198,6 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
 // If eventFormat=true, sends the whole event else just the content.
-// nolint:gocyclo
 func OnIncomingStateTypeRequest(
 	ctx context.Context, device *userapi.Device, rsAPI api.ClientRoomserverAPI,
 	roomID, evType, stateKey string, eventFormat bool,
@@ -215,7 +214,7 @@ func OnIncomingStateTypeRequest(
 	}
 
 	// Translate user ID state keys to room keys in pseudo ID rooms
-	if roomVer == gomatrixserverlib.RoomVersionPseudoIDs || roomVer == gomatrixserverlib.RoomVersionCryptoIDs {
+	if roomVer == gomatrixserverlib.RoomVersionPseudoIDs {
 		parsedRoomID, err := spec.NewRoomID(roomID)
 		if err != nil {
 			return util.JSONResponse{

cmd/dendrite-demo-pinecone/monolith/monolith.go

@@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite(
 	)
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 

cmd/dendrite-demo-pinecone/relay/retriever.go

@@ -17,13 +17,13 @@ package relay
 import (
 	"context"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 )
 
 const (
@@ -54,7 +54,7 @@ func NewRelayServerRetriever(
 		federationAPI:       federationAPI,
 		relayAPI:            relayAPI,
 		relayServersQueried: make(map[spec.ServerName]bool),
-		running:             *atomic.NewBool(false),
+		running:             atomic.Bool{},
 		quit:                quit,
 	}
 }

cmd/dendrite-demo-yggdrasil/main.go

@@ -213,14 +213,15 @@ func main() {
 	natsInstance := jetstream.NATSInstance{}
 	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation)
-
-	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
-	rsAPI.SetAppserviceAPI(asAPI)
 	fsAPI := federationapi.NewInternalAPI(
 		processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true,
 	)
 
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
+
+	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
+	rsAPI.SetAppserviceAPI(asAPI)
+
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
 	monolith := setup.Monolith{

cmd/dendrite/main.go

@@ -162,7 +162,7 @@ func main() {
 	// dependency. Other components also need updating after their dependencies are up.
 	rsAPI.SetFederationAPI(fsAPI, keyRing)
 
-	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient)
+	userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff)
 	asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
 
 	rsAPI.SetAppserviceAPI(asAPI)

cmd/resolve-state/main.go

@@ -11,13 +11,11 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/setup/jetstream"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -35,6 +33,19 @@ var roomVersion = flag.String("roomversion", "5", "the room version to parse eve
 var filterType = flag.String("filtertype", "", "the event types to filter on")
 var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots")
 
+// dummyQuerier implements QuerySenderIDAPI. Does **NOT** do any "magic" for pseudoID rooms
+// to avoid having to "start" a full roomserver API.
+type dummyQuerier struct{}
+
+func (d dummyQuerier) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) {
+	s := spec.SenderIDFromUserID(userID)
+	return &s, nil
+}
+
+func (d dummyQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	return senderID.ToUserID(), nil
+}
+
 // nolint:gocyclo
 func main() {
 	ctx := context.Background()
@@ -56,27 +67,32 @@ func main() {
 		}
 	}
 
-	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
-
 	processCtx := process.NewProcessContext()
 	cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+
+	dbOpts := cfg.RoomServer.Database
+	if dbOpts.ConnectionString == "" {
+		dbOpts = cfg.Global.DatabaseOptions
+	}
+
+	fmt.Println("Opening database")
 	roomserverDB, err := storage.Open(
-		processCtx.Context(), cm, &cfg.RoomServer.Database,
-		caching.NewRistrettoCache(128*1024*1024, time.Hour, true),
+		processCtx.Context(), cm, &dbOpts,
+		caching.NewRistrettoCache(8*1024*1024, time.Minute*5, caching.DisableMetrics),
 	)
 	if err != nil {
 		panic(err)
 	}
 
-	natsInstance := &jetstream.NATSInstance{}
-	rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm,
-		natsInstance, caching.NewRistrettoCache(128*1024*1024, time.Hour, true), false)
+	rsAPI := dummyQuerier{}
 
 	roomInfo := &types.RoomInfo{
 		RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion),
 	}
 	stateres := state.NewStateResolution(roomserverDB, roomInfo, rsAPI)
 
+	fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs")
+
 	if *difference {
 		if len(snapshotNIDs) != 2 {
 			panic("need exactly two state snapshot NIDs to calculate difference")
@@ -186,12 +202,25 @@ func main() {
 		authEvents[i] = authEventEntries[i].PDU
 	}
 
+	// Get the roomNID
+	roomInfo, err = roomserverDB.RoomInfo(ctx, authEvents[0].RoomID().String())
+	if err != nil {
+		panic(err)
+	}
+
 	fmt.Println("Resolving state")
 	var resolved Events
 	resolved, err = gomatrixserverlib.ResolveConflicts(
 		gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		},
+		func(eventID string) bool {
+			isRejected, rejectedErr := roomserverDB.IsEventRejected(ctx, roomInfo.RoomNID, eventID)
+			if rejectedErr != nil {
+				return true
+			}
+			return isRejected
+		},
 	)
 	if err != nil {
 		panic(err)

dendrite-sample.yaml

@@ -154,6 +154,13 @@ app_service_api:
   # to be sent to an insecure endpoint.
   disable_tls_validation: false
 
+  # Send the access_token query parameter with appservice requests in addition
+  # to the Authorization header. This can cause hs_tokens to be saved to logs,
+  # so it should not be enabled unless absolutely necessary.
+  legacy_auth: false
+  # Use the legacy unprefixed paths for appservice requests.
+  legacy_paths: false
+
   # Appservice configuration files to load into this homeserver.
   config_files:
   #  - /path/to/appservice_registration.yaml
@@ -325,6 +332,10 @@ user_api:
   auto_join_rooms:
   #  - "#main:matrix.org"
 
+  # The number of workers to start for the DeviceListUpdater. Defaults to 8.
+  # This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely.
+  # worker_count: 8
+
 # Configuration for Opentracing.
 # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
 # how this works and how to set it up.

docs/FAQ.md

@@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo
 
 Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be 
 specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting 
-the development efforts through [contributing](../development/contributing). 
+the development efforts through [contributing](./development/CONTRIBUTING.md). 
 
 ## Is there a migration path from Synapse to Dendrite?
 
@@ -105,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom <ve
 
 ## How do I reset somebody's password on my server?
 
-Use the admin endpoint [resetpassword](./administration/adminapi#post-_dendriteadminresetpassworduserid)
+Use the admin endpoint [resetpassword](./administration/4_adminapi.md#post-_dendriteadminresetpassworduserid)
 
 ## Should I use PostgreSQL or SQLite for my databases?
 
@@ -117,6 +117,7 @@ The list of files that need to be stored is:
 - matrix-key.pem
 - dendrite.yaml
 - the postgres or sqlite DB
+- the jetstream directory
 - the media store
 - the search index (although this can be regenerated)
 

docs/Gemfile.lock

@@ -231,9 +231,9 @@ GEM
       jekyll-seo-tag (~> 2.1)
     minitest (5.17.0)
     multipart-post (2.1.1)
-    nokogiri (1.14.3-arm64-darwin)
+    nokogiri (1.16.2-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.3-x86_64-linux)
+    nokogiri (1.16.2-x86_64-linux)
       racc (~> 1.4)
     octokit (4.22.0)
       faraday (>= 0.9)
@@ -241,7 +241,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
-    racc (1.6.2)
+    racc (1.7.3)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)

docs/development/CONTRIBUTING.md

@@ -109,7 +109,7 @@ To configure the connection to a remote Postgres, you can use the following envi
 
 ```bash
 POSTGRES_USER=postgres
-POSTGERS_PASSWORD=yourPostgresPassword
+POSTGRES_PASSWORD=yourPostgresPassword
 POSTGRES_HOST=localhost
 POSTGRES_DB=postgres # the superuser database to use
 ```

docs/installation/docker/1_docker.md

@@ -26,6 +26,8 @@ docker run --rm --entrypoint="/usr/bin/generate-keys" \
   -v $(pwd)/config:/mnt \
   matrixdotorg/dendrite-monolith:latest \
   -private-key /mnt/matrix_key.pem
+
+# Windows equivalent: docker run --rm --entrypoint="/usr/bin/generate-keys" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -private-key /mnt/matrix_key.pem
 ```
 (**NOTE**: This only needs to be executed **once**, as you otherwise overwrite the key)
 
@@ -44,6 +46,8 @@ docker run --rm --entrypoint="/bin/sh" \
     -dir /var/dendrite/ \
     -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable \
     -server YourDomainHere > /mnt/dendrite.yaml"
+
+# Windows equivalent: docker run --rm --entrypoint="/bin/sh" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -c "/usr/bin/generate-config -dir /var/dendrite/ -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable -server YourDomainHere > /mnt/dendrite.yaml"
 ```
 
 You can then change `config/dendrite.yaml` to your liking.

federationapi/api/api.go

@@ -58,18 +58,12 @@ type RoomserverFederationAPI interface {
 	PerformDirectoryLookup(ctx context.Context, request *PerformDirectoryLookupRequest, response *PerformDirectoryLookupResponse) error
 	// Handle an instruction to make_join & send_join with a remote server.
 	PerformJoin(ctx context.Context, request *PerformJoinRequest, response *PerformJoinResponse)
-	PerformMakeJoin(ctx context.Context, request *PerformJoinRequest) (gomatrixserverlib.PDU, gomatrixserverlib.RoomVersion, spec.ServerName, error)
-	PerformSendJoin(ctx context.Context, request *PerformSendJoinRequestCryptoIDs, response *PerformJoinResponse)
 	// Handle an instruction to make_leave & send_leave with a remote server.
-	PerformLeave(ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse, cryptoIDs bool) error
+	PerformLeave(ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse) error
 	// Handle sending an invite to a remote server.
 	SendInvite(ctx context.Context, event gomatrixserverlib.PDU, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error)
 	// Handle sending an invite to a remote server.
 	SendInviteV3(ctx context.Context, event gomatrixserverlib.ProtoEvent, invitee spec.UserID, version gomatrixserverlib.RoomVersion, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error)
-	// Handle sending an invite to a remote server.
-	MakeInviteCryptoIDs(ctx context.Context, event gomatrixserverlib.ProtoEvent, invitee spec.UserID, version gomatrixserverlib.RoomVersion, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error)
-	// Handle sending an invite to a remote server.
-	SendInviteCryptoIDs(ctx context.Context, event gomatrixserverlib.PDU, invitee spec.UserID, version gomatrixserverlib.RoomVersion) error
 	// Handle an instruction to peek a room on a remote server.
 	PerformOutboundPeek(ctx context.Context, request *PerformOutboundPeekRequest, response *PerformOutboundPeekResponse) error
 	// Query the server names of the joined hosts in a room.
@@ -174,15 +168,6 @@ type PerformJoinRequest struct {
 	Unsigned    map[string]interface{} `json:"unsigned"`
 }
 
-type PerformSendJoinRequestCryptoIDs struct {
-	RoomID string `json:"room_id"`
-	UserID string `json:"user_id"`
-	// The sorted list of servers to try. Servers will be tried sequentially, after de-duplication.
-	ServerNames types.ServerNames      `json:"server_names"`
-	Unsigned    map[string]interface{} `json:"unsigned"`
-	Event       gomatrixserverlib.PDU
-}
-
 type PerformJoinResponse struct {
 	JoinedVia spec.ServerName
 	LastError *gomatrix.HTTPError

federationapi/federationapi.go

@@ -24,7 +24,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/sirupsen/logrus"
 
-	"github.com/matrix-org/dendrite/federationapi/api"
 	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
 	"github.com/matrix-org/dendrite/federationapi/internal"
@@ -102,7 +101,7 @@ func NewInternalAPI(
 	caches *caching.Caches,
 	keyRing *gomatrixserverlib.KeyRing,
 	resetBlacklist bool,
-) api.FederationInternalAPI {
+) *internal.FederationInternalAPI {
 	cfg := &dendriteCfg.FederationAPI
 
 	federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName)
@@ -114,10 +113,7 @@ func NewInternalAPI(
 		_ = federationDB.RemoveAllServersFromBlacklist()
 	}
 
-	stats := statistics.NewStatistics(
-		federationDB,
-		cfg.FederationMaxRetries+1,
-		cfg.P2PFederationRetriesUntilAssumedOffline+1)
+	stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1, cfg.P2PFederationRetriesUntilAssumedOffline+1, cfg.EnableRelays)
 
 	js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)
 
@@ -126,7 +122,7 @@ func NewInternalAPI(
 	queues := queue.NewOutgoingQueues(
 		federationDB, processContext,
 		cfg.Matrix.DisableFederation,
-		cfg.Matrix.ServerName, federation, rsAPI, &stats,
+		cfg.Matrix.ServerName, federation, &stats,
 		signingInfo,
 	)
 

federationapi/federationapi_test.go

@@ -5,11 +5,14 @@ import (
 	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
@@ -17,7 +20,10 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationapi/api"
@@ -362,3 +368,126 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 		}
 	}
 }
+
+func TestNotaryServer(t *testing.T) {
+	testCases := []struct {
+		name          string
+		httpBody      string
+		pubKeyRequest *gomatrixserverlib.PublicKeyNotaryLookupRequest
+		validateFunc  func(t *testing.T, response util.JSONResponse)
+	}{
+		{
+			name: "empty httpBody",
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusBadRequest, resp.Code)
+				nk, ok := resp.JSON.(spec.MatrixError)
+				assert.True(t, ok)
+				assert.Equal(t, spec.ErrorBadJSON, nk.ErrCode)
+			},
+		},
+		{
+			name:     "valid but empty httpBody",
+			httpBody: "{}",
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				want := util.JSONResponse{
+					Code: http.StatusOK,
+					JSON: routing.NotaryKeysResponse{ServerKeys: []json.RawMessage{}},
+				}
+				assert.Equal(t, want, resp)
+			},
+		},
+		{
+			name:     "request all keys using an empty criteria",
+			httpBody: `{"server_keys":{"servera":{}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request all keys using null as the criteria",
+			httpBody: `{"server_keys":{"servera":null}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request specific key",
+			httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
+				assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
+			},
+		},
+		{
+			name:     "request multiple servers",
+			httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}},"serverb":{"ed25519:someID":{}}}}`,
+			validateFunc: func(t *testing.T, resp util.JSONResponse) {
+				assert.Equal(t, http.StatusOK, resp.Code)
+				nk, ok := resp.JSON.(routing.NotaryKeysResponse)
+				assert.True(t, ok)
+				wantServers := map[string]struct{}{
+					"servera": {},
+					"serverb": {},
+				}
+				for _, js := range nk.ServerKeys {
+					serverName := gjson.GetBytes(js, "server_name").Str
+					_, ok = wantServers[serverName]
+					assert.True(t, ok, "unexpected servername: %s", serverName)
+					delete(wantServers, serverName)
+					assert.True(t, gjson.GetBytes(js, "verify_keys.ed25519:someID").Exists())
+				}
+				if len(wantServers) > 0 {
+					t.Fatalf("expected response to also contain: %#v", wantServers)
+				}
+			},
+		},
+	}
+
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
+		defer close()
+		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
+		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
+		natsInstance := jetstream.NATSInstance{}
+		fc := &fedClient{
+			keys: map[spec.ServerName]struct {
+				key   ed25519.PrivateKey
+				keyID gomatrixserverlib.KeyID
+			}{
+				"servera": {
+					key:   test.PrivateKeyA,
+					keyID: "ed25519:someID",
+				},
+				"serverb": {
+					key:   test.PrivateKeyB,
+					keyID: "ed25519:someID",
+				},
+			},
+		}
+
+		fedAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, nil, caches, nil, true)
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.httpBody))
+				req.Host = string(cfg.Global.ServerName)
+
+				resp := routing.NotaryKeys(req, &cfg.FederationAPI, fedAPI, tc.pubKeyRequest)
+				// assert that we received the expected response
+				tc.validateFunc(t, resp)
+			})
+		}
+
+	})
+}

federationapi/internal/api.go

@@ -112,7 +112,7 @@ func NewFederationInternalAPI(
 	}
 }
 
-func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) {
+func (a *FederationInternalAPI) IsBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) {
 	stats := a.statistics.ForServer(s)
 	if stats.Blacklisted() {
 		return stats, &api.FederationClientError{
@@ -151,7 +151,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti
 func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
 	s spec.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
-	stats, err := a.isBlacklistedOrBackingOff(s)
+	stats, err := a.IsBlacklistedOrBackingOff(s)
 	if err != nil {
 		return nil, err
 	}

federationapi/internal/federationclient_test.go

@@ -61,11 +61,11 @@ func TestFederationClientQueryKeys(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -92,11 +92,11 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -122,11 +122,11 @@ func TestFederationClientQueryKeysFailure(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{shouldFail: true}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -152,11 +152,11 @@ func TestFederationClientClaimKeys(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{
@@ -183,11 +183,11 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedapi := FederationInternalAPI{

federationapi/internal/perform.go

@@ -239,319 +239,6 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	return nil
 }
 
-// PerformMakeJoin implements api.FederationInternalAPI
-func (r *FederationInternalAPI) PerformMakeJoin(
-	ctx context.Context,
-	request *api.PerformJoinRequest,
-) (gomatrixserverlib.PDU, gomatrixserverlib.RoomVersion, spec.ServerName, error) {
-	// Check that a join isn't already in progress for this user/room.
-	j := federatedJoin{request.UserID, request.RoomID}
-	if _, found := r.joins.Load(j); found {
-		return nil, "", "", &gomatrix.HTTPError{
-			Code: 429,
-			Message: `{
-				"errcode": "M_LIMIT_EXCEEDED",
-				"error": "There is already a federated join to this room in progress. Please wait for it to finish."
-			}`, // TODO: Why do none of our error types play nicely with each other?
-		}
-	}
-	r.joins.Store(j, nil)
-	defer r.joins.Delete(j)
-
-	// Deduplicate the server names we were provided but keep the ordering
-	// as this encodes useful information about which servers are most likely
-	// to respond.
-	seenSet := make(map[spec.ServerName]bool)
-	var uniqueList []spec.ServerName
-	for _, srv := range request.ServerNames {
-		if seenSet[srv] || r.cfg.Matrix.IsLocalServerName(srv) {
-			continue
-		}
-		seenSet[srv] = true
-		uniqueList = append(uniqueList, srv)
-	}
-	request.ServerNames = uniqueList
-
-	// Try each server that we were provided until we land on one that
-	// successfully completes the make-join send-join dance.
-	var lastErr error
-	for _, serverName := range request.ServerNames {
-		var joinEvent gomatrixserverlib.PDU
-		var roomVersion gomatrixserverlib.RoomVersion
-		var err error
-		if joinEvent, roomVersion, _, err = r.performMakeJoinUsingServer(
-			ctx,
-			request.RoomID,
-			request.UserID,
-			request.Content,
-			serverName,
-		); err != nil {
-			logrus.WithError(err).WithFields(logrus.Fields{
-				"server_name": serverName,
-				"room_id":     request.RoomID,
-			}).Warnf("Failed to join room through server")
-			lastErr = err
-			continue
-		}
-
-		// We're all good.
-		return joinEvent, roomVersion, serverName, err
-	}
-
-	// If we reach here then we didn't complete a join for some reason.
-	var httpErr gomatrix.HTTPError
-	var lastError *gomatrix.HTTPError
-	if ok := errors.As(lastErr, &httpErr); ok {
-		httpErr.Message = string(httpErr.Contents)
-		lastError = &httpErr
-	} else {
-		lastError = &gomatrix.HTTPError{
-			Code:         0,
-			WrappedError: nil,
-			Message:      "Unknown HTTP error",
-		}
-		if lastErr != nil {
-			lastError.Message = lastErr.Error()
-		}
-	}
-
-	logrus.Errorf(
-		"failed to join user %q to room %q through %d server(s): last error %s",
-		request.UserID, request.RoomID, len(request.ServerNames), lastError,
-	)
-	return nil, "", "", lastError
-}
-
-func (r *FederationInternalAPI) performMakeJoinUsingServer(
-	ctx context.Context,
-	roomID, userID string,
-	content map[string]interface{},
-	serverName spec.ServerName,
-) (gomatrixserverlib.PDU, gomatrixserverlib.RoomVersion, spec.SenderID, error) {
-	if !r.shouldAttemptDirectFederation(serverName) {
-		return nil, "", "", fmt.Errorf("relay servers have no meaningful response for join.")
-	}
-
-	user, err := spec.NewUserID(userID, true)
-	if err != nil {
-		return nil, "", "", err
-	}
-	room, err := spec.NewRoomID(roomID)
-	if err != nil {
-		return nil, "", "", err
-	}
-
-	joinInput := gomatrixserverlib.PerformMakeJoinInput{
-		UserID:     user,
-		RoomID:     room,
-		ServerName: serverName,
-		Content:    content,
-		PrivateKey: r.cfg.Matrix.PrivateKey,
-		KeyID:      r.cfg.Matrix.KeyID,
-		KeyRing:    r.keyRing,
-		GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) {
-			// assign a roomNID, otherwise we can't create a private key for the user
-			_, nidErr := r.rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion))
-			if nidErr != nil {
-				return "", nil, nidErr
-			}
-			key, keyErr := r.rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID)
-			if keyErr != nil {
-				return "", nil, keyErr
-			}
-			return spec.SenderIDFromPseudoIDKey(key), key, nil
-		},
-	}
-	joinEvent, version, senderID, joinErr := gomatrixserverlib.PerformMakeJoin(ctx, r, joinInput)
-
-	if joinErr != nil {
-		if !joinErr.Reachable {
-			r.statistics.ForServer(joinErr.ServerName).Failure()
-		} else {
-			r.statistics.ForServer(joinErr.ServerName).Success(statistics.SendDirect)
-		}
-		return nil, "", "", joinErr.Err
-	}
-	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
-	if joinEvent == nil {
-		return nil, "", "", fmt.Errorf("Received nil joinEvent response from gomatrixserverlib.PerformJoin")
-	}
-
-	return joinEvent, version, senderID, nil
-}
-
-// PerformSendJoin implements api.FederationInternalAPI
-func (r *FederationInternalAPI) PerformSendJoin(
-	ctx context.Context,
-	request *api.PerformSendJoinRequestCryptoIDs,
-	response *api.PerformJoinResponse,
-) {
-	// Check that a join isn't already in progress for this user/room.
-	j := federatedJoin{request.UserID, request.RoomID}
-	if _, found := r.joins.Load(j); found {
-		response.LastError = &gomatrix.HTTPError{
-			Code: 429,
-			Message: `{
-				"errcode": "M_LIMIT_EXCEEDED",
-				"error": "There is already a federated join to this room in progress. Please wait for it to finish."
-			}`,
-		}
-		return
-	}
-	r.joins.Store(j, nil)
-	defer r.joins.Delete(j)
-
-	// Deduplicate the server names we were provided but keep the ordering
-	// as this encodes useful information about which servers are most likely
-	// to respond.
-	seenSet := make(map[spec.ServerName]bool)
-	var uniqueList []spec.ServerName
-	for _, srv := range request.ServerNames {
-		if seenSet[srv] || r.cfg.Matrix.IsLocalServerName(srv) {
-			continue
-		}
-		seenSet[srv] = true
-		uniqueList = append(uniqueList, srv)
-	}
-	request.ServerNames = uniqueList
-
-	// Try each server that we were provided until we land on one that
-	// successfully completes the make-join send-join dance.
-	var lastErr error
-	for _, serverName := range request.ServerNames {
-		if err := r.performSendJoinUsingServer(
-			ctx,
-			request.RoomID,
-			request.UserID,
-			request.Unsigned,
-			request.Event,
-			serverName,
-		); err != nil {
-			logrus.WithError(err).WithFields(logrus.Fields{
-				"server_name": serverName,
-				"room_id":     request.RoomID,
-			}).Warnf("Failed to join room through server")
-			lastErr = err
-			continue
-		}
-
-		// We're all good.
-		return
-	}
-
-	// If we reach here then we didn't complete a join for some reason.
-	var httpErr gomatrix.HTTPError
-	var lastError *gomatrix.HTTPError
-	if ok := errors.As(lastErr, &httpErr); ok {
-		httpErr.Message = string(httpErr.Contents)
-		lastError = &httpErr
-	} else {
-		lastError = &gomatrix.HTTPError{
-			Code:         0,
-			WrappedError: nil,
-			Message:      "Unknown HTTP error",
-		}
-		if lastErr != nil {
-			lastError.Message = lastErr.Error()
-		}
-	}
-
-	logrus.Errorf(
-		"failed to join user %q to room %q through %d server(s): last error %s",
-		request.UserID, request.RoomID, len(request.ServerNames), lastError,
-	)
-}
-
-func (r *FederationInternalAPI) performSendJoinUsingServer(
-	ctx context.Context,
-	roomID, userID string,
-	unsigned map[string]interface{},
-	event gomatrixserverlib.PDU,
-	serverName spec.ServerName,
-) error {
-	user, err := spec.NewUserID(userID, true)
-	if err != nil {
-		return err
-	}
-	room, err := spec.NewRoomID(roomID)
-	if err != nil {
-		return err
-	}
-	senderID, err := r.rsAPI.QuerySenderIDForUser(ctx, *room, *user)
-	if err != nil {
-		return err
-	}
-
-	joinInput := gomatrixserverlib.PerformSendJoinInput{
-		RoomID:      room,
-		ServerName:  serverName,
-		Unsigned:    unsigned,
-		Origin:      user.Domain(),
-		SenderID:    *senderID,
-		KeyRing:     r.keyRing,
-		Event:       event,
-		RoomVersion: event.Version(),
-		EventProvider: federatedEventProvider(ctx, r.federation, r.keyRing, user.Domain(), serverName, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
-			return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
-		}),
-		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
-			return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
-		},
-		StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error {
-			storeUserID, userErr := spec.NewUserID(userIDRaw, true)
-			if userErr != nil {
-				return userErr
-			}
-			return r.rsAPI.StoreUserRoomPublicKey(ctx, senderID, *storeUserID, roomID)
-		},
-	}
-	response, joinErr := gomatrixserverlib.PerformSendJoin(ctx, r, joinInput)
-	if joinErr != nil {
-		if !joinErr.Reachable {
-			r.statistics.ForServer(joinErr.ServerName).Failure()
-		} else {
-			r.statistics.ForServer(joinErr.ServerName).Success(statistics.SendDirect)
-		}
-		return joinErr.Err
-	}
-	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
-	if response == nil {
-		return fmt.Errorf("Received nil response from gomatrixserverlib.PerformSendJoin")
-	}
-
-	// We need to immediately update our list of joined hosts for this room now as we are technically
-	// joined. We must do this synchronously: we cannot rely on the roomserver output events as they
-	// will happen asyncly. If we don't update this table, you can end up with bad failure modes like
-	// joining a room, waiting for 200 OK then changing device keys and have those keys not be sent
-	// to other servers (this was a cause of a flakey sytest "Local device key changes get to remote servers")
-	// The events are trusted now as we performed auth checks above.
-	joinedHosts, err := consumers.JoinedHostsFromEvents(ctx, response.StateSnapshot.GetStateEvents().TrustedEvents(response.JoinEvent.Version(), false), r.rsAPI)
-	if err != nil {
-		return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
-	}
-
-	logrus.WithField("room", roomID).Infof("Joined federated room with %d hosts", len(joinedHosts))
-	if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil {
-		return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err)
-	}
-
-	// TODO: Can I change this to not take respState but instead just take an opaque list of events?
-	if err = roomserverAPI.SendEventWithState(
-		context.Background(),
-		r.rsAPI,
-		user.Domain(),
-		roomserverAPI.KindNew,
-		response.StateSnapshot,
-		&types.HeaderedEvent{PDU: response.JoinEvent},
-		serverName,
-		nil,
-		false,
-	); err != nil {
-		return fmt.Errorf("roomserverAPI.SendEventWithState: %w", err)
-	}
-	return nil
-}
-
 // PerformOutboundPeekRequest implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformOutboundPeek(
 	ctx context.Context,
@@ -746,7 +433,6 @@ func (r *FederationInternalAPI) PerformLeave(
 	ctx context.Context,
 	request *api.PerformLeaveRequest,
 	response *api.PerformLeaveResponse,
-	cryptoIDs bool,
 ) (err error) {
 	userID, err := spec.NewUserID(request.UserID, true)
 	if err != nil {
@@ -963,87 +649,6 @@ func (r *FederationInternalAPI) SendInviteV3(
 	return inviteEvent, nil
 }
 
-// MakeInviteCryptoIDs implements api.FederationInternalAPI
-func (r *FederationInternalAPI) MakeInviteCryptoIDs(ctx context.Context, event gomatrixserverlib.ProtoEvent, invitee spec.UserID, version gomatrixserverlib.RoomVersion, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error) {
-	validRoomID, err := spec.NewRoomID(event.RoomID)
-	if err != nil {
-		return nil, err
-	}
-	verImpl, err := gomatrixserverlib.GetRoomVersion(version)
-	if err != nil {
-		return nil, err
-	}
-
-	inviter, err := r.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(event.SenderID))
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO (devon): This should be allowed via a relay. Currently only transactions
-	// can be sent to relays. Would need to extend relays to handle invites.
-	if !r.shouldAttemptDirectFederation(invitee.Domain()) {
-		return nil, fmt.Errorf("relay servers have no meaningful response for invite.")
-	}
-
-	logrus.WithFields(logrus.Fields{
-		"user_id":      invitee.String(),
-		"room_id":      event.RoomID,
-		"room_version": version,
-		"destination":  invitee.Domain(),
-	}).Info("Sending /send_invite")
-
-	inviteReq, err := fclient.NewInviteV3Request(event, version, strippedState)
-	if err != nil {
-		return nil, fmt.Errorf("gomatrixserverlib.NewInviteV3Request: %w", err)
-	}
-
-	inviteRes, err := r.federation.MakeInviteCryptoIDs(ctx, inviter.Domain(), invitee.Domain(), inviteReq, invitee)
-	if err != nil {
-		return nil, fmt.Errorf("r.federation.SendInviteCryptoIDs: failed to send invite: %w", err)
-	}
-
-	inviteEvent, err := verImpl.NewEventFromUntrustedJSON(inviteRes.Event)
-	if err != nil {
-		return nil, fmt.Errorf("r.federation.SendInviteCryptoIDs failed to decode event response: %w", err)
-	}
-	return inviteEvent, nil
-}
-
-// SendInviteCryptoIDs implements api.FederationInternalAPI
-func (r *FederationInternalAPI) SendInviteCryptoIDs(ctx context.Context, event gomatrixserverlib.PDU, invitee spec.UserID, version gomatrixserverlib.RoomVersion) error {
-	validRoomID := event.RoomID()
-
-	inviter, err := r.rsAPI.QueryUserIDForSender(ctx, validRoomID, event.SenderID())
-	if err != nil {
-		return err
-	}
-
-	// TODO (devon): This should be allowed via a relay. Currently only transactions
-	// can be sent to relays. Would need to extend relays to handle invites.
-	if !r.shouldAttemptDirectFederation(invitee.Domain()) {
-		return fmt.Errorf("relay servers have no meaningful response for invite.")
-	}
-
-	logrus.WithFields(logrus.Fields{
-		"user_id":      invitee.String(),
-		"room_id":      event.RoomID,
-		"room_version": version,
-		"destination":  invitee.Domain(),
-	}).Info("Sending /make_invite")
-
-	inviteReq, err := fclient.NewSendInviteCryptoIDsRequest(event, version)
-	if err != nil {
-		return fmt.Errorf("gomatrixserverlib.NewInviteV3Request: %w", err)
-	}
-
-	err = r.federation.SendInviteCryptoIDs(ctx, inviter.Domain(), invitee.Domain(), inviteReq, invitee)
-	if err != nil {
-		return fmt.Errorf("r.federation.SendInviteCryptoIDs: failed to send invite: %w", err)
-	}
-
-	return nil
-}
-
 // PerformServersAlive implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformBroadcastEDU(
 	ctx context.Context,

federationapi/internal/perform_test.go

@@ -66,11 +66,11 @@ func TestPerformWakeupServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -112,11 +112,11 @@ func TestQueryRelayServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -153,11 +153,11 @@ func TestRemoveRelayServers(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -193,11 +193,11 @@ func TestPerformDirectoryLookup(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(
@@ -232,11 +232,11 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 		},
 	}
 	fedClient := &testFedClient{}
-	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
 	queues := queue.NewOutgoingQueues(
 		testDB, process.NewProcessContext(),
 		false,
-		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		cfg.Matrix.ServerName, fedClient, &stats,
 		nil,
 	)
 	fedAPI := NewFederationInternalAPI(

federationapi/internal/query.go

@@ -43,6 +43,15 @@ func (a *FederationInternalAPI) fetchServerKeysFromCache(
 	ctx context.Context, req *api.QueryServerKeysRequest,
 ) ([]gomatrixserverlib.ServerKeys, error) {
 	var results []gomatrixserverlib.ServerKeys
+
+	// We got a request for _all_ server keys, return them.
+	if len(req.KeyIDToCriteria) == 0 {
+		serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{})
+		if len(serverKeysResponses) == 0 {
+			return nil, fmt.Errorf("failed to find server key response for server %s", req.ServerName)
+		}
+		return serverKeysResponses, nil
+	}
 	for keyID, criteria := range req.KeyIDToCriteria {
 		serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{keyID})
 		if len(serverKeysResponses) == 0 {

federationapi/queue/destinationqueue.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/matrix-org/gomatrix"
@@ -26,12 +27,10 @@ import (
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -53,7 +52,6 @@ type destinationQueue struct {
 	db                 storage.Database
 	process            *process.ProcessContext
 	signing            map[spec.ServerName]*fclient.SigningIdentity
-	rsAPI              api.FederationRoomserverAPI
 	client             fclient.FederationClient        // federation client
 	origin             spec.ServerName                 // origin of requests
 	destination        spec.ServerName                 // destination of requests
@@ -296,6 +294,10 @@ func (oq *destinationQueue) checkNotificationsOnClose() {
 
 // backgroundSend is the worker goroutine for sending events.
 func (oq *destinationQueue) backgroundSend() {
+	// Don't try to send transactions if we are shutting down.
+	if oq.process.Context().Err() != nil {
+		return
+	}
 	// Check if a worker is already running, and if it isn't, then
 	// mark it as started.
 	if !oq.running.CompareAndSwap(false, true) {

federationapi/queue/queue.go

@@ -27,12 +27,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
-	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
-	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/process"
 )
@@ -43,7 +41,6 @@ type OutgoingQueues struct {
 	db          storage.Database
 	process     *process.ProcessContext
 	disabled    bool
-	rsAPI       api.FederationRoomserverAPI
 	origin      spec.ServerName
 	client      fclient.FederationClient
 	statistics  *statistics.Statistics
@@ -90,7 +87,6 @@ func NewOutgoingQueues(
 	disabled bool,
 	origin spec.ServerName,
 	client fclient.FederationClient,
-	rsAPI api.FederationRoomserverAPI,
 	statistics *statistics.Statistics,
 	signing []*fclient.SigningIdentity,
 ) *OutgoingQueues {
@@ -98,7 +94,6 @@ func NewOutgoingQueues(
 		disabled:   disabled,
 		process:    process,
 		db:         db,
-		rsAPI:      rsAPI,
 		origin:     origin,
 		client:     client,
 		statistics: statistics,
@@ -162,7 +157,6 @@ func (oqs *OutgoingQueues) getQueue(destination spec.ServerName) *destinationQue
 			queues:      oqs,
 			db:          oqs.db,
 			process:     oqs.process,
-			rsAPI:       oqs.rsAPI,
 			origin:      oqs.origin,
 			destination: destination,
 			client:      oqs.client,
@@ -213,18 +207,6 @@ func (oqs *OutgoingQueues) SendEvent(
 		delete(destmap, local)
 	}
 
-	// Check if any of the destinations are prohibited by server ACLs.
-	for destination := range destmap {
-		if api.IsServerBannedFromRoom(
-			oqs.process.Context(),
-			oqs.rsAPI,
-			ev.RoomID().String(),
-			destination,
-		) {
-			delete(destmap, destination)
-		}
-	}
-
 	// If there are no remaining destinations then give up.
 	if len(destmap) == 0 {
 		return nil
@@ -303,24 +285,6 @@ func (oqs *OutgoingQueues) SendEDU(
 		delete(destmap, local)
 	}
 
-	// There is absolutely no guarantee that the EDU will have a room_id
-	// field, as it is not required by the spec. However, if it *does*
-	// (e.g. typing notifications) then we should try to make sure we don't
-	// bother sending them to servers that are prohibited by the server
-	// ACLs.
-	if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() {
-		for destination := range destmap {
-			if api.IsServerBannedFromRoom(
-				oqs.process.Context(),
-				oqs.rsAPI,
-				result.Str,
-				destination,
-			) {
-				delete(destmap, destination)
-			}
-		}
-	}
-
 	// If there are no remaining destinations then give up.
 	if len(destmap) == 0 {
 		return nil

federationapi/queue/queue_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -26,7 +27,6 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
-	"go.uber.org/atomic"
 	"gotest.tools/v3/poll"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -34,7 +34,6 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/statistics"
 	"github.com/matrix-org/dendrite/federationapi/storage"
-	rsapi "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
@@ -65,15 +64,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
 	}
 }
 
-type stubFederationRoomServerAPI struct {
-	rsapi.FederationRoomserverAPI
-}
-
-func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error {
-	res.Banned = false
-	return nil
-}
-
 type stubFederationClient struct {
 	fclient.FederationClient
 	shouldTxSucceed      bool
@@ -123,12 +113,11 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 	fc := &stubFederationClient{
 		shouldTxSucceed:      shouldTxSucceed,
 		shouldTxRelaySucceed: shouldTxRelaySucceed,
-		txCount:              *atomic.NewUint32(0),
-		txRelayCount:         *atomic.NewUint32(0),
+		txCount:              atomic.Uint32{},
+		txRelayCount:         atomic.Uint32{},
 	}
-	rs := &stubFederationRoomServerAPI{}
 
-	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
+	stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline, false)
 	signingInfo := []*fclient.SigningIdentity{
 		{
 			KeyID:      "ed21019:auto",
@@ -136,7 +125,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
 			ServerName: "localhost",
 		},
 	}
-	queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, rs, &stats, signingInfo)
+	queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, &stats, signingInfo)
 
 	return db, fc, queues, processContext, close
 }

federationapi/routing/backfill.go

@@ -95,6 +95,12 @@ func Backfill(
 		}
 	}
 
+	// Enforce a limit of 100 events, as not to hit the DB to hard.
+	// Synapse has a hard limit of 100 events as well.
+	if req.Limit > 100 {
+		req.Limit = 100
+	}
+
 	// Query the Roomserver.
 	if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed")

federationapi/routing/keys.go

@@ -197,6 +197,10 @@ func localKeys(cfg *config.FederationAPI, serverName spec.ServerName) (*gomatrix
 	return &keys, err
 }
 
+type NotaryKeysResponse struct {
+	ServerKeys []json.RawMessage `json:"server_keys"`
+}
+
 func NotaryKeys(
 	httpReq *http.Request, cfg *config.FederationAPI,
 	fsAPI federationAPI.FederationInternalAPI,
@@ -217,10 +221,9 @@ func NotaryKeys(
 		}
 	}
 
-	var response struct {
-		ServerKeys []json.RawMessage `json:"server_keys"`
+	response := NotaryKeysResponse{
+		ServerKeys: []json.RawMessage{},
 	}
-	response.ServerKeys = []json.RawMessage{}
 
 	for serverName, kidToCriteria := range req.ServerKeys {
 		var keyList []gomatrixserverlib.ServerKeys

federationapi/routing/profile_test.go

@@ -26,7 +26,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -67,11 +66,8 @@ func TestHandleQueryProfile(t *testing.T) {
 		keyRing := serverKeyAPI.KeyRing()
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
-		if !ok {
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
+
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/query.go

@@ -146,7 +146,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques
 	}
 
 	walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1)
-	discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1)
+	discoveredRooms, inaccessibleRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1)
 
 	if err != nil {
 		switch err.(type) {
@@ -177,6 +177,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques
 		JSON: fclient.RoomHierarchyResponse{
 			Room:                 discoveredRooms[0],
 			Children:             discoveredRooms[1:],
+			InaccessibleChildren: inaccessibleRooms,
 		},
 	}
 }

federationapi/routing/query_test.go

@@ -25,7 +25,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -65,11 +64,8 @@ func TestHandleQueryDirectory(t *testing.T) {
 		keyRing := serverKeyAPI.KeyRing()
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true)
 		userapi := fakeUserAPI{}
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
-		if !ok {
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
+
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/routing/routing.go

@@ -647,6 +647,8 @@ func MakeFedAPI(
 		// add the user to Sentry, if enabled
 		hub := sentry.GetHubFromContext(req.Context())
 		if hub != nil {
+			// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
+			hub = hub.Clone()
 			hub.Scope().SetTag("origin", string(fedReq.Origin()))
 			hub.Scope().SetTag("uri", fedReq.RequestURI())
 		}

federationapi/routing/send_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
 	fedAPI "github.com/matrix-org/dendrite/federationapi"
-	fedInternal "github.com/matrix-org/dendrite/federationapi/internal"
 	"github.com/matrix-org/dendrite/federationapi/routing"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
@@ -62,11 +61,8 @@ func TestHandleSend(t *testing.T) {
 		fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true)
 		serverKeyAPI := &signing.YggdrasilKeys{}
 		keyRing := serverKeyAPI.KeyRing()
-		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
-		if !ok {
-			panic("This is a programming error.")
-		}
-		routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics)
+
+		routing.Setup(routers, cfg, nil, fedapi, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics)
 
 		handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
 		_, sk, _ := ed25519.GenerateKey(nil)

federationapi/statistics/statistics.go

@@ -5,10 +5,10 @@ import (
 	"math"
 	"math/rand"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
 
 	"github.com/matrix-org/dendrite/federationapi/storage"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -34,12 +34,15 @@ type Statistics struct {
 	// mark the destination as offline. At this point we should attempt
 	// to send messages to the user's async relay servers if we know them.
 	FailuresUntilAssumedOffline uint32
+
+	enableRelays bool
 }
 
 func NewStatistics(
 	db storage.Database,
 	failuresUntilBlacklist uint32,
 	failuresUntilAssumedOffline uint32,
+	enableRelays bool,
 ) Statistics {
 	return Statistics{
 		DB:                          db,
@@ -47,6 +50,7 @@ func NewStatistics(
 		FailuresUntilAssumedOffline: failuresUntilAssumedOffline,
 		backoffTimers:               make(map[spec.ServerName]*time.Timer),
 		servers:                     make(map[spec.ServerName]*ServerStatistics),
+		enableRelays:                enableRelays,
 	}
 }
 
@@ -73,6 +77,13 @@ func (s *Statistics) ForServer(serverName spec.ServerName) *ServerStatistics {
 		} else {
 			server.blacklisted.Store(blacklisted)
 		}
+
+		// Don't bother hitting the database 2 additional times
+		// if we don't want to use relays.
+		if !s.enableRelays {
+			return server
+		}
+
 		assumedOffline, err := s.DB.IsServerAssumedOffline(context.Background(), serverName)
 		if err != nil {
 			logrus.WithError(err).Errorf("Failed to get assumed offline entry %q", serverName)
@@ -158,7 +169,7 @@ func (s *ServerStatistics) Success(method SendMethod) {
 	// NOTE : Sending to the final destination vs. a relay server has
 	// slightly different semantics.
 	if method == SendDirect {
-		s.successCounter.Inc()
+		s.successCounter.Add(1)
 		if s.blacklisted.Load() && s.statistics.DB != nil {
 			if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil {
 				logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName)
@@ -184,7 +195,7 @@ func (s *ServerStatistics) Failure() (time.Time, bool) {
 	// start a goroutine which will wait out the backoff and
 	// unset the backoffStarted flag when done.
 	if s.backoffStarted.CompareAndSwap(false, true) {
-		backoffCount := s.backoffCount.Inc()
+		backoffCount := s.backoffCount.Add(1)
 
 		if backoffCount >= s.statistics.FailuresUntilAssumedOffline {
 			s.assumedOffline.CompareAndSwap(false, true)

federationapi/statistics/statistics_test.go

@@ -16,7 +16,7 @@ const (
 )
 
 func TestBackoff(t *testing.T) {
-	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	server := ServerStatistics{
 		statistics: &stats,
 		serverName: "test.com",
@@ -106,7 +106,7 @@ func TestBackoff(t *testing.T) {
 }
 
 func TestRelayServersListing(t *testing.T) {
-	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
 	server := ServerStatistics{statistics: &stats}
 	server.AddRelayServers([]spec.ServerName{"relay1", "relay1", "relay2"})
 	relayServers := server.KnownRelayServers()

federationapi/storage/postgres/notary_server_keys_metadata_table.go

@@ -151,7 +151,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn
 		}
 		results = append(results, sk)
 	}
-	return results, nil
+	return results, rows.Err()
 }
 
 func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error {

federationapi/storage/postgres/queue_json_table.go

@@ -109,5 +109,5 @@ func (s *queueJSONStatements) SelectQueueJSON(
 		}
 		blobs[nid] = blob
 	}
-	return blobs, err
+	return blobs, rows.Err()
 }

federationapi/storage/postgres/relay_servers_table.go

@@ -110,7 +110,7 @@ func (s *relayServersStatements) SelectRelayServers(
 		}
 		result = append(result, spec.ServerName(relayServer))
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *relayServersStatements) DeleteRelayServers(

federationapi/storage/postgres/server_key_table.go

@@ -94,12 +94,14 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 	}
 	defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed")
 	results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
-	for rows.Next() {
+
 	var serverName string
 	var keyID string
 	var key string
 	var validUntilTS int64
 	var expiredTS int64
+	var vk gomatrixserverlib.VerifyKey
+	for rows.Next() {
 		if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil {
 			return nil, err
 		}
@@ -107,7 +109,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 			ServerName: spec.ServerName(serverName),
 			KeyID:      gomatrixserverlib.KeyID(keyID),
 		}
-		vk := gomatrixserverlib.VerifyKey{}
 		err = vk.Key.Decode(key)
 		if err != nil {
 			return nil, err

federationapi/storage/sqlite3/joined_hosts_table.go

@@ -216,5 +216,5 @@ func joinedHostsFromStmt(
 		})
 	}
 
-	return result, nil
+	return result, rows.Err()
 }

federationapi/storage/sqlite3/notary_server_keys_metadata_table.go

@@ -154,7 +154,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn
 		}
 		results = append(results, sk)
 	}
-	return results, nil
+	return results, rows.Err()
 }
 
 func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error {

federationapi/storage/sqlite3/queue_json_table.go

@@ -135,5 +135,5 @@ func (s *queueJSONStatements) SelectQueueJSON(
 		}
 		blobs[nid] = blob
 	}
-	return blobs, err
+	return blobs, rows.Err()
 }

federationapi/storage/sqlite3/relay_servers_table.go

@@ -109,7 +109,7 @@ func (s *relayServersStatements) SelectRelayServers(
 		}
 		result = append(result, spec.ServerName(relayServer))
 	}
-	return result, nil
+	return result, rows.Err()
 }
 
 func (s *relayServersStatements) DeleteRelayServers(

federationapi/storage/sqlite3/server_key_table.go

@@ -98,12 +98,13 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 	err := sqlutil.RunLimitedVariablesQuery(
 		ctx, bulkSelectServerSigningKeysSQL, s.db, iKeyIDs, sqlutil.SQLite3MaxVariables,
 		func(rows *sql.Rows) error {
-			for rows.Next() {
 			var serverName string
 			var keyID string
 			var key string
 			var validUntilTS int64
 			var expiredTS int64
+			var vk gomatrixserverlib.VerifyKey
+			for rows.Next() {
 				if err := rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil {
 					return fmt.Errorf("bulkSelectServerKeys: %v", err)
 				}
@@ -111,7 +112,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys(
 					ServerName: spec.ServerName(serverName),
 					KeyID:      gomatrixserverlib.KeyID(keyID),
 				}
-				vk := gomatrixserverlib.VerifyKey{}
 				err := vk.Key.Decode(key)
 				if err != nil {
 					return fmt.Errorf("bulkSelectServerKeys: %v", err)

federationapi/storage/tables/server_key_table_test.go

@@ -0,0 +1,116 @@
+package tables_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/postgres"
+	"github.com/matrix-org/dendrite/federationapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/federationapi/storage/tables"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/stretchr/testify/assert"
+)
+
+func mustCreateServerKeyDB(t *testing.T, dbType test.DBType) (tables.FederationServerSigningKeys, func()) {
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	if err != nil {
+		t.Fatalf("failed to open database: %s", err)
+	}
+	var tab tables.FederationServerSigningKeys
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresServerSigningKeysTable(db)
+	case test.DBTypeSQLite:
+		tab, err = sqlite3.NewSQLiteServerSigningKeysTable(db)
+	}
+	if err != nil {
+		t.Fatalf("failed to create table: %s", err)
+	}
+	return tab, close
+}
+
+func TestServerKeysTable(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		ctx, cancel := context.WithCancel(context.Background())
+		tab, close := mustCreateServerKeyDB(t, dbType)
+		t.Cleanup(func() {
+			close()
+			cancel()
+		})
+
+		req := gomatrixserverlib.PublicKeyLookupRequest{
+			ServerName: "localhost",
+			KeyID:      "ed25519:test",
+		}
+		expectedTimestamp := spec.AsTimestamp(time.Now().Add(time.Hour))
+		res := gomatrixserverlib.PublicKeyLookupResult{
+			VerifyKey:    gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)},
+			ExpiredTS:    0,
+			ValidUntilTS: expectedTimestamp,
+		}
+
+		// Insert the key
+		err := tab.UpsertServerKeys(ctx, nil, req, res)
+		assert.NoError(t, err)
+
+		selectKeys := map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{
+			req: spec.AsTimestamp(time.Now()),
+		}
+		gotKeys, err := tab.BulkSelectServerKeys(ctx, nil, selectKeys)
+		assert.NoError(t, err)
+
+		// Now we should have a key for the req above
+		assert.NotNil(t, gotKeys[req])
+		assert.Equal(t, res, gotKeys[req])
+
+		// "Expire" the key by setting ExpireTS to a non-zero value and ValidUntilTS to 0
+		expectedTimestamp = spec.AsTimestamp(time.Now())
+		res.ExpiredTS = expectedTimestamp
+		res.ValidUntilTS = 0
+
+		// Update the key
+		err = tab.UpsertServerKeys(ctx, nil, req, res)
+		assert.NoError(t, err)
+
+		gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys)
+		assert.NoError(t, err)
+
+		// The key should be expired
+		assert.NotNil(t, gotKeys[req])
+		assert.Equal(t, res, gotKeys[req])
+
+		// Upsert a different key to validate querying multiple keys
+		req2 := gomatrixserverlib.PublicKeyLookupRequest{
+			ServerName: "notlocalhost",
+			KeyID:      "ed25519:test2",
+		}
+		expectedTimestamp2 := spec.AsTimestamp(time.Now().Add(time.Hour))
+		res2 := gomatrixserverlib.PublicKeyLookupResult{
+			VerifyKey:    gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)},
+			ExpiredTS:    0,
+			ValidUntilTS: expectedTimestamp2,
+		}
+
+		err = tab.UpsertServerKeys(ctx, nil, req2, res2)
+		assert.NoError(t, err)
+
+		// Select multiple keys
+		selectKeys[req2] = spec.AsTimestamp(time.Now())
+
+		gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys)
+		assert.NoError(t, err)
+
+		// We now should receive two keys, one of which is expired
+		assert.Equal(t, 2, len(gotKeys))
+		assert.Equal(t, res2, gotKeys[req2])
+		assert.Equal(t, res, gotKeys[req])
+	})
+}

go.mod

@@ -1,23 +1,20 @@
 module github.com/matrix-org/dendrite
 
-//replace github.com/matrix-org/gomatrixserverlib => ../../gomatrixserverlib/crypto-ids/
-
-//replace github.com/matrix-org/gomatrixserverlib => /src/gmsl/
-
 require (
 	github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2
 	github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/MFAshby/stdemuxerhook v1.0.0
 	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/blevesearch/bleve/v2 v2.3.8
+	github.com/blevesearch/bleve/v2 v2.4.0
 	github.com/codeclysm/extract v2.2.0+incompatible
 	github.com/dgraph-io/ristretto v0.1.1
-	github.com/docker/docker v24.0.5+incompatible
+	github.com/docker/docker v24.0.9+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
+	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/gologme/log v1.3.0
-	github.com/google/go-cmp v0.5.9
+	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
@@ -26,66 +23,70 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20231219232834-bbfb4a048862
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5
 	github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
-	github.com/mattn/go-sqlite3 v1.14.17
-	github.com/nats-io/nats-server/v2 v2.9.19
-	github.com/nats-io/nats.go v1.27.0
+	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/nats-io/nats-server/v2 v2.10.7
+	github.com/nats-io/nats.go v1.31.0
 	github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_golang v1.19.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.2
-	github.com/tidwall/gjson v1.17.0
+	github.com/stretchr/testify v1.9.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/tidwall/sjson v1.2.5
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
-	go.uber.org/atomic v1.10.0
-	golang.org/x/crypto v0.17.0
-	golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
-	golang.org/x/image v0.5.0
+	go.uber.org/atomic v1.11.0
+	golang.org/x/crypto v0.22.0
+	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678
+	golang.org/x/image v0.10.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
-	golang.org/x/sync v0.3.0
-	golang.org/x/term v0.15.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/term v0.19.0
 	gopkg.in/h2non/bimg.v1 v1.1.9
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.4.0
 	maunium.net/go/mautrix v0.15.1
-	modernc.org/sqlite v1.23.1
+	modernc.org/sqlite v1.29.5
 	nhooyr.io/websocket v1.8.7
 )
 
 require (
+	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/RoaringBitmap/roaring v1.2.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.5.0 // indirect
-	github.com/blevesearch/bleve_index_api v1.0.5 // indirect
-	github.com/blevesearch/geo v0.1.17 // indirect
+	github.com/blevesearch/bleve_index_api v1.1.6 // indirect
+	github.com/blevesearch/geo v0.1.20 // indirect
+	github.com/blevesearch/go-faiss v1.0.13 // indirect
 	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
 	github.com/blevesearch/gtreap v0.1.1 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
-	github.com/blevesearch/scorch_segment_api/v2 v2.1.4 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.2.9 // indirect
 	github.com/blevesearch/segment v0.9.1 // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
 	github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
-	github.com/blevesearch/vellum v1.0.9 // indirect
-	github.com/blevesearch/zapx/v11 v11.3.7 // indirect
-	github.com/blevesearch/zapx/v12 v12.3.7 // indirect
-	github.com/blevesearch/zapx/v13 v13.3.7 // indirect
-	github.com/blevesearch/zapx/v14 v14.3.7 // indirect
-	github.com/blevesearch/zapx/v15 v15.3.10 // indirect
+	github.com/blevesearch/vellum v1.0.10 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.10 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.10 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.10 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.10 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.13 // indirect
+	github.com/blevesearch/zapx/v16 v16.0.12 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
@@ -95,56 +96,52 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v1.0.0 // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/nats-io/jwt/v2 v2.4.1 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
+	github.com/nats-io/jwt/v2 v2.5.5 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
-	github.com/quic-go/quic-go v0.37.4 // indirect
+	github.com/quic-go/quic-go v0.37.7 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/zerolog v1.29.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.12.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/macaroon.v2 v2.1.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
 	maunium.net/go/maulogger/v2 v2.4.1 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.5 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.0.1 // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	modernc.org/libc v1.41.0 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
 )
 
 go 1.20

go.sum

@@ -5,6 +5,8 @@ github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
+github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
@@ -36,38 +38,42 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
 github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/blevesearch/bleve/v2 v2.3.8 h1:IqFyMJ73n4gY8AmVqM8Sa6EtAZ5beE8yramVqCvs2kQ=
-github.com/blevesearch/bleve/v2 v2.3.8/go.mod h1:Lh9aZEHrLKxwPnW4z4lsBEGnflZQ1V/aWP/t+htsiDw=
-github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw=
-github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
-github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA=
-github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
+github.com/blevesearch/bleve/v2 v2.4.0 h1:2xyg+Wv60CFHYccXc+moGxbL+8QKT/dZK09AewHgKsg=
+github.com/blevesearch/bleve/v2 v2.4.0/go.mod h1:IhQHoFAbHgWKYavb9rQgQEJJVMuY99cKdQ0wPpst2aY=
+github.com/blevesearch/bleve_index_api v1.1.6 h1:orkqDFCBuNU2oHW9hN2YEJmet+TE9orml3FCGbl1cKk=
+github.com/blevesearch/bleve_index_api v1.1.6/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
+github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
+github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
+github.com/blevesearch/go-faiss v1.0.13 h1:zfFs7ZYD0NqXVSY37j0JZjZT1BhE9AE4peJfcx/NB4A=
+github.com/blevesearch/go-faiss v1.0.13/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
 github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
 github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
 github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.9 h1:3nBaSBRFokjE4FtPW3eUDgcAu3KphBg1GP07zy/6Uyk=
+github.com/blevesearch/scorch_segment_api/v2 v2.2.9/go.mod h1:ckbeb7knyOOvAdZinn/ASbB7EA3HoagnJkmEV3J7+sg=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
 github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
 github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
-github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ=
-github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
-github.com/blevesearch/zapx/v11 v11.3.7 h1:Y6yIAF/DVPiqZUA/jNgSLXmqewfzwHzuwfKyfdG+Xaw=
-github.com/blevesearch/zapx/v11 v11.3.7/go.mod h1:Xk9Z69AoAWIOvWudNDMlxJDqSYGf90LS0EfnaAIvXCA=
-github.com/blevesearch/zapx/v12 v12.3.7 h1:DfQ6rsmZfEK4PzzJJRXjiM6AObG02+HWvprlXQ1Y7eI=
-github.com/blevesearch/zapx/v12 v12.3.7/go.mod h1:SgEtYIBGvM0mgIBn2/tQE/5SdrPXaJUaT/kVqpAPxm0=
-github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ubM4d2l8=
-github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s=
-github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20=
-github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w=
-github.com/blevesearch/zapx/v15 v15.3.10 h1:bQ9ZxJCj6rKp873EuVJu2JPxQ+EWQZI1cjJGeroovaQ=
-github.com/blevesearch/zapx/v15 v15.3.10/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY=
+github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
+github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
+github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
+github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
+github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
+github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
+github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
+github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
+github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
+github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
+github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
+github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
+github.com/blevesearch/zapx/v16 v16.0.12 h1:Uccxvjmn+hQ6ywQP+wIiTpdq9LnAviGoryJOmGwAo/I=
+github.com/blevesearch/zapx/v16 v16.0.12/go.mod h1:MYnOshRfSm4C4drxx1LGRI+MVFByykJ2anDY1fxdk9Q=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
@@ -89,8 +95,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY=
-github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
+github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -111,8 +117,12 @@ github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwv
 github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8=
 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
+github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
+github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -141,7 +151,6 @@ github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -158,8 +167,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c=
 github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
@@ -174,6 +183,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -185,13 +196,11 @@ github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0=
 github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
-github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -208,8 +217,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231219232834-bbfb4a048862 h1:Kuya3qas85ZvVVkuOpemwhgvdJbLojvwvt3xyJTp1dY=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20231219232834-bbfb4a048862/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5 h1:GuxmpyjZQoqb6UFQgKq8Td3wIITlXln/sItqp1jbTTA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20240328203753-c2391f7113a5/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@@ -222,10 +231,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
@@ -242,16 +249,18 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
-github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
-github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
-github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA=
-github.com/nats-io/nats-server/v2 v2.9.19/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw=
-github.com/nats-io/nats.go v1.27.0 h1:3o9fsPhmoKm+yK7rekH2GtWoE+D9jFbw8N3/ayI1C00=
-github.com/nats-io/nats.go v1.27.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
-github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
-github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
+github.com/nats-io/jwt/v2 v2.5.5 h1:ROfXb50elFq5c9+1ztaUbdlrArNFl2+fQWP6B8HGEq4=
+github.com/nats-io/jwt/v2 v2.5.5/go.mod h1:ZdWS1nZa6WMZfFwwgpEaqBV8EPGVgOTDHN/wTbz0Y5A=
+github.com/nats-io/nats-server/v2 v2.10.7 h1:f5VDy+GMu7JyuFA0Fef+6TfulfCs5nBTgq7MMkFJx5Y=
+github.com/nats-io/nats-server/v2 v2.10.7/go.mod h1:V2JHOvPiPdtfDXTuEUsthUnCvSDeFrK4Xn9hRo6du7c=
+github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
+github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
@@ -276,23 +285,21 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
-github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
-github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
-github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
 github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4=
-github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU=
+github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
@@ -304,22 +311,18 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
@@ -343,10 +346,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
+go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -354,20 +357,20 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U=
-golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI=
-golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4=
+golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M=
+golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4=
 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY=
@@ -376,8 +379,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -385,29 +389,31 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -418,22 +424,26 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -445,8 +455,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
-golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -457,8 +468,8 @@ gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6d
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -478,34 +489,25 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
-lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
-lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8=
 maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho=
 maunium.net/go/mautrix v0.15.1 h1:pmCtMjYRpd83+2UL+KTRFYQo5to0373yulimvLK+1k0=
 maunium.net/go/mautrix v0.15.1/go.mod h1:icQIrvz2NldkRLTuzSGzmaeuMUmw+fzO7UVycPeauN8=
-modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
-modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
-modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
-modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
-modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
-modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
-modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
-modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
-modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
-modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
-modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
-modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
+modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.29.5 h1:8l/SQKAjDtZFo9lkJLdk8g9JEOeYRG4/ghStDCCTiTE=
+modernc.org/sqlite v1.29.5/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

helm/dendrite/Chart.yaml

@@ -1,9 +1,10 @@
 apiVersion: v2
 name: dendrite
-version: "0.13.4"
-appVersion: "0.13.3"
+version: "0.14.1"
+appVersion: "0.13.7"
 description: Dendrite Matrix Homeserver
 type: application
+icon: https://avatars.githubusercontent.com/u/8418310?s=48&v=4
 keywords:
   - matrix
   - chat
@@ -13,7 +14,7 @@ home: https://github.com/matrix-org/dendrite
 sources:
   - https://github.com/matrix-org/dendrite
 dependencies:
-- name: postgresql
-  version: 12.1.7
+  - name: postgresql
+    version: 14.2.3
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled

helm/dendrite/README.md

@@ -1,7 +1,7 @@
 
 # dendrite
 
-![Version: 0.13.4](https://img.shields.io/badge/Version-0.13.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.3](https://img.shields.io/badge/AppVersion-0.13.3-informational?style=flat-square)
+![Version: 0.14.0](https://img.shields.io/badge/Version-0.14.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.7](https://img.shields.io/badge/AppVersion-0.13.7-informational?style=flat-square)
 Dendrite Matrix Homeserver
 
 Status: **NOT PRODUCTION READY**
@@ -37,7 +37,7 @@ Create a folder `appservices` and place your configurations in there.  The confi
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 |
+| https://charts.bitnami.com/bitnami | postgresql | 14.2.3 |
 ## Values
 
 | Key | Type | Default | Description |
@@ -48,19 +48,19 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | signing_key.create | bool | `true` | Create a new signing key, if not exists |
 | signing_key.existingSecret | string | `""` | Use an existing secret |
 | resources | object | sets some sane default values | Default resource requests/limits. |
-| persistence.storageClass | string | `""` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. |
+| persistence.storageClass | string | `nil` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack)  |
 | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream |
 | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume |
-| persistence.jetstream.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass |
+| persistence.jetstream.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files |
 | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume |
-| persistence.media.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass |
+| persistence.media.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index |
 | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume |
-| persistence.search.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass |
+| persistence.search.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is   set, choosing the default provisioner.  (gp2 on AWS, standard on   GKE, AWS & OpenStack) |
 | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod |
 | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod |
-| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
+| strategy.type | string | `"Recreate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |
 | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process |
 | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods |
 | dendrite_config.version | int | `2` |  |
@@ -142,7 +142,7 @@ Create a folder `appservices` and place your configurations in there.  The confi
 | dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration |
 | postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. |
 | postgresql.image.repository | string | `"bitnami/postgresql"` |  |
-| postgresql.image.tag | string | `"15.1.0"` |  |
+| postgresql.image.tag | string | `"16.2.0"` |  |
 | postgresql.auth.username | string | `"dendrite"` |  |
 | postgresql.auth.password | string | `"changeme"` |  |
 | postgresql.auth.database | string | `"dendrite"` |  |
@@ -189,3 +189,5 @@ grafana:
 ```
 PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`)
 
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

helm/dendrite/grafana_dashboards/dendrite-rev2.json

@@ -119,7 +119,7 @@
           "refId": "A"
         }
       ],
-      "title": "Registerd Users",
+      "title": "Registered Users",
       "type": "stat"
     },
     {

helm/dendrite/templates/ingress.yaml

@@ -4,6 +4,7 @@
 {{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}}
 {{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}}
 {{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}}
+
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
@@ -56,7 +57,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -72,7 +73,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -88,7 +89,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http
@@ -105,7 +106,7 @@ spec:
               service:
                 name: {{ $fullName }}
                 port:
-                  name: http
+                  number: {{ $.Values.service.port }}
               {{- else }}
               serviceName: {{ $fullName }}
               servicePort: http